diff --git a/Documentation/00-INDEX b/Documentation/00-INDEX index 3acc4f1a6f8420..4a5a887181c30c 100644 --- a/Documentation/00-INDEX +++ b/Documentation/00-INDEX @@ -436,6 +436,8 @@ sysrq.txt - info on the magic SysRq key. target/ - directory with info on generating TCM v4 fabric .ko modules +tee.txt + - info on the TEE subsystem and drivers this_cpu_ops.txt - List rationale behind and the way to use this_cpu operations. thermal/ diff --git a/Documentation/ABI/testing/sysfs-class-dual-role-usb b/Documentation/ABI/testing/sysfs-class-dual-role-usb new file mode 100644 index 00000000000000..a900fd75430c66 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-class-dual-role-usb @@ -0,0 +1,71 @@ +What: /sys/class/dual_role_usb/.../ +Date: June 2015 +Contact: Badhri Jagan Sridharan +Description: + Provide a generic interface to monitor and change + the state of dual role usb ports. The name here + refers to the name mentioned in the + dual_role_phy_desc that is passed while registering + the dual_role_phy_intstance through + devm_dual_role_instance_register. + +What: /sys/class/dual_role_usb/.../supported_modes +Date: June 2015 +Contact: Badhri Jagan Sridharan +Description: + This is a static node, once initialized this + is not expected to change during runtime. "dfp" + refers to "downstream facing port" i.e. port can + only act as host. "ufp" refers to "upstream + facing port" i.e. port can only act as device. + "dfp ufp" refers to "dual role port" i.e. the port + can either be a host port or a device port. + +What: /sys/class/dual_role_usb/.../mode +Date: June 2015 +Contact: Badhri Jagan Sridharan +Description: + The mode node refers to the current mode in which the + port is operating. "dfp" for host ports. "ufp" for device + ports and "none" when cable is not connected. + + On devices where the USB mode is software-controllable, + userspace can change the mode by writing "dfp" or "ufp". + On devices where the USB mode is fixed in hardware, + this attribute is read-only. + +What: /sys/class/dual_role_usb/.../power_role +Date: June 2015 +Contact: Badhri Jagan Sridharan +Description: + The power_role node mentions whether the port + is "sink"ing or "source"ing power. "none" if + they are not connected. + + On devices implementing USB Power Delivery, + userspace can control the power role by writing "sink" or + "source". On devices without USB-PD, this attribute is + read-only. + +What: /sys/class/dual_role_usb/.../data_role +Date: June 2015 +Contact: Badhri Jagan Sridharan +Description: + The data_role node mentions whether the port + is acting as "host" or "device" for USB data connection. + "none" if there is no active data link. + + On devices implementing USB Power Delivery, userspace + can control the data role by writing "host" or "device". + On devices without USB-PD, this attribute is read-only + +What: /sys/class/dual_role_usb/.../powers_vconn +Date: June 2015 +Contact: Badhri Jagan Sridharan +Description: + The powers_vconn node mentions whether the port + is supplying power for VCONN pin. + + On devices with software control of VCONN, + userspace can disable the power supply to VCONN by writing "n", + or enable the power supply by writing "y". diff --git a/Documentation/ABI/testing/sysfs-kernel-wakeup_reasons b/Documentation/ABI/testing/sysfs-kernel-wakeup_reasons new file mode 100644 index 00000000000000..acb19b91c192b3 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-kernel-wakeup_reasons @@ -0,0 +1,16 @@ +What: /sys/kernel/wakeup_reasons/last_resume_reason +Date: February 2014 +Contact: Ruchi Kandoi +Description: + The /sys/kernel/wakeup_reasons/last_resume_reason is + used to report wakeup reasons after system exited suspend. + +What: /sys/kernel/wakeup_reasons/last_suspend_time +Date: March 2015 +Contact: jinqian +Description: + The /sys/kernel/wakeup_reasons/last_suspend_time is + used to report time spent in last suspend cycle. It contains + two numbers (in seconds) separated by space. First number is + the time spent in suspend and resume processes. Second number + is the time spent in sleep state. \ No newline at end of file diff --git a/Documentation/android.txt b/Documentation/android.txt new file mode 100644 index 00000000000000..0f40a78b045f77 --- /dev/null +++ b/Documentation/android.txt @@ -0,0 +1,121 @@ + ============= + A N D R O I D + ============= + +Copyright (C) 2009 Google, Inc. +Written by Mike Chan + +CONTENTS: +--------- + +1. Android + 1.1 Required enabled config options + 1.2 Required disabled config options + 1.3 Recommended enabled config options +2. Contact + + +1. Android +========== + +Android (www.android.com) is an open source operating system for mobile devices. +This document describes configurations needed to run the Android framework on +top of the Linux kernel. + +To see a working defconfig look at msm_defconfig or goldfish_defconfig +which can be found at http://android.git.kernel.org in kernel/common.git +and kernel/msm.git + + +1.1 Required enabled config options +----------------------------------- +After building a standard defconfig, ensure that these options are enabled in +your .config or defconfig if they are not already. Based off the msm_defconfig. +You should keep the rest of the default options enabled in the defconfig +unless you know what you are doing. + +ANDROID_PARANOID_NETWORK +ASHMEM +CONFIG_FB_MODE_HELPERS +CONFIG_FONT_8x16 +CONFIG_FONT_8x8 +CONFIG_YAFFS_SHORT_NAMES_IN_RAM +DAB +EARLYSUSPEND +FB +FB_CFB_COPYAREA +FB_CFB_FILLRECT +FB_CFB_IMAGEBLIT +FB_DEFERRED_IO +FB_TILEBLITTING +HIGH_RES_TIMERS +INOTIFY +INOTIFY_USER +INPUT_EVDEV +INPUT_GPIO +INPUT_MISC +LEDS_CLASS +LEDS_GPIO +LOCK_KERNEL +LkOGGER +LOW_MEMORY_KILLER +MISC_DEVICES +NEW_LEDS +NO_HZ +POWER_SUPPLY +PREEMPT +RAMFS +RTC_CLASS +RTC_LIB +SWITCH +SWITCH_GPIO +TMPFS +UID_STAT +UID16 +USB_FUNCTION +USB_FUNCTION_ADB +USER_WAKELOCK +VIDEO_OUTPUT_CONTROL +WAKELOCK +YAFFS_AUTO_YAFFS2 +YAFFS_FS +YAFFS_YAFFS1 +YAFFS_YAFFS2 + + +1.2 Required disabled config options +------------------------------------ +CONFIG_YAFFS_DISABLE_LAZY_LOAD +DNOTIFY + + +1.3 Recommended enabled config options +------------------------------ +ANDROID_PMEM +PSTORE_CONSOLE +PSTORE_RAM +SCHEDSTATS +DEBUG_PREEMPT +DEBUG_MUTEXES +DEBUG_SPINLOCK_SLEEP +DEBUG_INFO +FRAME_POINTER +CPU_FREQ +CPU_FREQ_TABLE +CPU_FREQ_DEFAULT_GOV_ONDEMAND +CPU_FREQ_GOV_ONDEMAND +CRC_CCITT +EMBEDDED +INPUT_TOUCHSCREEN +I2C +I2C_BOARDINFO +LOG_BUF_SHIFT=17 +SERIAL_CORE +SERIAL_CORE_CONSOLE + + +2. Contact +========== +website: http://android.git.kernel.org + +mailing-lists: android-kernel@googlegroups.com diff --git a/Documentation/block/00-INDEX b/Documentation/block/00-INDEX index e55103ace382a0..a542b9f2a30db6 100644 --- a/Documentation/block/00-INDEX +++ b/Documentation/block/00-INDEX @@ -30,3 +30,9 @@ switching-sched.txt - Switching I/O schedulers at runtime writeback_cache_control.txt - Control of volatile write back caches +mmc-max-speed.txt + - eMMC layer speed simulation, related to /sys/block/mmcblk*/ + attributes: + max_read_speed + max_write_speed + cache_size diff --git a/Documentation/block/mmc-max-speed.txt b/Documentation/block/mmc-max-speed.txt new file mode 100644 index 00000000000000..3f052b9fb99996 --- /dev/null +++ b/Documentation/block/mmc-max-speed.txt @@ -0,0 +1,38 @@ +eMMC Block layer simulation speed controls in /sys/block/mmcblk*/ +=============================================== + +Turned on with CONFIG_MMC_SIMULATE_MAX_SPEED which enables MMC device speed +limiting. Used to test and simulate the behavior of the system when +confronted with a slow MMC. + +Enables max_read_speed, max_write_speed and cache_size attributes and module +default parameters to control the write or read maximum KB/second speed +behaviors. + +NB: There is room for improving the algorithm for aspects tied directly to +eMMC specific behavior. For instance, wear leveling and stalls from an +exhausted erase pool. We would expect that if there was a need to provide +similar speed simulation controls to other types of block devices, aspects of +their behavior are modelled separately (e.g. head seek times, heat assist, +shingling and rotational latency). + +/sys/block/mmcblk0/max_read_speed: + +Number of KB/second reads allowed to the block device. Used to test and +simulate the behavior of the system when confronted with a slow reading MMC. +Set to 0 or "off" to place no speed limit. + +/sys/block/mmcblk0/max_write_speed: + +Number of KB/second writes allowed to the block device. Used to test and +simulate the behavior of the system when confronted with a slow writing MMC. +Set to 0 or "off" to place no speed limit. + +/sys/block/mmcblk0/cache_size: + +Number of MB of high speed memory or high speed SLC cache expected on the +eMMC device being simulated. Used to help simulate the write-back behavior +more accurately. The assumption is the cache has no delay, but draws down +in the background to the MLC/TLC primary store at the max_write_speed rate. +Any write speed delays will show up when the cache is full, or when an I/O +request to flush is issued. diff --git a/Documentation/conf.py b/Documentation/conf.py index bf6f310e517053..d769cd89a9f7d8 100644 --- a/Documentation/conf.py +++ b/Documentation/conf.py @@ -37,7 +37,7 @@ extensions = ['kernel-doc', 'rstFlatTable', 'kernel_include', 'cdomain'] # The name of the math extension changed on Sphinx 1.4 -if minor > 3: +if major == 1 and minor > 3: extensions.append("sphinx.ext.imgmath") else: extensions.append("sphinx.ext.pngmath") @@ -332,6 +332,10 @@ ''' } +# Fix reference escape troubles with Sphinx 1.4.x +if major == 1 and minor > 3: + latex_elements['preamble'] += '\\renewcommand*{\\DUrole}[2]{ #2 }\n' + # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). diff --git a/Documentation/cpu-freq/governors.txt b/Documentation/cpu-freq/governors.txt index c15aa75f52275b..0cf9a6bff6a548 100644 --- a/Documentation/cpu-freq/governors.txt +++ b/Documentation/cpu-freq/governors.txt @@ -28,6 +28,7 @@ Contents: 2.3 Userspace 2.4 Ondemand 2.5 Conservative +2.6 Interactive 3. The Governor Interface in the CPUfreq Core @@ -218,6 +219,91 @@ a decision on when to decrease the frequency while running in any speed. Load for frequency increase is still evaluated every sampling rate. +2.6 Interactive +--------------- + +The CPUfreq governor "interactive" is designed for latency-sensitive, +interactive workloads. This governor sets the CPU speed depending on +usage, similar to "ondemand" and "conservative" governors, but with a +different set of configurable behaviors. + +The tunable values for this governor are: + +above_hispeed_delay: When speed is at or above hispeed_freq, wait for +this long before raising speed in response to continued high load. +The format is a single delay value, optionally followed by pairs of +CPU speeds and the delay to use at or above those speeds. Colons can +be used between the speeds and associated delays for readability. For +example: + + 80000 1300000:200000 1500000:40000 + +uses delay 80000 uS until CPU speed 1.3 GHz, at which speed delay +200000 uS is used until speed 1.5 GHz, at which speed (and above) +delay 40000 uS is used. If speeds are specified these must appear in +ascending order. Default is 20000 uS. + +boost: If non-zero, immediately boost speed of all CPUs to at least +hispeed_freq until zero is written to this attribute. If zero, allow +CPU speeds to drop below hispeed_freq according to load as usual. +Default is zero. + +boostpulse: On each write, immediately boost speed of all CPUs to +hispeed_freq for at least the period of time specified by +boostpulse_duration, after which speeds are allowed to drop below +hispeed_freq according to load as usual. Its a write-only file. + +boostpulse_duration: Length of time to hold CPU speed at hispeed_freq +on a write to boostpulse, before allowing speed to drop according to +load as usual. Default is 80000 uS. + +go_hispeed_load: The CPU load at which to ramp to hispeed_freq. +Default is 99%. + +hispeed_freq: An intermediate "high speed" at which to initially ramp +when CPU load hits the value specified in go_hispeed_load. If load +stays high for the amount of time specified in above_hispeed_delay, +then speed may be bumped higher. Default is the maximum speed allowed +by the policy at governor initialization time. + +io_is_busy: If set, the governor accounts IO time as CPU busy time. + +min_sample_time: The minimum amount of time to spend at the current +frequency before ramping down. Default is 80000 uS. + +target_loads: CPU load values used to adjust speed to influence the +current CPU load toward that value. In general, the lower the target +load, the more often the governor will raise CPU speeds to bring load +below the target. The format is a single target load, optionally +followed by pairs of CPU speeds and CPU loads to target at or above +those speeds. Colons can be used between the speeds and associated +target loads for readability. For example: + + 85 1000000:90 1700000:99 + +targets CPU load 85% below speed 1GHz, 90% at or above 1GHz, until +1.7GHz and above, at which load 99% is targeted. If speeds are +specified these must appear in ascending order. Higher target load +values are typically specified for higher speeds, that is, target load +values also usually appear in an ascending order. The default is +target load 90% for all speeds. + +timer_rate: Sample rate for reevaluating CPU load when the CPU is not +idle. A deferrable timer is used, such that the CPU will not be woken +from idle to service this timer until something else needs to run. +(The maximum time to allow deferring this timer when not running at +minimum speed is configurable via timer_slack.) Default is 20000 uS. + +timer_slack: Maximum additional time to defer handling the governor +sampling timer beyond timer_rate when running at speeds above the +minimum. For platforms that consume additional power at idle when +CPUs are running at speeds greater than minimum, this places an upper +bound on how long the timer will be deferred prior to re-evaluating +load and dropping speed. For example, if timer_rate is 20000uS and +timer_slack is 10000uS then timers will be deferred for up to 30msec +when not at lowest speed. A value of -1 means defer timers +indefinitely at all speeds. Default is 80000 uS. + 3. The Governor Interface in the CPUfreq Core ============================================= diff --git a/Documentation/device-mapper/boot.txt b/Documentation/device-mapper/boot.txt new file mode 100644 index 00000000000000..adcaad5e5e3235 --- /dev/null +++ b/Documentation/device-mapper/boot.txt @@ -0,0 +1,42 @@ +Boot time creation of mapped devices +=================================== + +It is possible to configure a device mapper device to act as the root +device for your system in two ways. + +The first is to build an initial ramdisk which boots to a minimal +userspace which configures the device, then pivot_root(8) in to it. + +For simple device mapper configurations, it is possible to boot directly +using the following kernel command line: + +dm=" ,table line 1,...,table line n" + +name = the name to associate with the device + after boot, udev, if used, will use that name to label + the device node. +uuid = may be 'none' or the UUID desired for the device. +ro = may be "ro" or "rw". If "ro", the device and device table will be + marked read-only. + +Each table line may be as normal when using the dmsetup tool except for +two variations: +1. Any use of commas will be interpreted as a newline +2. Quotation marks cannot be escaped and cannot be used without + terminating the dm= argument. + +Unless renamed by udev, the device node created will be dm-0 as the +first minor number for the device-mapper is used during early creation. + +Example +======= + +- Booting to a linear array made up of user-mode linux block devices: + + dm="lroot none 0, 0 4096 linear 98:16 0, 4096 4096 linear 98:32 0" \ + root=/dev/dm-0 + +Will boot to a rw dm-linear target of 8192 sectors split across two +block devices identified by their major:minor numbers. After boot, udev +will rename this target to /dev/mapper/lroot (depending on the rules). +No uuid was assigned. diff --git a/Documentation/devicetree/bindings/arm/firmware/linaro,optee-tz.txt b/Documentation/devicetree/bindings/arm/firmware/linaro,optee-tz.txt new file mode 100644 index 00000000000000..d38834c67dffe9 --- /dev/null +++ b/Documentation/devicetree/bindings/arm/firmware/linaro,optee-tz.txt @@ -0,0 +1,31 @@ +OP-TEE Device Tree Bindings + +OP-TEE is a piece of software using hardware features to provide a Trusted +Execution Environment. The security can be provided with ARM TrustZone, but +also by virtualization or a separate chip. + +We're using "linaro" as the first part of the compatible property for +the reference implementation maintained by Linaro. + +* OP-TEE based on ARM TrustZone required properties: + +- compatible : should contain "linaro,optee-tz" + +- method : The method of calling the OP-TEE Trusted OS. Permitted + values are: + + "smc" : SMC #0, with the register assignments specified + in drivers/tee/optee/optee_smc.h + + "hvc" : HVC #0, with the register assignments specified + in drivers/tee/optee/optee_smc.h + + + +Example: + firmware { + optee { + compatible = "linaro,optee-tz"; + method = "smc"; + }; + }; diff --git a/Documentation/devicetree/bindings/clock/imx31-clock.txt b/Documentation/devicetree/bindings/clock/imx31-clock.txt index 19df842c694f0b..8163d565f69770 100644 --- a/Documentation/devicetree/bindings/clock/imx31-clock.txt +++ b/Documentation/devicetree/bindings/clock/imx31-clock.txt @@ -77,7 +77,7 @@ Examples: clks: ccm@53f80000{ compatible = "fsl,imx31-ccm"; reg = <0x53f80000 0x4000>; - interrupts = <0 31 0x04 0 53 0x04>; + interrupts = <31>, <53>; #clock-cells = <1>; }; diff --git a/Documentation/devicetree/bindings/mfd/tps65086.txt b/Documentation/devicetree/bindings/mfd/tps65086.txt index d3705612a84655..9cfa886fe99fc1 100644 --- a/Documentation/devicetree/bindings/mfd/tps65086.txt +++ b/Documentation/devicetree/bindings/mfd/tps65086.txt @@ -23,7 +23,7 @@ Required properties: defined below. Optional regulator properties: - - ti,regulator-step-size-25mv : This is applicable for buck[1,2,6], set this + - ti,regulator-step-size-25mv : This is applicable for buck[1-6], set this if the regulator is factory set with a 25mv step voltage mapping. - ti,regulator-decay : This is applicable for buck[1-6], set this if diff --git a/Documentation/devicetree/bindings/misc/memory-state-time.txt b/Documentation/devicetree/bindings/misc/memory-state-time.txt new file mode 100644 index 00000000000000..c99a506c030d9b --- /dev/null +++ b/Documentation/devicetree/bindings/misc/memory-state-time.txt @@ -0,0 +1,8 @@ +Memory bandwidth and frequency state tracking + +Required properties: +- compatible : should be: + "memory-state-time" +- freq-tbl: Should contain entries with each frequency in Hz. +- bw-buckets: Should contain upper-bound limits for each bandwidth bucket in Mbps. + Must match the framework power_profile.xml for the device. diff --git a/Documentation/devicetree/bindings/of/overlay_mgr.txt b/Documentation/devicetree/bindings/of/overlay_mgr.txt new file mode 100644 index 00000000000000..5f3ce4c6d48130 --- /dev/null +++ b/Documentation/devicetree/bindings/of/overlay_mgr.txt @@ -0,0 +1,32 @@ +overlay_mgr + +Required properties: +- compatible: "linux,overlay_manager"; + +Optional properties: +- starts from the word "hardware": hardware_cfg_0 + +These properties can be chosen from kernel command line: +overlay_mgr.overlay_dt_entry=hardware_cfg_0 +DT contains main overlay_mng entry with all possible +HW config setups. And then kernel command line option +will allow to choose between them. + +Example: + overlay_mgr { + compatible = "linux,overlay_manager"; + hardware_cfg_0 { + overlay@0 { + fragment@0 { + __overlay__ { + }; + }; + }; + overlay@1 { + fragment@0 { + __overlay__ { + }; + }; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/sound/hisilicon,hi6210-i2s.txt b/Documentation/devicetree/bindings/sound/hisilicon,hi6210-i2s.txt new file mode 100644 index 00000000000000..4be451cf471642 --- /dev/null +++ b/Documentation/devicetree/bindings/sound/hisilicon,hi6210-i2s.txt @@ -0,0 +1,32 @@ +* Hisilicon 6210 i2s controller + +Required properties: + +- compatible: should be one of the following: + - "hisilicon,hi6210-i2s" +- reg: physical base address of the i2s controller unit and length of + memory mapped region. +- interrupts: should contain the i2s interrupt. +- clocks: a list of phandle + clock-specifier pairs, one for each entry + in clock-names. +- clock-names: should contain following: + - "dacodec" + - "i2s-base" +- dmas: DMA specifiers for tx dma. See the DMA client binding, + Documentation/devicetree/bindings/dma/dma.txt +- dma-names: should be "tx" and "rx" +- hisilicon,sysctrl-syscon: phandle to sysctrl syscon + +Example for the hi6210 i2s controller: + +i2s0: hi6210_i2s { + compatible = "hisilicon,hi6210-i2s"; + reg = <0x0 0xf7118000 0x0 0x8000>; /* i2s unit */ + interrupts = ; /* 155 "DigACodec_intr"-32 */ + clocks = <&sys_ctrl HI6220_DACODEC_PCLK>, + <&sys_ctrl HI6220_BBPPLL0_DIV>; + clock-names = "dacodec", "i2s-base"; + dmas = <&dma0 15 &dma0 14>; + dma-names = "rx", "tx"; + hisilicon,sysctrl-syscon = <&sys_ctrl>; +}; diff --git a/Documentation/devicetree/bindings/usb/dwc2.txt b/Documentation/devicetree/bindings/usb/dwc2.txt index 2c30a5479069b9..dc77e64afda061 100644 --- a/Documentation/devicetree/bindings/usb/dwc2.txt +++ b/Documentation/devicetree/bindings/usb/dwc2.txt @@ -29,6 +29,11 @@ Refer to phy/phy-bindings.txt for generic phy consumer properties - g-rx-fifo-size: size of rx fifo size in gadget mode. - g-np-tx-fifo-size: size of non-periodic tx fifo size in gadget mode. - g-tx-fifo-size: size of periodic tx fifo per endpoint (except ep0) in gadget mode. +- extcon: phandles to external connector devices. First phandle should point to + external connector, which provide "USB" cable events, the second should point + to external connector device, which provide "USB-HOST" cable events. If one + of the external connector devices is not required, empty <0> phandle should + be specified. Example: diff --git a/Documentation/devicetree/bindings/vendor-prefixes.txt b/Documentation/devicetree/bindings/vendor-prefixes.txt index f0a48ea78659c9..6ab5d047c05f5b 100644 --- a/Documentation/devicetree/bindings/vendor-prefixes.txt +++ b/Documentation/devicetree/bindings/vendor-prefixes.txt @@ -155,6 +155,7 @@ lacie LaCie lantiq Lantiq Semiconductor lenovo Lenovo Group Ltd. lg LG Corporation +linaro Linaro Limited linux Linux-specific binding lltc Linear Technology Corporation lsi LSI Corp. (LSI Logic) diff --git a/Documentation/filesystems/proc.txt b/Documentation/filesystems/proc.txt index 74329fd0add223..6e027ae50d7e39 100644 --- a/Documentation/filesystems/proc.txt +++ b/Documentation/filesystems/proc.txt @@ -392,6 +392,8 @@ is not associated with a file: [stack] = the stack of the main process [vdso] = the "virtual dynamic shared object", the kernel system call handler + [anon:] = an anonymous mapping that has been + named by userspace or if empty, the mapping is anonymous. @@ -419,6 +421,7 @@ KernelPageSize: 4 kB MMUPageSize: 4 kB Locked: 0 kB VmFlags: rd ex mr mw me dw +Name: name from userspace the first of these lines shows the same information as is displayed for the mapping in /proc/PID/maps. The remaining lines show the size of the mapping @@ -486,6 +489,9 @@ Note that there is no guarantee that every flag and associated mnemonic will be present in all further kernel releases. Things get changed, the flags may be vanished or the reverse -- new added. +The "Name" field will only be present on a mapping that has been named by +userspace, and will show the name passed in by userspace. + This file is only present if the CONFIG_MMU kernel configuration option is enabled. diff --git a/Documentation/ioctl/ioctl-number.txt b/Documentation/ioctl/ioctl-number.txt index 81c7f2bb7daf09..efb38da700c8d9 100644 --- a/Documentation/ioctl/ioctl-number.txt +++ b/Documentation/ioctl/ioctl-number.txt @@ -308,6 +308,7 @@ Code Seq#(hex) Include File Comments 0xA3 80-8F Port ACL in development: 0xA3 90-9F linux/dtlk.h +0xA4 00-1F uapi/linux/tee.h Generic TEE subsystem 0xAA 00-3F linux/uapi/linux/userfaultfd.h 0xAB 00-1F linux/nbd.h 0xAC 00-1F linux/raw.h diff --git a/Documentation/kernel-parameters.txt b/Documentation/kernel-parameters.txt index 37babf91f2cb6d..4729b59e89e7fb 100644 --- a/Documentation/kernel-parameters.txt +++ b/Documentation/kernel-parameters.txt @@ -87,6 +87,7 @@ parameter is applicable: BLACKFIN Blackfin architecture is enabled. CLK Common clock infrastructure is enabled. CMA Contiguous Memory Area support is enabled. + DM Device mapper support is enabled. DRM Direct Rendering Management support is enabled. DYNAMIC_DEBUG Build in debug messages and enable them at runtime EDD BIOS Enhanced Disk Drive Services (EDD) is enabled @@ -1025,6 +1026,11 @@ bytes respectively. Such letter suffixes can also be entirely omitted. dis_ucode_ldr [X86] Disable the microcode loader. + dm= [DM] Allows early creation of a device-mapper device. + See Documentation/device-mapper/boot.txt. + + dmasound= [HW,OSS] Sound subsystem buff + dma_debug=off If the kernel is compiled with DMA_API_DEBUG support, this option disables the debugging code at boot. @@ -3998,10 +4004,11 @@ bytes respectively. Such letter suffixes can also be entirely omitted. it if 0 is given (See Documentation/cgroup-v1/memory.txt) swiotlb= [ARM,IA-64,PPC,MIPS,X86] - Format: { | force } + Format: { | force | noforce } -- Number of I/O TLB slabs force -- force using of bounce buffers even if they wouldn't be automatically used by the kernel + noforce -- Never use bounce buffers (for debugging) switches= [HW,M68k] diff --git a/Documentation/media/index.rst b/Documentation/media/index.rst index e347a3e7bdef4e..7f8f0af620ceb2 100644 --- a/Documentation/media/index.rst +++ b/Documentation/media/index.rst @@ -1,11 +1,6 @@ Linux Media Subsystem Documentation =================================== -.. Sphinx 1.4.x has a definition for DUrole that doesn't work on alltt blocks -.. raw:: latex - - \renewcommand*{\DUrole}[2]{ #2 } - Contents: .. toctree:: diff --git a/Documentation/networking/ip-sysctl.txt b/Documentation/networking/ip-sysctl.txt index 3db8c67d2c8db9..da258de13bd779 100644 --- a/Documentation/networking/ip-sysctl.txt +++ b/Documentation/networking/ip-sysctl.txt @@ -603,6 +603,16 @@ tcp_fastopen - INTEGER Note that that additional client or server features are only effective if the basic support (0x1 and 0x2) are enabled respectively. +tcp_fwmark_accept - BOOLEAN + If set, incoming connections to listening sockets that do not have a + socket mark will set the mark of the accepting socket to the fwmark of + the incoming SYN packet. This will cause all packets on that connection + (starting from the first SYNACK) to be sent with that fwmark. The + listening socket's mark is unchanged. Listening sockets that already + have a fwmark set via setsockopt(SOL_SOCKET, SO_MARK, ...) are + unaffected. + Default: 0 + tcp_syn_retries - INTEGER Number of times initial SYNs for an active TCP connection attempt will be retransmitted. Should not be higher than 127. Default value diff --git a/Documentation/scheduler/sched-energy.txt b/Documentation/scheduler/sched-energy.txt new file mode 100644 index 00000000000000..dab2f9088b336f --- /dev/null +++ b/Documentation/scheduler/sched-energy.txt @@ -0,0 +1,362 @@ +Energy cost model for energy-aware scheduling (EXPERIMENTAL) + +Introduction +============= + +The basic energy model uses platform energy data stored in sched_group_energy +data structures attached to the sched_groups in the sched_domain hierarchy. The +energy cost model offers two functions that can be used to guide scheduling +decisions: + +1. static unsigned int sched_group_energy(struct energy_env *eenv) +2. static int energy_diff(struct energy_env *eenv) + +sched_group_energy() estimates the energy consumed by all cpus in a specific +sched_group including any shared resources owned exclusively by this group of +cpus. Resources shared with other cpus are excluded (e.g. later level caches). + +energy_diff() estimates the total energy impact of a utilization change. That +is, adding, removing, or migrating utilization (tasks). + +Both functions use a struct energy_env to specify the scenario to be evaluated: + + struct energy_env { + struct sched_group *sg_top; + struct sched_group *sg_cap; + int cap_idx; + int util_delta; + int src_cpu; + int dst_cpu; + int energy; + }; + +sg_top: sched_group to be evaluated. Not used by energy_diff(). + +sg_cap: sched_group covering the cpus in the same frequency domain. Set by +sched_group_energy(). + +cap_idx: Capacity state to be used for energy calculations. Set by +find_new_capacity(). + +util_delta: Amount of utilization to be added, removed, or migrated. + +src_cpu: Source cpu from where 'util_delta' utilization is removed. Should be +-1 if no source (e.g. task wake-up). + +dst_cpu: Destination cpu where 'util_delta' utilization is added. Should be -1 +if utilization is removed (e.g. terminating tasks). + +energy: Result of sched_group_energy(). + +The metric used to represent utilization is the actual per-entity running time +averaged over time using a geometric series. Very similar to the existing +per-entity load-tracking, but _not_ scaled by task priority and capped by the +capacity of the cpu. The latter property does mean that utilization may +underestimate the compute requirements for task on fully/over utilized cpus. +The greatest potential for energy savings without affecting performance too much +is scenarios where the system isn't fully utilized. If the system is deemed +fully utilized load-balancing should be done with task load (includes task +priority) instead in the interest of fairness and performance. + + +Background and Terminology +=========================== + +To make it clear from the start: + +energy = [joule] (resource like a battery on powered devices) +power = energy/time = [joule/second] = [watt] + +The goal of energy-aware scheduling is to minimize energy, while still getting +the job done. That is, we want to maximize: + + performance [inst/s] + -------------------- + power [W] + +which is equivalent to minimizing: + + energy [J] + ----------- + instruction + +while still getting 'good' performance. It is essentially an alternative +optimization objective to the current performance-only objective for the +scheduler. This alternative considers two objectives: energy-efficiency and +performance. Hence, there needs to be a user controllable knob to switch the +objective. Since it is early days, this is currently a sched_feature +(ENERGY_AWARE). + +The idea behind introducing an energy cost model is to allow the scheduler to +evaluate the implications of its decisions rather than applying energy-saving +techniques blindly that may only have positive effects on some platforms. At +the same time, the energy cost model must be as simple as possible to minimize +the scheduler latency impact. + +Platform topology +------------------ + +The system topology (cpus, caches, and NUMA information, not peripherals) is +represented in the scheduler by the sched_domain hierarchy which has +sched_groups attached at each level that covers one or more cpus (see +sched-domains.txt for more details). To add energy awareness to the scheduler +we need to consider power and frequency domains. + +Power domain: + +A power domain is a part of the system that can be powered on/off +independently. Power domains are typically organized in a hierarchy where you +may be able to power down just a cpu or a group of cpus along with any +associated resources (e.g. shared caches). Powering up a cpu means that all +power domains it is a part of in the hierarchy must be powered up. Hence, it is +more expensive to power up the first cpu that belongs to a higher level power +domain than powering up additional cpus in the same high level domain. Two +level power domain hierarchy example: + + Power source + +-------------------------------+----... +per group PD G G + | +----------+ | + +--------+-------| Shared | (other groups) +per-cpu PD G G | resource | + | | +----------+ + +-------+ +-------+ + | CPU 0 | | CPU 1 | + +-------+ +-------+ + +Frequency domain: + +Frequency domains (P-states) typically cover the same group of cpus as one of +the power domain levels. That is, there might be several smaller power domains +sharing the same frequency (P-state) or there might be a power domain spanning +multiple frequency domains. + +From a scheduling point of view there is no need to know the actual frequencies +[Hz]. All the scheduler cares about is the compute capacity available at the +current state (P-state) the cpu is in and any other available states. For that +reason, and to also factor in any cpu micro-architecture differences, compute +capacity scaling states are called 'capacity states' in this document. For SMP +systems this is equivalent to P-states. For mixed micro-architecture systems +(like ARM big.LITTLE) it is P-states scaled according to the micro-architecture +performance relative to the other cpus in the system. + +Energy modelling: +------------------ + +Due to the hierarchical nature of the power domains, the most obvious way to +model energy costs is therefore to associate power and energy costs with +domains (groups of cpus). Energy costs of shared resources are associated with +the group of cpus that share the resources, only the cost of powering the +cpu itself and any private resources (e.g. private L1 caches) is associated +with the per-cpu groups (lowest level). + +For example, for an SMP system with per-cpu power domains and a cluster level +(group of cpus) power domain we get the overall energy costs to be: + + energy = energy_cluster + n * energy_cpu + +where 'n' is the number of cpus powered up and energy_cluster is the cost paid +as soon as any cpu in the cluster is powered up. + +The power and frequency domains can naturally be mapped onto the existing +sched_domain hierarchy and sched_groups by adding the necessary data to the +existing data structures. + +The energy model considers energy consumption from two contributors (shown in +the illustration below): + +1. Busy energy: Energy consumed while a cpu and the higher level groups that it +belongs to are busy running tasks. Busy energy is associated with the state of +the cpu, not an event. The time the cpu spends in this state varies. Thus, the +most obvious platform parameter for this contribution is busy power +(energy/time). + +2. Idle energy: Energy consumed while a cpu and higher level groups that it +belongs to are idle (in a C-state). Like busy energy, idle energy is associated +with the state of the cpu. Thus, the platform parameter for this contribution +is idle power (energy/time). + +Energy consumed during transitions from an idle-state (C-state) to a busy state +(P-state) or going the other way is ignored by the model to simplify the energy +model calculations. + + + Power + ^ + | busy->idle idle->busy + | transition transition + | + | _ __ + | / \ / \__________________ + |______________/ \ / + | \ / + | Busy \ Idle / Busy + | low P-state \____________/ high P-state + | + +------------------------------------------------------------> time + +Busy |--------------| |-----------------| + +Wakeup |------| |------| + +Idle |------------| + + +The basic algorithm +==================== + +The basic idea is to determine the total energy impact when utilization is +added or removed by estimating the impact at each level in the sched_domain +hierarchy starting from the bottom (sched_group contains just a single cpu). +The energy cost comes from busy time (sched_group is awake because one or more +cpus are busy) and idle time (in an idle-state). Energy model numbers account +for energy costs associated with all cpus in the sched_group as a group. + + for_each_domain(cpu, sd) { + sg = sched_group_of(cpu) + energy_before = curr_util(sg) * busy_power(sg) + + (1-curr_util(sg)) * idle_power(sg) + energy_after = new_util(sg) * busy_power(sg) + + (1-new_util(sg)) * idle_power(sg) + energy_diff += energy_before - energy_after + + } + + return energy_diff + +{curr, new}_util: The cpu utilization at the lowest level and the overall +non-idle time for the entire group for higher levels. Utilization is in the +range 0.0 to 1.0 in the pseudo-code. + +busy_power: The power consumption of the sched_group. + +idle_power: The power consumption of the sched_group when idle. + +Note: It is a fundamental assumption that the utilization is (roughly) scale +invariant. Task utilization tracking factors in any frequency scaling and +performance scaling differences due to difference cpu microarchitectures such +that task utilization can be used across the entire system. + + +Platform energy data +===================== + +struct sched_group_energy can be attached to sched_groups in the sched_domain +hierarchy and has the following members: + +cap_states: + List of struct capacity_state representing the supported capacity states + (P-states). struct capacity_state has two members: cap and power, which + represents the compute capacity and the busy_power of the state. The + list must be ordered by capacity low->high. + +nr_cap_states: + Number of capacity states in cap_states list. + +idle_states: + List of struct idle_state containing idle_state power cost for each + idle-state supported by the system orderd by shallowest state first. + All states must be included at all level in the hierarchy, i.e. a + sched_group spanning just a single cpu must also include coupled + idle-states (cluster states). In addition to the cpuidle idle-states, + the list must also contain an entry for the idling using the arch + default idle (arch_idle_cpu()). Despite this state may not be a true + hardware idle-state it is considered the shallowest idle-state in the + energy model and must be the first entry. cpus may enter this state + (possibly 'active idling') if cpuidle decides not enter a cpuidle + idle-state. Default idle may not be used when cpuidle is enabled. + In this case, it should just be a copy of the first cpuidle idle-state. + +nr_idle_states: + Number of idle states in idle_states list. + +There are no unit requirements for the energy cost data. Data can be normalized +with any reference, however, the normalization must be consistent across all +energy cost data. That is, one bogo-joule/watt must be the same quantity for +data, but we don't care what it is. + +A recipe for platform characterization +======================================= + +Obtaining the actual model data for a particular platform requires some way of +measuring power/energy. There isn't a tool to help with this (yet). This +section provides a recipe for use as reference. It covers the steps used to +characterize the ARM TC2 development platform. This sort of measurements is +expected to be done anyway when tuning cpuidle and cpufreq for a given +platform. + +The energy model needs two types of data (struct sched_group_energy holds +these) for each sched_group where energy costs should be taken into account: + +1. Capacity state information + +A list containing the compute capacity and power consumption when fully +utilized attributed to the group as a whole for each available capacity state. +At the lowest level (group contains just a single cpu) this is the power of the +cpu alone without including power consumed by resources shared with other cpus. +It basically needs to fit the basic modelling approach described in "Background +and Terminology" section: + + energy_system = energy_shared + n * energy_cpu + +for a system containing 'n' busy cpus. Only 'energy_cpu' should be included at +the lowest level. 'energy_shared' is included at the next level which +represents the group of cpus among which the resources are shared. + +This model is, of course, a simplification of reality. Thus, power/energy +attributions might not always exactly represent how the hardware is designed. +Also, busy power is likely to depend on the workload. It is therefore +recommended to use a representative mix of workloads when characterizing the +capacity states. + +If the group has no capacity scaling support, the list will contain a single +state where power is the busy power attributed to the group. The capacity +should be set to a default value (1024). + +When frequency domains include multiple power domains, the group representing +the frequency domain and all child groups share capacity states. This must be +indicated by setting the SD_SHARE_CAP_STATES sched_domain flag. All groups at +all levels that share the capacity state must have the list of capacity states +with the power set to the contribution of the individual group. + +2. Idle power information + +Stored in the idle_states list. The power number is the group idle power +consumption in each idle state as well when the group is idle but has not +entered an idle-state ('active idle' as mentioned earlier). Due to the way the +energy model is defined, the idle power of the deepest group idle state can +alternatively be accounted for in the parent group busy power. In that case the +group idle state power values are offset such that the idle power of the +deepest state is zero. It is less intuitive, but it is easier to measure as +idle power consumed by the group and the busy/idle power of the parent group +cannot be distinguished without per group measurement points. + +Measuring capacity states and idle power: + +The capacity states' capacity and power can be estimated by running a benchmark +workload at each available capacity state. By restricting the benchmark to run +on subsets of cpus it is possible to extrapolate the power consumption of +shared resources. + +ARM TC2 has two clusters of two and three cpus respectively. Each cluster has a +shared L2 cache. TC2 has on-chip energy counters per cluster. Running a +benchmark workload on just one cpu in a cluster means that power is consumed in +the cluster (higher level group) and a single cpu (lowest level group). Adding +another benchmark task to another cpu increases the power consumption by the +amount consumed by the additional cpu. Hence, it is possible to extrapolate the +cluster busy power. + +For platforms that don't have energy counters or equivalent instrumentation +built-in, it may be possible to use an external DAQ to acquire similar data. + +If the benchmark includes some performance score (for example sysbench cpu +benchmark), this can be used to record the compute capacity. + +Measuring idle power requires insight into the idle state implementation on the +particular platform. Specifically, if the platform has coupled idle-states (or +package states). To measure non-coupled per-cpu idle-states it is necessary to +keep one cpu busy to keep any shared resources alive to isolate the idle power +of the cpu from idle/busy power of the shared resources. The cpu can be tricked +into different per-cpu idle states by disabling the other states. Based on +various combinations of measurements with specific cpus busy and disabling +idle-states it is possible to extrapolate the idle-state power. diff --git a/Documentation/scheduler/sched-tune.txt b/Documentation/scheduler/sched-tune.txt new file mode 100644 index 00000000000000..9bd2231c01b146 --- /dev/null +++ b/Documentation/scheduler/sched-tune.txt @@ -0,0 +1,366 @@ + Central, scheduler-driven, power-performance control + (EXPERIMENTAL) + +Abstract +======== + +The topic of a single simple power-performance tunable, that is wholly +scheduler centric, and has well defined and predictable properties has come up +on several occasions in the past [1,2]. With techniques such as a scheduler +driven DVFS [3], we now have a good framework for implementing such a tunable. +This document describes the overall ideas behind its design and implementation. + + +Table of Contents +================= + +1. Motivation +2. Introduction +3. Signal Boosting Strategy +4. OPP selection using boosted CPU utilization +5. Per task group boosting +6. Question and Answers + - What about "auto" mode? + - What about boosting on a congested system? + - How CPUs are boosted when we have tasks with multiple boost values? +7. References + + +1. Motivation +============= + +Sched-DVFS [3] is a new event-driven cpufreq governor which allows the +scheduler to select the optimal DVFS operating point (OPP) for running a task +allocated to a CPU. The introduction of sched-DVFS enables running workloads at +the most energy efficient OPPs. + +However, sometimes it may be desired to intentionally boost the performance of +a workload even if that could imply a reasonable increase in energy +consumption. For example, in order to reduce the response time of a task, we +may want to run the task at a higher OPP than the one that is actually required +by it's CPU bandwidth demand. + +This last requirement is especially important if we consider that one of the +main goals of the sched-DVFS component is to replace all currently available +CPUFreq policies. Since sched-DVFS is event based, as opposed to the sampling +driven governors we currently have, it is already more responsive at selecting +the optimal OPP to run tasks allocated to a CPU. However, just tracking the +actual task load demand may not be enough from a performance standpoint. For +example, it is not possible to get behaviors similar to those provided by the +"performance" and "interactive" CPUFreq governors. + +This document describes an implementation of a tunable, stacked on top of the +sched-DVFS which extends its functionality to support task performance +boosting. + +By "performance boosting" we mean the reduction of the time required to +complete a task activation, i.e. the time elapsed from a task wakeup to its +next deactivation (e.g. because it goes back to sleep or it terminates). For +example, if we consider a simple periodic task which executes the same workload +for 5[s] every 20[s] while running at a certain OPP, a boosted execution of +that task must complete each of its activations in less than 5[s]. + +A previous attempt [5] to introduce such a boosting feature has not been +successful mainly because of the complexity of the proposed solution. The +approach described in this document exposes a single simple interface to +user-space. This single tunable knob allows the tuning of system wide +scheduler behaviours ranging from energy efficiency at one end through to +incremental performance boosting at the other end. This first tunable affects +all tasks. However, a more advanced extension of the concept is also provided +which uses CGroups to boost the performance of only selected tasks while using +the energy efficient default for all others. + +The rest of this document introduces in more details the proposed solution +which has been named SchedTune. + + +2. Introduction +=============== + +SchedTune exposes a simple user-space interface with a single power-performance +tunable: + + /proc/sys/kernel/sched_cfs_boost + +This permits expressing a boost value as an integer in the range [0..100]. + +A value of 0 (default) configures the CFS scheduler for maximum energy +efficiency. This means that sched-DVFS runs the tasks at the minimum OPP +required to satisfy their workload demand. +A value of 100 configures scheduler for maximum performance, which translates +to the selection of the maximum OPP on that CPU. + +The range between 0 and 100 can be set to satisfy other scenarios suitably. For +example to satisfy interactive response or depending on other system events +(battery level etc). + +A CGroup based extension is also provided, which permits further user-space +defined task classification to tune the scheduler for different goals depending +on the specific nature of the task, e.g. background vs interactive vs +low-priority. + +The overall design of the SchedTune module is built on top of "Per-Entity Load +Tracking" (PELT) signals and sched-DVFS by introducing a bias on the Operating +Performance Point (OPP) selection. +Each time a task is allocated on a CPU, sched-DVFS has the opportunity to tune +the operating frequency of that CPU to better match the workload demand. The +selection of the actual OPP being activated is influenced by the global boost +value, or the boost value for the task CGroup when in use. + +This simple biasing approach leverages existing frameworks, which means minimal +modifications to the scheduler, and yet it allows to achieve a range of +different behaviours all from a single simple tunable knob. +The only new concept introduced is that of signal boosting. + + +3. Signal Boosting Strategy +=========================== + +The whole PELT machinery works based on the value of a few load tracking signals +which basically track the CPU bandwidth requirements for tasks and the capacity +of CPUs. The basic idea behind the SchedTune knob is to artificially inflate +some of these load tracking signals to make a task or RQ appears more demanding +that it actually is. + +Which signals have to be inflated depends on the specific "consumer". However, +independently from the specific (signal, consumer) pair, it is important to +define a simple and possibly consistent strategy for the concept of boosting a +signal. + +A boosting strategy defines how the "abstract" user-space defined +sched_cfs_boost value is translated into an internal "margin" value to be added +to a signal to get its inflated value: + + margin := boosting_strategy(sched_cfs_boost, signal) + boosted_signal := signal + margin + +Different boosting strategies were identified and analyzed before selecting the +one found to be most effective. + +Signal Proportional Compensation (SPC) +-------------------------------------- + +In this boosting strategy the sched_cfs_boost value is used to compute a +margin which is proportional to the complement of the original signal. +When a signal has a maximum possible value, its complement is defined as +the delta from the actual value and its possible maximum. + +Since the tunable implementation uses signals which have SCHED_LOAD_SCALE as +the maximum possible value, the margin becomes: + + margin := sched_cfs_boost * (SCHED_LOAD_SCALE - signal) + +Using this boosting strategy: +- a 100% sched_cfs_boost means that the signal is scaled to the maximum value +- each value in the range of sched_cfs_boost effectively inflates the signal in + question by a quantity which is proportional to the maximum value. + +For example, by applying the SPC boosting strategy to the selection of the OPP +to run a task it is possible to achieve these behaviors: + +- 0% boosting: run the task at the minimum OPP required by its workload +- 100% boosting: run the task at the maximum OPP available for the CPU +- 50% boosting: run at the half-way OPP between minimum and maximum + +Which means that, at 50% boosting, a task will be scheduled to run at half of +the maximum theoretically achievable performance on the specific target +platform. + +A graphical representation of an SPC boosted signal is represented in the +following figure where: + a) "-" represents the original signal + b) "b" represents a 50% boosted signal + c) "p" represents a 100% boosted signal + + + ^ + | SCHED_LOAD_SCALE + +-----------------------------------------------------------------+ + |pppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppp + | + | boosted_signal + | bbbbbbbbbbbbbbbbbbbbbbbb + | + | original signal + | bbbbbbbbbbbbbbbbbbbbbbbb+----------------------+ + | | + |bbbbbbbbbbbbbbbbbb | + | | + | | + | | + | +-----------------------+ + | | + | | + | | + |------------------+ + | + | + +-----------------------------------------------------------------------> + +The plot above shows a ramped load signal (titled 'original_signal') and it's +boosted equivalent. For each step of the original signal the boosted signal +corresponding to a 50% boost is midway from the original signal and the upper +bound. Boosting by 100% generates a boosted signal which is always saturated to +the upper bound. + + +4. OPP selection using boosted CPU utilization +============================================== + +It is worth calling out that the implementation does not introduce any new load +signals. Instead, it provides an API to tune existing signals. This tuning is +done on demand and only in scheduler code paths where it is sensible to do so. +The new API calls are defined to return either the default signal or a boosted +one, depending on the value of sched_cfs_boost. This is a clean an non invasive +modification of the existing existing code paths. + +The signal representing a CPU's utilization is boosted according to the +previously described SPC boosting strategy. To sched-DVFS, this allows a CPU +(ie CFS run-queue) to appear more used then it actually is. + +Thus, with the sched_cfs_boost enabled we have the following main functions to +get the current utilization of a CPU: + + cpu_util() + boosted_cpu_util() + +The new boosted_cpu_util() is similar to the first but returns a boosted +utilization signal which is a function of the sched_cfs_boost value. + +This function is used in the CFS scheduler code paths where sched-DVFS needs to +decide the OPP to run a CPU at. +For example, this allows selecting the highest OPP for a CPU which has +the boost value set to 100%. + + +5. Per task group boosting +========================== + +The availability of a single knob which is used to boost all tasks in the +system is certainly a simple solution but it quite likely doesn't fit many +utilization scenarios, especially in the mobile device space. + +For example, on battery powered devices there usually are many background +services which are long running and need energy efficient scheduling. On the +other hand, some applications are more performance sensitive and require an +interactive response and/or maximum performance, regardless of the energy cost. +To better service such scenarios, the SchedTune implementation has an extension +that provides a more fine grained boosting interface. + +A new CGroup controller, namely "schedtune", could be enabled which allows to +defined and configure task groups with different boosting values. +Tasks that require special performance can be put into separate CGroups. +The value of the boost associated with the tasks in this group can be specified +using a single knob exposed by the CGroup controller: + + schedtune.boost + +This knob allows the definition of a boost value that is to be used for +SPC boosting of all tasks attached to this group. + +The current schedtune controller implementation is really simple and has these +main characteristics: + + 1) It is only possible to create 1 level depth hierarchies + + The root control groups define the system-wide boost value to be applied + by default to all tasks. Its direct subgroups are named "boost groups" and + they define the boost value for specific set of tasks. + Further nested subgroups are not allowed since they do not have a sensible + meaning from a user-space standpoint. + + 2) It is possible to define only a limited number of "boost groups" + + This number is defined at compile time and by default configured to 16. + This is a design decision motivated by two main reasons: + a) In a real system we do not expect utilization scenarios with more then few + boost groups. For example, a reasonable collection of groups could be + just "background", "interactive" and "performance". + b) It simplifies the implementation considerably, especially for the code + which has to compute the per CPU boosting once there are multiple + RUNNABLE tasks with different boost values. + +Such a simple design should allow servicing the main utilization scenarios identified +so far. It provides a simple interface which can be used to manage the +power-performance of all tasks or only selected tasks. +Moreover, this interface can be easily integrated by user-space run-times (e.g. +Android, ChromeOS) to implement a QoS solution for task boosting based on tasks +classification, which has been a long standing requirement. + +Setup and usage +--------------- + +0. Use a kernel with CGROUP_SCHEDTUNE support enabled + +1. Check that the "schedtune" CGroup controller is available: + + root@linaro-nano:~# cat /proc/cgroups + #subsys_name hierarchy num_cgroups enabled + cpuset 0 1 1 + cpu 0 1 1 + schedtune 0 1 1 + +2. Mount a tmpfs to create the CGroups mount point (Optional) + + root@linaro-nano:~# sudo mount -t tmpfs cgroups /sys/fs/cgroup + +3. Mount the "schedtune" controller + + root@linaro-nano:~# mkdir /sys/fs/cgroup/stune + root@linaro-nano:~# sudo mount -t cgroup -o schedtune stune /sys/fs/cgroup/stune + +4. Setup the system-wide boost value (Optional) + + If not configured the root control group has a 0% boost value, which + basically disables boosting for all tasks in the system thus running in + an energy-efficient mode. + + root@linaro-nano:~# echo $SYSBOOST > /sys/fs/cgroup/stune/schedtune.boost + +5. Create task groups and configure their specific boost value (Optional) + + For example here we create a "performance" boost group configure to boost + all its tasks to 100% + + root@linaro-nano:~# mkdir /sys/fs/cgroup/stune/performance + root@linaro-nano:~# echo 100 > /sys/fs/cgroup/stune/performance/schedtune.boost + +6. Move tasks into the boost group + + For example, the following moves the tasks with PID $TASKPID (and all its + threads) into the "performance" boost group. + + root@linaro-nano:~# echo "TASKPID > /sys/fs/cgroup/stune/performance/cgroup.procs + +This simple configuration allows only the threads of the $TASKPID task to run, +when needed, at the highest OPP in the most capable CPU of the system. + + +6. Question and Answers +======================= + +What about "auto" mode? +----------------------- + +The 'auto' mode as described in [5] can be implemented by interfacing SchedTune +with some suitable user-space element. This element could use the exposed +system-wide or cgroup based interface. + +How are multiple groups of tasks with different boost values managed? +--------------------------------------------------------------------- + +The current SchedTune implementation keeps track of the boosted RUNNABLE tasks +on a CPU. Once sched-DVFS selects the OPP to run a CPU at, the CPU utilization +is boosted with a value which is the maximum of the boost values of the +currently RUNNABLE tasks in its RQ. + +This allows sched-DVFS to boost a CPU only while there are boosted tasks ready +to run and switch back to the energy efficient mode as soon as the last boosted +task is dequeued. + + +7. References +============= +[1] http://lwn.net/Articles/552889 +[2] http://lkml.org/lkml/2012/5/18/91 +[3] http://lkml.org/lkml/2015/6/26/620 diff --git a/Documentation/sphinx/rstFlatTable.py b/Documentation/sphinx/rstFlatTable.py index 55f27579302830..25feb0d35e7abc 100755 --- a/Documentation/sphinx/rstFlatTable.py +++ b/Documentation/sphinx/rstFlatTable.py @@ -157,6 +157,11 @@ def __init__(self, directive): def buildTableNode(self): colwidths = self.directive.get_column_widths(self.max_cols) + if isinstance(colwidths, tuple): + # Since docutils 0.13, get_column_widths returns a (widths, + # colwidths) tuple, where widths is a string (i.e. 'auto'). + # See https://sourceforge.net/p/docutils/patches/120/. + colwidths = colwidths[1] stub_columns = self.directive.options.get('stub-columns', 0) header_rows = self.directive.options.get('header-rows', 0) diff --git a/Documentation/sync.txt b/Documentation/sync.txt new file mode 100644 index 00000000000000..a2d05e7fa1932a --- /dev/null +++ b/Documentation/sync.txt @@ -0,0 +1,75 @@ +Motivation: + +In complicated DMA pipelines such as graphics (multimedia, camera, gpu, display) +a consumer of a buffer needs to know when the producer has finished producing +it. Likewise the producer needs to know when the consumer is finished with the +buffer so it can reuse it. A particular buffer may be consumed by multiple +consumers which will retain the buffer for different amounts of time. In +addition, a consumer may consume multiple buffers atomically. +The sync framework adds an API which allows synchronization between the +producers and consumers in a generic way while also allowing platforms which +have shared hardware synchronization primitives to exploit them. + +Goals: + * provide a generic API for expressing synchronization dependencies + * allow drivers to exploit hardware synchronization between hardware + blocks + * provide a userspace API that allows a compositor to manage + dependencies. + * provide rich telemetry data to allow debugging slowdowns and stalls of + the graphics pipeline. + +Objects: + * sync_timeline + * sync_pt + * sync_fence + +sync_timeline: + +A sync_timeline is an abstract monotonically increasing counter. In general, +each driver/hardware block context will have one of these. They can be backed +by the appropriate hardware or rely on the generic sw_sync implementation. +Timelines are only ever created through their specific implementations +(i.e. sw_sync.) + +sync_pt: + +A sync_pt is an abstract value which marks a point on a sync_timeline. Sync_pts +have a single timeline parent. They have 3 states: active, signaled, and error. +They start in active state and transition, once, to either signaled (when the +timeline counter advances beyond the sync_pt’s value) or error state. + +sync_fence: + +Sync_fences are the primary primitives used by drivers to coordinate +synchronization of their buffers. They are a collection of sync_pts which may +or may not have the same timeline parent. A sync_pt can only exist in one fence +and the fence's list of sync_pts is immutable once created. Fences can be +waited on synchronously or asynchronously. Two fences can also be merged to +create a third fence containing a copy of the two fences’ sync_pts. Fences are +backed by file descriptors to allow userspace to coordinate the display pipeline +dependencies. + +Use: + +A driver implementing sync support should have a work submission function which: + * takes a fence argument specifying when to begin work + * asynchronously queues that work to kick off when the fence is signaled + * returns a fence to indicate when its work will be done. + * signals the returned fence once the work is completed. + +Consider an imaginary display driver that has the following API: +/* + * assumes buf is ready to be displayed. + * blocks until the buffer is on screen. + */ + void display_buffer(struct dma_buf *buf); + +The new API will become: +/* + * will display buf when fence is signaled. + * returns immediately with a fence that will signal when buf + * is no longer displayed. + */ +struct sync_fence* display_buffer(struct dma_buf *buf, + struct sync_fence *fence); diff --git a/Documentation/sysctl/kernel.txt b/Documentation/sysctl/kernel.txt index ffab8b5caa603d..52daff6d09fb71 100644 --- a/Documentation/sysctl/kernel.txt +++ b/Documentation/sysctl/kernel.txt @@ -659,12 +659,14 @@ allowed to execute. perf_event_paranoid: Controls use of the performance events system by unprivileged -users (without CAP_SYS_ADMIN). The default value is 2. +users (without CAP_SYS_ADMIN). The default value is 3 if +CONFIG_SECURITY_PERF_EVENTS_RESTRICT is set, or 2 otherwise. -1: Allow use of (almost) all events by all users >=0: Disallow raw tracepoint access by users without CAP_IOC_LOCK >=1: Disallow CPU event access by users without CAP_SYS_ADMIN >=2: Disallow kernel profiling by users without CAP_SYS_ADMIN +>=3: Disallow all event access by users without CAP_SYS_ADMIN ============================================================== diff --git a/Documentation/tee.txt b/Documentation/tee.txt new file mode 100644 index 00000000000000..7185993575964a --- /dev/null +++ b/Documentation/tee.txt @@ -0,0 +1,118 @@ +TEE subsystem +This document describes the TEE subsystem in Linux. + +A TEE (Trusted Execution Environment) is a trusted OS running in some +secure environment, for example, TrustZone on ARM CPUs, or a separate +secure co-processor etc. A TEE driver handles the details needed to +communicate with the TEE. + +This subsystem deals with: + +- Registration of TEE drivers + +- Managing shared memory between Linux and the TEE + +- Providing a generic API to the TEE + +The TEE interface +================= + +include/uapi/linux/tee.h defines the generic interface to a TEE. + +User space (the client) connects to the driver by opening /dev/tee[0-9]* or +/dev/teepriv[0-9]*. + +- TEE_IOC_SHM_ALLOC allocates shared memory and returns a file descriptor + which user space can mmap. When user space doesn't need the file + descriptor any more, it should be closed. When shared memory isn't needed + any longer it should be unmapped with munmap() to allow the reuse of + memory. + +- TEE_IOC_VERSION lets user space know which TEE this driver handles and + the its capabilities. + +- TEE_IOC_OPEN_SESSION opens a new session to a Trusted Application. + +- TEE_IOC_INVOKE invokes a function in a Trusted Application. + +- TEE_IOC_CANCEL may cancel an ongoing TEE_IOC_OPEN_SESSION or TEE_IOC_INVOKE. + +- TEE_IOC_CLOSE_SESSION closes a session to a Trusted Application. + +There are two classes of clients, normal clients and supplicants. The latter is +a helper process for the TEE to access resources in Linux, for example file +system access. A normal client opens /dev/tee[0-9]* and a supplicant opens +/dev/teepriv[0-9]. + +Much of the communication between clients and the TEE is opaque to the +driver. The main job for the driver is to receive requests from the +clients, forward them to the TEE and send back the results. In the case of +supplicants the communication goes in the other direction, the TEE sends +requests to the supplicant which then sends back the result. + +OP-TEE driver +============= + +The OP-TEE driver handles OP-TEE [1] based TEEs. Currently it is only the ARM +TrustZone based OP-TEE solution that is supported. + +Lowest level of communication with OP-TEE builds on ARM SMC Calling +Convention (SMCCC) [2], which is the foundation for OP-TEE's SMC interface +[3] used internally by the driver. Stacked on top of that is OP-TEE Message +Protocol [4]. + +OP-TEE SMC interface provides the basic functions required by SMCCC and some +additional functions specific for OP-TEE. The most interesting functions are: + +- OPTEE_SMC_FUNCID_CALLS_UID (part of SMCCC) returns the version information + which is then returned by TEE_IOC_VERSION + +- OPTEE_SMC_CALL_GET_OS_UUID returns the particular OP-TEE implementation, used + to tell, for instance, a TrustZone OP-TEE apart from an OP-TEE running on a + separate secure co-processor. + +- OPTEE_SMC_CALL_WITH_ARG drives the OP-TEE message protocol + +- OPTEE_SMC_GET_SHM_CONFIG lets the driver and OP-TEE agree on which memory + range to used for shared memory between Linux and OP-TEE. + +The GlobalPlatform TEE Client API [5] is implemented on top of the generic +TEE API. + +Picture of the relationship between the different components in the +OP-TEE architecture. + + User space Kernel Secure world + ~~~~~~~~~~ ~~~~~~ ~~~~~~~~~~~~ + +--------+ +-------------+ + | Client | | Trusted | + +--------+ | Application | + /\ +-------------+ + || +----------+ /\ + || |tee- | || + || |supplicant| \/ + || +----------+ +-------------+ + \/ /\ | TEE Internal| + +-------+ || | API | + + TEE | || +--------+--------+ +-------------+ + | Client| || | TEE | OP-TEE | | OP-TEE | + | API | \/ | subsys | driver | | Trusted OS | + +-------+----------------+----+-------+----+-----------+-------------+ + | Generic TEE API | | OP-TEE MSG | + | IOCTL (TEE_IOC_*) | | SMCCC (OPTEE_SMC_CALL_*) | + +-----------------------------+ +------------------------------+ + +RPC (Remote Procedure Call) are requests from secure world to kernel driver +or tee-supplicant. An RPC is identified by a special range of SMCCC return +values from OPTEE_SMC_CALL_WITH_ARG. RPC messages which are intended for the +kernel are handled by the kernel driver. Other RPC messages will be forwarded to +tee-supplicant without further involvement of the driver, except switching +shared memory buffer representation. + +References: +[1] https://github.com/OP-TEE/optee_os +[2] http://infocenter.arm.com/help/topic/com.arm.doc.den0028a/index.html +[3] drivers/tee/optee/optee_smc.h +[4] drivers/tee/optee/optee_msg.h +[5] http://www.globalplatform.org/specificationsdevice.asp look for + "TEE Client API Specification v1.0" and click download. diff --git a/Documentation/trace/events-power.txt b/Documentation/trace/events-power.txt index 21d514ced21243..4d817d5acc4090 100644 --- a/Documentation/trace/events-power.txt +++ b/Documentation/trace/events-power.txt @@ -25,6 +25,7 @@ cpufreq. cpu_idle "state=%lu cpu_id=%lu" cpu_frequency "state=%lu cpu_id=%lu" +cpu_frequency_limits "min=%lu max=%lu cpu_id=%lu" A suspend event is used to indicate the system going in and out of the suspend mode: diff --git a/Documentation/trace/ftrace.txt b/Documentation/trace/ftrace.txt index 185c39fea2a022..91723ed5347038 100644 --- a/Documentation/trace/ftrace.txt +++ b/Documentation/trace/ftrace.txt @@ -362,6 +362,26 @@ of ftrace. Here is a list of some of the key files: to correlate events across hypervisor/guest if tb_offset is known. + mono: This uses the fast monotonic clock (CLOCK_MONOTONIC) + which is monotonic and is subject to NTP rate adjustments. + + mono_raw: + This is the raw monotonic clock (CLOCK_MONOTONIC_RAW) + which is montonic but is not subject to any rate adjustments + and ticks at the same rate as the hardware clocksource. + + boot: This is the boot clock (CLOCK_BOOTTIME) and is based on the + fast monotonic clock, but also accounts for time spent in + suspend. Since the clock access is designed for use in + tracing in the suspend path, some side effects are possible + if clock is accessed after the suspend time is accounted before + the fast mono clock is updated. In this case, the clock update + appears to happen slightly sooner than it normally would have. + Also on 32-bit systems, it's possible that the 64-bit boot offset + sees a partial update. These effects are rare and post + processing should be able to handle them. See comments in the + ktime_get_boot_fast_ns() function for more information. + To set a clock, simply echo the clock name into this file. echo global > trace_clock @@ -2102,6 +2122,35 @@ will produce: 1) 1.449 us | } +You can disable the hierarchical function call formatting and instead print a +flat list of function entry and return events. This uses the format described +in the Output Formatting section and respects all the trace options that +control that formatting. Hierarchical formatting is the default. + + hierachical: echo nofuncgraph-flat > trace_options + flat: echo funcgraph-flat > trace_options + + ie: + + # tracer: function_graph + # + # entries-in-buffer/entries-written: 68355/68355 #P:2 + # + # _-----=> irqs-off + # / _----=> need-resched + # | / _---=> hardirq/softirq + # || / _--=> preempt-depth + # ||| / delay + # TASK-PID CPU# |||| TIMESTAMP FUNCTION + # | | | |||| | | + sh-1806 [001] d... 198.843443: graph_ent: func=_raw_spin_lock + sh-1806 [001] d... 198.843445: graph_ent: func=__raw_spin_lock + sh-1806 [001] d..1 198.843447: graph_ret: func=__raw_spin_lock + sh-1806 [001] d..1 198.843449: graph_ret: func=_raw_spin_lock + sh-1806 [001] d..1 198.843451: graph_ent: func=_raw_spin_unlock_irqrestore + sh-1806 [001] d... 198.843453: graph_ret: func=_raw_spin_unlock_irqrestore + + You might find other useful features for this tracer in the following "dynamic ftrace" section such as tracing only specific functions or tasks. diff --git a/Documentation/virtual/kvm/api.txt b/Documentation/virtual/kvm/api.txt index 6bbceb9a3a19d5..1f5eab4ef88fb3 100644 --- a/Documentation/virtual/kvm/api.txt +++ b/Documentation/virtual/kvm/api.txt @@ -2050,6 +2050,7 @@ registers, find a list below: PPC | KVM_REG_PPC_TM_VSCR | 32 PPC | KVM_REG_PPC_TM_DSCR | 64 PPC | KVM_REG_PPC_TM_TAR | 64 + PPC | KVM_REG_PPC_TM_XER | 64 | | MIPS | KVM_REG_MIPS_R0 | 64 ... diff --git a/MAINTAINERS b/MAINTAINERS index 63cefa62324cd1..a419303662bfa6 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -9010,6 +9010,11 @@ F: arch/*/oprofile/ F: drivers/oprofile/ F: include/linux/oprofile.h +OP-TEE DRIVER +M: Jens Wiklander +S: Maintained +F: drivers/tee/optee/ + ORACLE CLUSTER FILESYSTEM 2 (OCFS2) M: Mark Fasheh M: Joel Becker @@ -10655,6 +10660,14 @@ F: drivers/hwtracing/stm/ F: include/linux/stm.h F: include/uapi/linux/stm.h +TEE SUBSYSTEM +M: Jens Wiklander +S: Maintained +F: include/linux/tee_drv.h +F: include/uapi/linux/tee.h +F: drivers/tee/ +F: Documentation/tee.txt + THUNDERBOLT DRIVER M: Andreas Noever S: Maintained diff --git a/Makefile b/Makefile index b1037774e8e83c..d2fe757a979dc7 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ VERSION = 4 PATCHLEVEL = 9 -SUBLEVEL = 0 +SUBLEVEL = 10 EXTRAVERSION = NAME = Roaring Lionus diff --git a/arch/alpha/kernel/ptrace.c b/arch/alpha/kernel/ptrace.c index 940dfb40659108..04abdec7f49605 100644 --- a/arch/alpha/kernel/ptrace.c +++ b/arch/alpha/kernel/ptrace.c @@ -283,7 +283,7 @@ long arch_ptrace(struct task_struct *child, long request, /* When I and D space are separate, these will need to be fixed. */ case PTRACE_PEEKTEXT: /* read word at location addr. */ case PTRACE_PEEKDATA: - copied = access_process_vm(child, addr, &tmp, sizeof(tmp), + copied = ptrace_access_vm(child, addr, &tmp, sizeof(tmp), FOLL_FORCE); ret = -EIO; if (copied != sizeof(tmp)) diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig index bd204bfa29edd5..249e10190d20c5 100644 --- a/arch/arc/Kconfig +++ b/arch/arc/Kconfig @@ -28,7 +28,7 @@ config ARC select HAVE_KPROBES select HAVE_KRETPROBES select HAVE_MEMBLOCK - select HAVE_MOD_ARCH_SPECIFIC if ARC_DW2_UNWIND + select HAVE_MOD_ARCH_SPECIFIC select HAVE_OPROFILE select HAVE_PERF_EVENTS select HANDLE_DOMAIN_IRQ diff --git a/arch/arc/include/asm/cacheflush.h b/arch/arc/include/asm/cacheflush.h index a093adbdb01758..fc662f49c55ac9 100644 --- a/arch/arc/include/asm/cacheflush.h +++ b/arch/arc/include/asm/cacheflush.h @@ -85,6 +85,10 @@ void flush_anon_page(struct vm_area_struct *vma, */ #define PG_dc_clean PG_arch_1 +#define CACHE_COLORS_NUM 4 +#define CACHE_COLORS_MSK (CACHE_COLORS_NUM - 1) +#define CACHE_COLOR(addr) (((unsigned long)(addr) >> (PAGE_SHIFT)) & CACHE_COLORS_MSK) + /* * Simple wrapper over config option * Bootup code ensures that hardware matches kernel configuration @@ -94,8 +98,6 @@ static inline int cache_is_vipt_aliasing(void) return IS_ENABLED(CONFIG_ARC_CACHE_VIPT_ALIASING); } -#define CACHE_COLOR(addr) (((unsigned long)(addr) >> (PAGE_SHIFT)) & 1) - /* * checks if two addresses (after page aligning) index into same cache set */ diff --git a/arch/arc/include/asm/delay.h b/arch/arc/include/asm/delay.h index a36e8601114d2c..d5da2115d78a67 100644 --- a/arch/arc/include/asm/delay.h +++ b/arch/arc/include/asm/delay.h @@ -26,7 +26,9 @@ static inline void __delay(unsigned long loops) " lp 1f \n" " nop \n" "1: \n" - : : "r"(loops)); + : + : "r"(loops) + : "lp_count"); } extern void __bad_udelay(void); diff --git a/arch/arc/include/asm/module.h b/arch/arc/include/asm/module.h index 6e91d8b339c361..567590ea8f6c91 100644 --- a/arch/arc/include/asm/module.h +++ b/arch/arc/include/asm/module.h @@ -14,13 +14,13 @@ #include -#ifdef CONFIG_ARC_DW2_UNWIND struct mod_arch_specific { +#ifdef CONFIG_ARC_DW2_UNWIND void *unw_info; int unw_sec_idx; +#endif const char *secstr; }; -#endif #define MODULE_PROC_FAMILY "ARC700" diff --git a/arch/arc/kernel/module.c b/arch/arc/kernel/module.c index 42e964db296778..3d99a60913325d 100644 --- a/arch/arc/kernel/module.c +++ b/arch/arc/kernel/module.c @@ -32,8 +32,8 @@ int module_frob_arch_sections(Elf_Ehdr *hdr, Elf_Shdr *sechdrs, #ifdef CONFIG_ARC_DW2_UNWIND mod->arch.unw_sec_idx = 0; mod->arch.unw_info = NULL; - mod->arch.secstr = secstr; #endif + mod->arch.secstr = secstr; return 0; } @@ -113,8 +113,10 @@ int apply_relocate_add(Elf32_Shdr *sechdrs, } +#ifdef CONFIG_ARC_DW2_UNWIND if (strcmp(module->arch.secstr+sechdrs[tgtsec].sh_name, ".eh_frame") == 0) module->arch.unw_sec_idx = tgtsec; +#endif return 0; diff --git a/arch/arc/kernel/unaligned.c b/arch/arc/kernel/unaligned.c index abd961f3e7639f..5f69c3bd59bba4 100644 --- a/arch/arc/kernel/unaligned.c +++ b/arch/arc/kernel/unaligned.c @@ -241,8 +241,9 @@ int misaligned_fixup(unsigned long address, struct pt_regs *regs, if (state.fault) goto fault; + /* clear any remanants of delay slot */ if (delay_mode(regs)) { - regs->ret = regs->bta; + regs->ret = regs->bta & ~1U; regs->status32 &= ~STATUS_DE_MASK; } else { regs->ret += state.instr_len; diff --git a/arch/arc/mm/cache.c b/arch/arc/mm/cache.c index 50d71695cd4ecb..8147583c44340f 100644 --- a/arch/arc/mm/cache.c +++ b/arch/arc/mm/cache.c @@ -979,11 +979,16 @@ void arc_cache_init(void) /* check for D-Cache aliasing on ARCompact: ARCv2 has PIPT */ if (is_isa_arcompact()) { int handled = IS_ENABLED(CONFIG_ARC_CACHE_VIPT_ALIASING); - - if (dc->alias && !handled) - panic("Enable CONFIG_ARC_CACHE_VIPT_ALIASING\n"); - else if (!dc->alias && handled) + int num_colors = dc->sz_k/dc->assoc/TO_KB(PAGE_SIZE); + + if (dc->alias) { + if (!handled) + panic("Enable CONFIG_ARC_CACHE_VIPT_ALIASING\n"); + if (CACHE_COLORS_NUM != num_colors) + panic("CACHE_COLORS_NUM not optimized for config\n"); + } else if (!dc->alias && handled) { panic("Disable CONFIG_ARC_CACHE_VIPT_ALIASING\n"); + } } } diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index b5d529fdffab28..00be82f3929f40 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -1836,6 +1836,15 @@ config XEN help Say Y if you want to run Linux in a Virtual Machine on Xen on ARM. +config ARM_FLUSH_CONSOLE_ON_RESTART + bool "Force flush the console on restart" + help + If the console is locked while the system is rebooted, the messages + in the temporary logbuffer would not have propogated to all the + console drivers. This option forces the console lock to be + released if it failed to be acquired, which will cause all the + pending messages to be flushed. + endmenu menu "Boot options" @@ -1864,6 +1873,21 @@ config DEPRECATED_PARAM_STRUCT This was deprecated in 2001 and announced to live on for 5 years. Some old boot loaders still use this way. +config BUILD_ARM_APPENDED_DTB_IMAGE + bool "Build a concatenated zImage/dtb by default" + depends on OF + help + Enabling this option will cause a concatenated zImage and list of + DTBs to be built by default (instead of a standalone zImage.) + The image will built in arch/arm/boot/zImage-dtb + +config BUILD_ARM_APPENDED_DTB_IMAGE_NAMES + string "Default dtb names" + depends on BUILD_ARM_APPENDED_DTB_IMAGE + help + Space separated list of names of dtbs to append when + building a concatenated zImage-dtb. + # Compressed boot loader in ROM. Yes, we really want to ask about # TEXT and BSS so we preserve their values in the config files. config ZBOOT_ROM_TEXT diff --git a/arch/arm/Kconfig.debug b/arch/arm/Kconfig.debug index d83f7c369e514d..17dcd9416db3ac 100644 --- a/arch/arm/Kconfig.debug +++ b/arch/arm/Kconfig.debug @@ -1723,6 +1723,14 @@ config EARLY_PRINTK kernel low-level debugging functions. Add earlyprintk to your kernel parameters to enable this console. +config EARLY_PRINTK_DIRECT + bool "Early printk direct" + depends on DEBUG_LL + help + Say Y here if you want to have an early console using the + kernel low-level debugging functions and EARLY_PRINTK is + not early enough. + config ARM_KPROBES_TEST tristate "Kprobes test module" depends on KPROBES && MODULES diff --git a/arch/arm/Makefile b/arch/arm/Makefile index 6be9ee148b78b9..b53a7b4559d328 100644 --- a/arch/arm/Makefile +++ b/arch/arm/Makefile @@ -298,6 +298,8 @@ libs-y := arch/arm/lib/ $(libs-y) # Default target when executing plain make ifeq ($(CONFIG_XIP_KERNEL),y) KBUILD_IMAGE := xipImage +else ifeq ($(CONFIG_BUILD_ARM_APPENDED_DTB_IMAGE),y) +KBUILD_IMAGE := zImage-dtb else KBUILD_IMAGE := zImage endif @@ -349,6 +351,9 @@ ifeq ($(CONFIG_VDSO),y) $(Q)$(MAKE) $(build)=arch/arm/vdso $@ endif +zImage-dtb: vmlinux scripts dtbs + $(Q)$(MAKE) $(build)=$(boot) MACHINE=$(MACHINE) $(boot)/$@ + # We use MRPROPER_FILES and CLEAN_FILES now archclean: $(Q)$(MAKE) $(clean)=$(boot) diff --git a/arch/arm/boot/.gitignore b/arch/arm/boot/.gitignore index 3c79f85975aaa2..ad7a0253ea961a 100644 --- a/arch/arm/boot/.gitignore +++ b/arch/arm/boot/.gitignore @@ -4,3 +4,4 @@ xipImage bootpImage uImage *.dtb +zImage-dtb \ No newline at end of file diff --git a/arch/arm/boot/Makefile b/arch/arm/boot/Makefile index 50f8d1be7fcbe7..da75630c440da6 100644 --- a/arch/arm/boot/Makefile +++ b/arch/arm/boot/Makefile @@ -16,6 +16,7 @@ OBJCOPYFLAGS :=-O binary -R .comment -S ifneq ($(MACHINE),) include $(MACHINE)/Makefile.boot endif +include $(srctree)/arch/arm/boot/dts/Makefile # Note: the following conditions must always be true: # ZRELADDR == virt_to_phys(PAGE_OFFSET + TEXT_OFFSET) @@ -29,6 +30,14 @@ export ZRELADDR INITRD_PHYS PARAMS_PHYS targets := Image zImage xipImage bootpImage uImage +DTB_NAMES := $(subst $\",,$(CONFIG_BUILD_ARM_APPENDED_DTB_IMAGE_NAMES)) +ifneq ($(DTB_NAMES),) +DTB_LIST := $(addsuffix .dtb,$(DTB_NAMES)) +else +DTB_LIST := $(dtb-y) +endif +DTB_OBJS := $(addprefix $(obj)/dts/,$(DTB_LIST)) + ifeq ($(CONFIG_XIP_KERNEL),y) $(obj)/xipImage: vmlinux FORCE @@ -55,6 +64,10 @@ $(obj)/compressed/vmlinux: $(obj)/Image FORCE $(obj)/zImage: $(obj)/compressed/vmlinux FORCE $(call if_changed,objcopy) +$(obj)/zImage-dtb: $(obj)/zImage $(DTB_OBJS) FORCE + $(call if_changed,cat) + @echo ' Kernel: $@ is ready' + endif ifneq ($(LOADADDR),) diff --git a/arch/arm/boot/compressed/head.S b/arch/arm/boot/compressed/head.S index fc6d541549a2c5..51fc9fb6dc2c64 100644 --- a/arch/arm/boot/compressed/head.S +++ b/arch/arm/boot/compressed/head.S @@ -781,6 +781,8 @@ __armv7_mmu_cache_on: bic r6, r6, #1 << 31 @ 32-bit translation system bic r6, r6, #(7 << 0) | (1 << 4) @ use only ttbr0 mcrne p15, 0, r3, c2, c0, 0 @ load page table pointer + mcrne p15, 0, r0, c8, c7, 0 @ flush I,D TLBs + mcr p15, 0, r0, c7, c5, 4 @ ISB mcrne p15, 0, r1, c3, c0, 0 @ load domain access control mcrne p15, 0, r6, c2, c0, 2 @ load ttb control #endif diff --git a/arch/arm/boot/dts/Makefile b/arch/arm/boot/dts/Makefile index c558ba75cbccf9..54f95d365bf709 100644 --- a/arch/arm/boot/dts/Makefile +++ b/arch/arm/boot/dts/Makefile @@ -485,6 +485,7 @@ dtb-$(CONFIG_ARCH_OMAP3) += \ am3517-evm.dtb \ am3517_mt_ventoux.dtb \ logicpd-torpedo-37xx-devkit.dtb \ + logicpd-som-lv-37xx-devkit.dtb \ omap3430-sdp.dtb \ omap3-beagle.dtb \ omap3-beagle-xm.dtb \ @@ -959,5 +960,15 @@ endif dtstree := $(srctree)/$(src) dtb-$(CONFIG_OF_ALL_DTBS) := $(patsubst $(dtstree)/%.dts,%.dtb, $(wildcard $(dtstree)/*.dts)) -always := $(dtb-y) +DTB_NAMES := $(subst $\",,$(CONFIG_BUILD_ARM_APPENDED_DTB_IMAGE_NAMES)) +ifneq ($(DTB_NAMES),) +DTB_LIST := $(addsuffix .dtb,$(DTB_NAMES)) +else +DTB_LIST := $(dtb-y) +endif + +targets += dtbs dtbs_install +targets += $(DTB_LIST) + +always := $(DTB_LIST) clean-files := *.dtb diff --git a/arch/arm/boot/dts/am33xx.dtsi b/arch/arm/boot/dts/am33xx.dtsi index 194d884c9de137..795c1467fa5031 100644 --- a/arch/arm/boot/dts/am33xx.dtsi +++ b/arch/arm/boot/dts/am33xx.dtsi @@ -16,6 +16,7 @@ interrupt-parent = <&intc>; #address-cells = <1>; #size-cells = <1>; + chosen { }; aliases { i2c0 = &i2c0; diff --git a/arch/arm/boot/dts/am4372.dtsi b/arch/arm/boot/dts/am4372.dtsi index a275fa956813df..a20a71d9d22ef2 100644 --- a/arch/arm/boot/dts/am4372.dtsi +++ b/arch/arm/boot/dts/am4372.dtsi @@ -16,6 +16,7 @@ interrupt-parent = <&wakeupgen>; #address-cells = <1>; #size-cells = <1>; + chosen { }; memory@0 { device_type = "memory"; diff --git a/arch/arm/boot/dts/bcm283x.dtsi b/arch/arm/boot/dts/bcm283x.dtsi index 46d46d894a441d..74dd21b7373c75 100644 --- a/arch/arm/boot/dts/bcm283x.dtsi +++ b/arch/arm/boot/dts/bcm283x.dtsi @@ -104,7 +104,7 @@ reg = <0x7e104000 0x10>; }; - mailbox: mailbox@7e00b800 { + mailbox: mailbox@7e00b880 { compatible = "brcm,bcm2835-mbox"; reg = <0x7e00b880 0x40>; interrupts = <0 1>; diff --git a/arch/arm/boot/dts/da850-evm.dts b/arch/arm/boot/dts/da850-evm.dts index 41de15fe15a2f0..78492a0bbbab94 100644 --- a/arch/arm/boot/dts/da850-evm.dts +++ b/arch/arm/boot/dts/da850-evm.dts @@ -99,6 +99,7 @@ #size-cells = <1>; compatible = "m25p64"; spi-max-frequency = <30000000>; + m25p,fast-read; reg = <0>; partition@0 { label = "U-Boot-SPL"; diff --git a/arch/arm/boot/dts/dm814x.dtsi b/arch/arm/boot/dts/dm814x.dtsi index ff90a6ce6bdc24..d87efab24fa224 100644 --- a/arch/arm/boot/dts/dm814x.dtsi +++ b/arch/arm/boot/dts/dm814x.dtsi @@ -12,6 +12,7 @@ interrupt-parent = <&intc>; #address-cells = <1>; #size-cells = <1>; + chosen { }; aliases { i2c0 = &i2c1; diff --git a/arch/arm/boot/dts/dm816x.dtsi b/arch/arm/boot/dts/dm816x.dtsi index f1e0f771ff297d..cbdfbc4e4a26da 100644 --- a/arch/arm/boot/dts/dm816x.dtsi +++ b/arch/arm/boot/dts/dm816x.dtsi @@ -12,6 +12,7 @@ interrupt-parent = <&intc>; #address-cells = <1>; #size-cells = <1>; + chosen { }; aliases { i2c0 = &i2c1; diff --git a/arch/arm/boot/dts/dra7.dtsi b/arch/arm/boot/dts/dra7.dtsi index d4fcd68f634966..064d84f87e45f7 100644 --- a/arch/arm/boot/dts/dra7.dtsi +++ b/arch/arm/boot/dts/dra7.dtsi @@ -18,6 +18,7 @@ compatible = "ti,dra7xx"; interrupt-parent = <&crossbar_mpu>; + chosen { }; aliases { i2c0 = &i2c1; @@ -1376,6 +1377,7 @@ phy-names = "sata-phy"; clocks = <&sata_ref_clk>; ti,hwmods = "sata"; + ports-implemented = <0x1>; }; rtc: rtc@48838000 { diff --git a/arch/arm/boot/dts/imx31.dtsi b/arch/arm/boot/dts/imx31.dtsi index 1ce7ae94e7ad26..11e9e6bd8abbd1 100644 --- a/arch/arm/boot/dts/imx31.dtsi +++ b/arch/arm/boot/dts/imx31.dtsi @@ -30,11 +30,11 @@ }; }; - avic: avic-interrupt-controller@60000000 { + avic: interrupt-controller@68000000 { compatible = "fsl,imx31-avic", "fsl,avic"; interrupt-controller; #interrupt-cells = <1>; - reg = <0x60000000 0x100000>; + reg = <0x68000000 0x100000>; }; soc { @@ -118,13 +118,6 @@ interrupts = <19>; clocks = <&clks 25>; }; - - clks: ccm@53f80000{ - compatible = "fsl,imx31-ccm"; - reg = <0x53f80000 0x4000>; - interrupts = <0 31 0x04 0 53 0x04>; - #clock-cells = <1>; - }; }; aips@53f00000 { /* AIPS2 */ @@ -134,6 +127,13 @@ reg = <0x53f00000 0x100000>; ranges; + clks: ccm@53f80000{ + compatible = "fsl,imx31-ccm"; + reg = <0x53f80000 0x4000>; + interrupts = <31>, <53>; + #clock-cells = <1>; + }; + gpt: timer@53f90000 { compatible = "fsl,imx31-gpt"; reg = <0x53f90000 0x4000>; diff --git a/arch/arm/boot/dts/imx6dl.dtsi b/arch/arm/boot/dts/imx6dl.dtsi index 1ade1951e620da..7aa120fbdc71ea 100644 --- a/arch/arm/boot/dts/imx6dl.dtsi +++ b/arch/arm/boot/dts/imx6dl.dtsi @@ -137,7 +137,7 @@ &gpio4 { gpio-ranges = <&iomuxc 5 136 1>, <&iomuxc 6 145 1>, <&iomuxc 7 150 1>, <&iomuxc 8 146 1>, <&iomuxc 9 151 1>, <&iomuxc 10 147 1>, - <&iomuxc 11 151 1>, <&iomuxc 12 148 1>, <&iomuxc 13 153 1>, + <&iomuxc 11 152 1>, <&iomuxc 12 148 1>, <&iomuxc 13 153 1>, <&iomuxc 14 149 1>, <&iomuxc 15 154 1>, <&iomuxc 16 39 7>, <&iomuxc 23 56 1>, <&iomuxc 24 61 7>, <&iomuxc 31 46 1>; }; diff --git a/arch/arm/boot/dts/imx6q-cm-fx6.dts b/arch/arm/boot/dts/imx6q-cm-fx6.dts index 59bc5a4dce17b0..a150bca84daa42 100644 --- a/arch/arm/boot/dts/imx6q-cm-fx6.dts +++ b/arch/arm/boot/dts/imx6q-cm-fx6.dts @@ -183,7 +183,6 @@ MX6QDL_PAD_ENET_REF_CLK__ENET_TX_CLK 0x1b0b0 MX6QDL_PAD_ENET_MDIO__ENET_MDIO 0x1b0b0 MX6QDL_PAD_ENET_MDC__ENET_MDC 0x1b0b0 - MX6QDL_PAD_GPIO_16__ENET_REF_CLK 0x4001b0a8 >; }; diff --git a/arch/arm/boot/dts/imx6qdl-nitrogen6_max.dtsi b/arch/arm/boot/dts/imx6qdl-nitrogen6_max.dtsi index b0b3220a1fd905..01166ba36f2753 100644 --- a/arch/arm/boot/dts/imx6qdl-nitrogen6_max.dtsi +++ b/arch/arm/boot/dts/imx6qdl-nitrogen6_max.dtsi @@ -319,8 +319,6 @@ compatible = "fsl,imx6q-nitrogen6_max-sgtl5000", "fsl,imx-audio-sgtl5000"; model = "imx6q-nitrogen6_max-sgtl5000"; - pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_sgtl5000>; ssi-controller = <&ssi1>; audio-codec = <&codec>; audio-routing = @@ -402,6 +400,8 @@ codec: sgtl5000@0a { compatible = "fsl,sgtl5000"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_sgtl5000>; reg = <0x0a>; clocks = <&clks IMX6QDL_CLK_CKO>; VDDA-supply = <®_2p5v>; diff --git a/arch/arm/boot/dts/logicpd-som-lv-37xx-devkit.dts b/arch/arm/boot/dts/logicpd-som-lv-37xx-devkit.dts index da8598402ab8b3..38faa90007d7f0 100644 --- a/arch/arm/boot/dts/logicpd-som-lv-37xx-devkit.dts +++ b/arch/arm/boot/dts/logicpd-som-lv-37xx-devkit.dts @@ -158,7 +158,7 @@ &mmc1 { interrupts-extended = <&intc 83 &omap3_pmx_core 0x11a>; pinctrl-names = "default"; - pinctrl-0 = <&mmc1_pins &mmc1_cd>; + pinctrl-0 = <&mmc1_pins>; wp-gpios = <&gpio4 30 GPIO_ACTIVE_HIGH>; /* gpio_126 */ cd-gpios = <&gpio4 14 IRQ_TYPE_LEVEL_LOW>; /* gpio_110 */ vmmc-supply = <&vmmc1>; @@ -193,7 +193,8 @@ OMAP3_CORE1_IOPAD(0x214a, PIN_INPUT | MUX_MODE0) /* sdmmc1_dat1.sdmmc1_dat1 */ OMAP3_CORE1_IOPAD(0x214c, PIN_INPUT | MUX_MODE0) /* sdmmc1_dat2.sdmmc1_dat2 */ OMAP3_CORE1_IOPAD(0x214e, PIN_INPUT | MUX_MODE0) /* sdmmc1_dat3.sdmmc1_dat3 */ - OMAP3_CORE1_IOPAD(0x2132, PIN_INPUT_PULLUP | MUX_MODE4) /* cam_strobe.gpio_126 sdmmc1_wp*/ + OMAP3_CORE1_IOPAD(0x2132, PIN_INPUT_PULLUP | MUX_MODE4) /* cam_strobe.gpio_126 */ + OMAP3_CORE1_IOPAD(0x212c, PIN_INPUT_PULLUP | MUX_MODE4) /* cam_d11.gpio_110 */ >; }; @@ -242,12 +243,6 @@ OMAP3_WKUP_IOPAD(0x2a16, PIN_OUTPUT | PIN_OFF_OUTPUT_LOW | MUX_MODE4) /* sys_boot6.gpio_8 */ >; }; - - mmc1_cd: pinmux_mmc1_cd { - pinctrl-single,pins = < - OMAP3_WKUP_IOPAD(0x212c, PIN_INPUT_PULLUP | MUX_MODE4) /* cam_d11.gpio_110 */ - >; - }; }; diff --git a/arch/arm/boot/dts/omap2.dtsi b/arch/arm/boot/dts/omap2.dtsi index 4f793a025a721b..f1d6de8b3c193e 100644 --- a/arch/arm/boot/dts/omap2.dtsi +++ b/arch/arm/boot/dts/omap2.dtsi @@ -17,6 +17,7 @@ interrupt-parent = <&intc>; #address-cells = <1>; #size-cells = <1>; + chosen { }; aliases { serial0 = &uart1; diff --git a/arch/arm/boot/dts/omap3.dtsi b/arch/arm/boot/dts/omap3.dtsi index 353d818ce5a6dc..2008648b8c9f90 100644 --- a/arch/arm/boot/dts/omap3.dtsi +++ b/arch/arm/boot/dts/omap3.dtsi @@ -17,6 +17,7 @@ interrupt-parent = <&intc>; #address-cells = <1>; #size-cells = <1>; + chosen { }; aliases { i2c0 = &i2c1; diff --git a/arch/arm/boot/dts/omap4.dtsi b/arch/arm/boot/dts/omap4.dtsi index 0ced079b7ae3d7..9c289ddab3df6e 100644 --- a/arch/arm/boot/dts/omap4.dtsi +++ b/arch/arm/boot/dts/omap4.dtsi @@ -15,6 +15,7 @@ interrupt-parent = <&wakeupgen>; #address-cells = <1>; #size-cells = <1>; + chosen { }; aliases { i2c0 = &i2c1; diff --git a/arch/arm/boot/dts/omap5.dtsi b/arch/arm/boot/dts/omap5.dtsi index 25262118ec3d04..1d1d8e90cd8050 100644 --- a/arch/arm/boot/dts/omap5.dtsi +++ b/arch/arm/boot/dts/omap5.dtsi @@ -17,6 +17,7 @@ compatible = "ti,omap5"; interrupt-parent = <&wakeupgen>; + chosen { }; aliases { i2c0 = &i2c1; @@ -985,6 +986,7 @@ phy-names = "sata-phy"; clocks = <&sata_ref_clk>; ti,hwmods = "sata"; + ports-implemented = <0x1>; }; dss: dss@58000000 { diff --git a/arch/arm/boot/dts/r8a7794.dtsi b/arch/arm/boot/dts/r8a7794.dtsi index 9365580a194fac..7e860d3737ff00 100644 --- a/arch/arm/boot/dts/r8a7794.dtsi +++ b/arch/arm/boot/dts/r8a7794.dtsi @@ -319,7 +319,7 @@ "ch12"; clocks = <&mstp5_clks R8A7794_CLK_AUDIO_DMAC0>; clock-names = "fck"; - power-domains = <&cpg_clocks>; + power-domains = <&sysc R8A7794_PD_ALWAYS_ON>; #dma-cells = <1>; dma-channels = <13>; }; @@ -1025,8 +1025,7 @@ clocks = <&extal_clk &usb_extal_clk>; #clock-cells = <1>; clock-output-names = "main", "pll0", "pll1", "pll3", - "lb", "qspi", "sdh", "sd0", "z", - "rcan"; + "lb", "qspi", "sdh", "sd0", "rcan"; #power-domain-cells = <0>; }; /* Variable factor clocks */ @@ -1260,7 +1259,7 @@ mstp7_clks: mstp7_clks@e615014c { compatible = "renesas,r8a7794-mstp-clocks", "renesas,cpg-mstp-clocks"; reg = <0 0xe615014c 0 4>, <0 0xe61501c4 0 4>; - clocks = <&mp_clk>, <&mp_clk>, + clocks = <&mp_clk>, <&hp_clk>, <&zs_clk>, <&p_clk>, <&p_clk>, <&zs_clk>, <&zs_clk>, <&p_clk>, <&p_clk>, <&p_clk>, <&p_clk>, <&zx_clk>; @@ -1483,7 +1482,7 @@ "mix.0", "mix.1", "dvc.0", "dvc.1", "clk_a", "clk_b", "clk_c", "clk_i"; - power-domains = <&cpg_clocks>; + power-domains = <&sysc R8A7794_PD_ALWAYS_ON>; status = "disabled"; diff --git a/arch/arm/boot/dts/sun7i-a20-bananapi-m1-plus.dts b/arch/arm/boot/dts/sun7i-a20-bananapi-m1-plus.dts index ba5bca0fe997b7..44377a98cc89b2 100644 --- a/arch/arm/boot/dts/sun7i-a20-bananapi-m1-plus.dts +++ b/arch/arm/boot/dts/sun7i-a20-bananapi-m1-plus.dts @@ -227,3 +227,8 @@ pinctrl-0 = <&uart0_pins_a>; status = "okay"; }; + +&usbphy { + /* VBUS on usb host ports are tied to DC5V and therefore always on */ + status = "okay"; +}; diff --git a/arch/arm/common/Kconfig b/arch/arm/common/Kconfig index 9353184d730dfd..ce01364a96e3b3 100644 --- a/arch/arm/common/Kconfig +++ b/arch/arm/common/Kconfig @@ -17,3 +17,7 @@ config SHARP_PARAM config SHARP_SCOOP bool + +config FIQ_GLUE + bool + select FIQ diff --git a/arch/arm/common/Makefile b/arch/arm/common/Makefile index 27f23b15b1ea27..04aca896b3389e 100644 --- a/arch/arm/common/Makefile +++ b/arch/arm/common/Makefile @@ -4,6 +4,7 @@ obj-y += firmware.o +obj-$(CONFIG_FIQ_GLUE) += fiq_glue.o fiq_glue_setup.o obj-$(CONFIG_ICST) += icst.o obj-$(CONFIG_SA1111) += sa1111.o obj-$(CONFIG_DMABOUNCE) += dmabounce.o diff --git a/arch/arm/common/fiq_glue.S b/arch/arm/common/fiq_glue.S new file mode 100644 index 00000000000000..24b42cec4813fa --- /dev/null +++ b/arch/arm/common/fiq_glue.S @@ -0,0 +1,118 @@ +/* + * Copyright (C) 2008 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include + + .text + + .global fiq_glue_end + + /* fiq stack: r0-r15,cpsr,spsr of interrupted mode */ + +ENTRY(fiq_glue) + /* store pc, cpsr from previous mode, reserve space for spsr */ + mrs r12, spsr + sub lr, lr, #4 + subs r10, #1 + bne nested_fiq + + str r12, [sp, #-8]! + str lr, [sp, #-4]! + + /* store r8-r14 from previous mode */ + sub sp, sp, #(7 * 4) + stmia sp, {r8-r14}^ + nop + + /* store r0-r7 from previous mode */ + stmfd sp!, {r0-r7} + + /* setup func(data,regs) arguments */ + mov r0, r9 + mov r1, sp + mov r3, r8 + + mov r7, sp + + /* Get sp and lr from non-user modes */ + and r4, r12, #MODE_MASK + cmp r4, #USR_MODE + beq fiq_from_usr_mode + + mov r7, sp + orr r4, r4, #(PSR_I_BIT | PSR_F_BIT) + msr cpsr_c, r4 + str sp, [r7, #(4 * 13)] + str lr, [r7, #(4 * 14)] + mrs r5, spsr + str r5, [r7, #(4 * 17)] + + cmp r4, #(SVC_MODE | PSR_I_BIT | PSR_F_BIT) + /* use fiq stack if we reenter this mode */ + subne sp, r7, #(4 * 3) + +fiq_from_usr_mode: + msr cpsr_c, #(SVC_MODE | PSR_I_BIT | PSR_F_BIT) + mov r2, sp + sub sp, r7, #12 + stmfd sp!, {r2, ip, lr} + /* call func(data,regs) */ + blx r3 + ldmfd sp, {r2, ip, lr} + mov sp, r2 + + /* restore/discard saved state */ + cmp r4, #USR_MODE + beq fiq_from_usr_mode_exit + + msr cpsr_c, r4 + ldr sp, [r7, #(4 * 13)] + ldr lr, [r7, #(4 * 14)] + msr spsr_cxsf, r5 + +fiq_from_usr_mode_exit: + msr cpsr_c, #(FIQ_MODE | PSR_I_BIT | PSR_F_BIT) + + ldmfd sp!, {r0-r7} + ldr lr, [sp, #(4 * 7)] + ldr r12, [sp, #(4 * 8)] + add sp, sp, #(10 * 4) +exit_fiq: + msr spsr_cxsf, r12 + add r10, #1 + cmp r11, #0 + moveqs pc, lr + bx r11 /* jump to custom fiq return function */ + +nested_fiq: + orr r12, r12, #(PSR_F_BIT) + b exit_fiq + +fiq_glue_end: + +ENTRY(fiq_glue_setup) /* func, data, sp, smc call number */ + stmfd sp!, {r4} + mrs r4, cpsr + msr cpsr_c, #(FIQ_MODE | PSR_I_BIT | PSR_F_BIT) + movs r8, r0 + mov r9, r1 + mov sp, r2 + mov r11, r3 + moveq r10, #0 + movne r10, #1 + msr cpsr_c, r4 + ldmfd sp!, {r4} + bx lr + diff --git a/arch/arm/common/fiq_glue_setup.c b/arch/arm/common/fiq_glue_setup.c new file mode 100644 index 00000000000000..8cb1b611c6d57d --- /dev/null +++ b/arch/arm/common/fiq_glue_setup.c @@ -0,0 +1,147 @@ +/* + * Copyright (C) 2010 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include + +extern unsigned char fiq_glue, fiq_glue_end; +extern void fiq_glue_setup(void *func, void *data, void *sp, + fiq_return_handler_t fiq_return_handler); + +static struct fiq_handler fiq_debbuger_fiq_handler = { + .name = "fiq_glue", +}; +DEFINE_PER_CPU(void *, fiq_stack); +static struct fiq_glue_handler *current_handler; +static fiq_return_handler_t fiq_return_handler; +static DEFINE_MUTEX(fiq_glue_lock); + +static void fiq_glue_setup_helper(void *info) +{ + struct fiq_glue_handler *handler = info; + fiq_glue_setup(handler->fiq, handler, + __get_cpu_var(fiq_stack) + THREAD_START_SP, + fiq_return_handler); +} + +int fiq_glue_register_handler(struct fiq_glue_handler *handler) +{ + int ret; + int cpu; + + if (!handler || !handler->fiq) + return -EINVAL; + + mutex_lock(&fiq_glue_lock); + if (fiq_stack) { + ret = -EBUSY; + goto err_busy; + } + + for_each_possible_cpu(cpu) { + void *stack; + stack = (void *)__get_free_pages(GFP_KERNEL, THREAD_SIZE_ORDER); + if (WARN_ON(!stack)) { + ret = -ENOMEM; + goto err_alloc_fiq_stack; + } + per_cpu(fiq_stack, cpu) = stack; + } + + ret = claim_fiq(&fiq_debbuger_fiq_handler); + if (WARN_ON(ret)) + goto err_claim_fiq; + + current_handler = handler; + on_each_cpu(fiq_glue_setup_helper, handler, true); + set_fiq_handler(&fiq_glue, &fiq_glue_end - &fiq_glue); + + mutex_unlock(&fiq_glue_lock); + return 0; + +err_claim_fiq: +err_alloc_fiq_stack: + for_each_possible_cpu(cpu) { + __free_pages(per_cpu(fiq_stack, cpu), THREAD_SIZE_ORDER); + per_cpu(fiq_stack, cpu) = NULL; + } +err_busy: + mutex_unlock(&fiq_glue_lock); + return ret; +} + +static void fiq_glue_update_return_handler(void (*fiq_return)(void)) +{ + fiq_return_handler = fiq_return; + if (current_handler) + on_each_cpu(fiq_glue_setup_helper, current_handler, true); +} + +int fiq_glue_set_return_handler(void (*fiq_return)(void)) +{ + int ret; + + mutex_lock(&fiq_glue_lock); + if (fiq_return_handler) { + ret = -EBUSY; + goto err_busy; + } + fiq_glue_update_return_handler(fiq_return); + ret = 0; +err_busy: + mutex_unlock(&fiq_glue_lock); + + return ret; +} +EXPORT_SYMBOL(fiq_glue_set_return_handler); + +int fiq_glue_clear_return_handler(void (*fiq_return)(void)) +{ + int ret; + + mutex_lock(&fiq_glue_lock); + if (WARN_ON(fiq_return_handler != fiq_return)) { + ret = -EINVAL; + goto err_inval; + } + fiq_glue_update_return_handler(NULL); + ret = 0; +err_inval: + mutex_unlock(&fiq_glue_lock); + + return ret; +} +EXPORT_SYMBOL(fiq_glue_clear_return_handler); + +/** + * fiq_glue_resume - Restore fiqs after suspend or low power idle states + * + * This must be called before calling local_fiq_enable after returning from a + * power state where the fiq mode registers were lost. If a driver provided + * a resume hook when it registered the handler it will be called. + */ + +void fiq_glue_resume(void) +{ + if (!current_handler) + return; + fiq_glue_setup(current_handler->fiq, current_handler, + __get_cpu_var(fiq_stack) + THREAD_START_SP, + fiq_return_handler); + if (current_handler->resume) + current_handler->resume(current_handler); +} + diff --git a/arch/arm/configs/qcom_defconfig b/arch/arm/configs/qcom_defconfig index c2dff4fd5fc4d7..9f6d2a69a6f7a5 100644 --- a/arch/arm/configs/qcom_defconfig +++ b/arch/arm/configs/qcom_defconfig @@ -162,8 +162,8 @@ CONFIG_APQ_MMCC_8084=y CONFIG_IPQ_LCC_806X=y CONFIG_MSM_GCC_8660=y CONFIG_MSM_LCC_8960=y -CONFIG_MSM_GCC_9615=y -CONFIG_MSM_LCC_9615=y +CONFIG_MDM_GCC_9615=y +CONFIG_MDM_LCC_9615=y CONFIG_MSM_MMCC_8960=y CONFIG_MSM_MMCC_8974=y CONFIG_HWSPINLOCK_QCOM=y diff --git a/arch/arm/configs/ranchu_defconfig b/arch/arm/configs/ranchu_defconfig new file mode 100644 index 00000000000000..49e7bbd5825a71 --- /dev/null +++ b/arch/arm/configs/ranchu_defconfig @@ -0,0 +1,316 @@ +# CONFIG_LOCALVERSION_AUTO is not set +CONFIG_AUDIT=y +CONFIG_NO_HZ=y +CONFIG_HIGH_RES_TIMERS=y +CONFIG_TASKSTATS=y +CONFIG_TASK_DELAY_ACCT=y +CONFIG_TASK_XACCT=y +CONFIG_TASK_IO_ACCOUNTING=y +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y +CONFIG_LOG_BUF_SHIFT=14 +CONFIG_CGROUPS=y +CONFIG_CGROUP_DEBUG=y +CONFIG_CGROUP_FREEZER=y +CONFIG_CPUSETS=y +CONFIG_CGROUP_CPUACCT=y +CONFIG_CGROUP_SCHED=y +CONFIG_RT_GROUP_SCHED=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_KALLSYMS_ALL=y +CONFIG_EMBEDDED=y +CONFIG_PROFILING=y +CONFIG_OPROFILE=y +CONFIG_ARCH_MMAP_RND_BITS=16 +# CONFIG_BLK_DEV_BSG is not set +# CONFIG_IOSCHED_DEADLINE is not set +# CONFIG_IOSCHED_CFQ is not set +CONFIG_ARCH_VIRT=y +CONFIG_ARM_KERNMEM_PERMS=y +CONFIG_SMP=y +CONFIG_PREEMPT=y +CONFIG_AEABI=y +CONFIG_HIGHMEM=y +CONFIG_KSM=y +CONFIG_SECCOMP=y +CONFIG_CMDLINE="console=ttyAMA0" +CONFIG_VFP=y +CONFIG_NEON=y +# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set +CONFIG_PM_AUTOSLEEP=y +CONFIG_PM_WAKELOCKS=y +CONFIG_PM_WAKELOCKS_LIMIT=0 +# CONFIG_PM_WAKELOCKS_GC is not set +CONFIG_PM_DEBUG=y +CONFIG_NET=y +CONFIG_PACKET=y +CONFIG_UNIX=y +CONFIG_XFRM_USER=y +CONFIG_NET_KEY=y +CONFIG_INET=y +CONFIG_INET_DIAG_DESTROY=y +CONFIG_IP_MULTICAST=y +CONFIG_IP_ADVANCED_ROUTER=y +CONFIG_IP_MULTIPLE_TABLES=y +CONFIG_IP_PNP=y +CONFIG_IP_PNP_DHCP=y +CONFIG_IP_PNP_BOOTP=y +CONFIG_INET_ESP=y +# CONFIG_INET_LRO is not set +CONFIG_IPV6_ROUTER_PREF=y +CONFIG_IPV6_ROUTE_INFO=y +CONFIG_IPV6_OPTIMISTIC_DAD=y +CONFIG_INET6_AH=y +CONFIG_INET6_ESP=y +CONFIG_INET6_IPCOMP=y +CONFIG_IPV6_MIP6=y +CONFIG_IPV6_MULTIPLE_TABLES=y +CONFIG_NETFILTER=y +CONFIG_NF_CONNTRACK=y +CONFIG_NF_CONNTRACK_SECMARK=y +CONFIG_NF_CONNTRACK_EVENTS=y +CONFIG_NF_CT_PROTO_DCCP=y +CONFIG_NF_CT_PROTO_SCTP=y +CONFIG_NF_CT_PROTO_UDPLITE=y +CONFIG_NF_CONNTRACK_AMANDA=y +CONFIG_NF_CONNTRACK_FTP=y +CONFIG_NF_CONNTRACK_H323=y +CONFIG_NF_CONNTRACK_IRC=y +CONFIG_NF_CONNTRACK_NETBIOS_NS=y +CONFIG_NF_CONNTRACK_PPTP=y +CONFIG_NF_CONNTRACK_SANE=y +CONFIG_NF_CONNTRACK_TFTP=y +CONFIG_NF_CT_NETLINK=y +CONFIG_NETFILTER_XT_TARGET_CLASSIFY=y +CONFIG_NETFILTER_XT_TARGET_CONNMARK=y +CONFIG_NETFILTER_XT_TARGET_CONNSECMARK=y +CONFIG_NETFILTER_XT_TARGET_IDLETIMER=y +CONFIG_NETFILTER_XT_TARGET_MARK=y +CONFIG_NETFILTER_XT_TARGET_NFLOG=y +CONFIG_NETFILTER_XT_TARGET_NFQUEUE=y +CONFIG_NETFILTER_XT_TARGET_TPROXY=y +CONFIG_NETFILTER_XT_TARGET_TRACE=y +CONFIG_NETFILTER_XT_TARGET_SECMARK=y +CONFIG_NETFILTER_XT_TARGET_TCPMSS=y +CONFIG_NETFILTER_XT_MATCH_COMMENT=y +CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=y +CONFIG_NETFILTER_XT_MATCH_CONNMARK=y +CONFIG_NETFILTER_XT_MATCH_CONNTRACK=y +CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=y +CONFIG_NETFILTER_XT_MATCH_HELPER=y +CONFIG_NETFILTER_XT_MATCH_IPRANGE=y +CONFIG_NETFILTER_XT_MATCH_LENGTH=y +CONFIG_NETFILTER_XT_MATCH_LIMIT=y +CONFIG_NETFILTER_XT_MATCH_MAC=y +CONFIG_NETFILTER_XT_MATCH_MARK=y +CONFIG_NETFILTER_XT_MATCH_POLICY=y +CONFIG_NETFILTER_XT_MATCH_PKTTYPE=y +CONFIG_NETFILTER_XT_MATCH_QTAGUID=y +CONFIG_NETFILTER_XT_MATCH_QUOTA=y +CONFIG_NETFILTER_XT_MATCH_QUOTA2=y +CONFIG_NETFILTER_XT_MATCH_SOCKET=y +CONFIG_NETFILTER_XT_MATCH_STATE=y +CONFIG_NETFILTER_XT_MATCH_STATISTIC=y +CONFIG_NETFILTER_XT_MATCH_STRING=y +CONFIG_NETFILTER_XT_MATCH_TIME=y +CONFIG_NETFILTER_XT_MATCH_U32=y +CONFIG_NF_CONNTRACK_IPV4=y +CONFIG_IP_NF_IPTABLES=y +CONFIG_IP_NF_MATCH_AH=y +CONFIG_IP_NF_MATCH_ECN=y +CONFIG_IP_NF_MATCH_TTL=y +CONFIG_IP_NF_FILTER=y +CONFIG_IP_NF_TARGET_REJECT=y +CONFIG_IP_NF_MANGLE=y +CONFIG_IP_NF_RAW=y +CONFIG_IP_NF_SECURITY=y +CONFIG_IP_NF_ARPTABLES=y +CONFIG_IP_NF_ARPFILTER=y +CONFIG_IP_NF_ARP_MANGLE=y +CONFIG_NF_CONNTRACK_IPV6=y +CONFIG_IP6_NF_IPTABLES=y +CONFIG_IP6_NF_FILTER=y +CONFIG_IP6_NF_TARGET_REJECT=y +CONFIG_IP6_NF_MANGLE=y +CONFIG_IP6_NF_RAW=y +CONFIG_BRIDGE=y +CONFIG_NET_SCHED=y +CONFIG_NET_SCH_HTB=y +CONFIG_NET_CLS_U32=y +CONFIG_NET_EMATCH=y +CONFIG_NET_EMATCH_U32=y +CONFIG_NET_CLS_ACT=y +# CONFIG_WIRELESS is not set +CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" +CONFIG_MTD=y +CONFIG_MTD_CMDLINE_PARTS=y +CONFIG_MTD_BLOCK=y +CONFIG_MTD_CFI=y +CONFIG_MTD_CFI_INTELEXT=y +CONFIG_MTD_CFI_AMDSTD=y +CONFIG_BLK_DEV_LOOP=y +CONFIG_BLK_DEV_RAM=y +CONFIG_BLK_DEV_RAM_SIZE=8192 +CONFIG_VIRTIO_BLK=y +CONFIG_MD=y +CONFIG_BLK_DEV_DM=y +CONFIG_DM_CRYPT=y +CONFIG_DM_UEVENT=y +CONFIG_DM_VERITY=y +CONFIG_DM_VERITY_FEC=y +CONFIG_NETDEVICES=y +CONFIG_TUN=y +CONFIG_VIRTIO_NET=y +CONFIG_SMSC911X=y +CONFIG_PPP=y +CONFIG_PPP_BSDCOMP=y +CONFIG_PPP_DEFLATE=y +CONFIG_PPP_MPPE=y +CONFIG_PPPOLAC=y +CONFIG_PPPOPNS=y +CONFIG_USB_USBNET=y +# CONFIG_WLAN is not set +CONFIG_INPUT_EVDEV=y +CONFIG_INPUT_KEYRESET=y +CONFIG_KEYBOARD_GOLDFISH_EVENTS=y +# CONFIG_INPUT_MOUSE is not set +CONFIG_INPUT_JOYSTICK=y +CONFIG_JOYSTICK_XPAD=y +CONFIG_JOYSTICK_XPAD_FF=y +CONFIG_JOYSTICK_XPAD_LEDS=y +CONFIG_INPUT_TABLET=y +CONFIG_TABLET_USB_ACECAD=y +CONFIG_TABLET_USB_AIPTEK=y +CONFIG_TABLET_USB_GTCO=y +CONFIG_TABLET_USB_HANWANG=y +CONFIG_TABLET_USB_KBTAB=y +CONFIG_INPUT_MISC=y +CONFIG_INPUT_KEYCHORD=y +CONFIG_INPUT_UINPUT=y +CONFIG_INPUT_GPIO=y +# CONFIG_SERIO_SERPORT is not set +CONFIG_SERIO_AMBAKMI=y +# CONFIG_VT is not set +# CONFIG_LEGACY_PTYS is not set +# CONFIG_DEVMEM is not set +# CONFIG_DEVKMEM is not set +CONFIG_SERIAL_AMBA_PL011=y +CONFIG_SERIAL_AMBA_PL011_CONSOLE=y +CONFIG_VIRTIO_CONSOLE=y +# CONFIG_HW_RANDOM is not set +# CONFIG_HWMON is not set +CONFIG_MEDIA_SUPPORT=y +CONFIG_FB=y +CONFIG_FB_GOLDFISH=y +CONFIG_FB_SIMPLE=y +CONFIG_BACKLIGHT_LCD_SUPPORT=y +CONFIG_LOGO=y +# CONFIG_LOGO_LINUX_MONO is not set +# CONFIG_LOGO_LINUX_VGA16 is not set +CONFIG_SOUND=y +CONFIG_SND=y +CONFIG_HIDRAW=y +CONFIG_UHID=y +CONFIG_HID_A4TECH=y +CONFIG_HID_ACRUX=y +CONFIG_HID_ACRUX_FF=y +CONFIG_HID_APPLE=y +CONFIG_HID_BELKIN=y +CONFIG_HID_CHERRY=y +CONFIG_HID_CHICONY=y +CONFIG_HID_PRODIKEYS=y +CONFIG_HID_CYPRESS=y +CONFIG_HID_DRAGONRISE=y +CONFIG_DRAGONRISE_FF=y +CONFIG_HID_EMS_FF=y +CONFIG_HID_ELECOM=y +CONFIG_HID_EZKEY=y +CONFIG_HID_HOLTEK=y +CONFIG_HID_KEYTOUCH=y +CONFIG_HID_KYE=y +CONFIG_HID_UCLOGIC=y +CONFIG_HID_WALTOP=y +CONFIG_HID_GYRATION=y +CONFIG_HID_TWINHAN=y +CONFIG_HID_KENSINGTON=y +CONFIG_HID_LCPOWER=y +CONFIG_HID_LOGITECH=y +CONFIG_HID_LOGITECH_DJ=y +CONFIG_LOGITECH_FF=y +CONFIG_LOGIRUMBLEPAD2_FF=y +CONFIG_LOGIG940_FF=y +CONFIG_HID_MAGICMOUSE=y +CONFIG_HID_MICROSOFT=y +CONFIG_HID_MONTEREY=y +CONFIG_HID_MULTITOUCH=y +CONFIG_HID_NTRIG=y +CONFIG_HID_ORTEK=y +CONFIG_HID_PANTHERLORD=y +CONFIG_PANTHERLORD_FF=y +CONFIG_HID_PETALYNX=y +CONFIG_HID_PICOLCD=y +CONFIG_HID_PRIMAX=y +CONFIG_HID_ROCCAT=y +CONFIG_HID_SAITEK=y +CONFIG_HID_SAMSUNG=y +CONFIG_HID_SONY=y +CONFIG_HID_SPEEDLINK=y +CONFIG_HID_SUNPLUS=y +CONFIG_HID_GREENASIA=y +CONFIG_GREENASIA_FF=y +CONFIG_HID_SMARTJOYPLUS=y +CONFIG_SMARTJOYPLUS_FF=y +CONFIG_HID_TIVO=y +CONFIG_HID_TOPSEED=y +CONFIG_HID_THRUSTMASTER=y +CONFIG_HID_WACOM=y +CONFIG_HID_WIIMOTE=y +CONFIG_HID_ZEROPLUS=y +CONFIG_HID_ZYDACRON=y +CONFIG_USB_HIDDEV=y +CONFIG_USB_ANNOUNCE_NEW_DEVICES=y +CONFIG_USB_EHCI_HCD=y +CONFIG_USB_OTG_WAKELOCK=y +CONFIG_RTC_CLASS=y +CONFIG_RTC_DRV_PL031=y +CONFIG_VIRTIO_MMIO=y +CONFIG_STAGING=y +CONFIG_ASHMEM=y +CONFIG_ANDROID_LOW_MEMORY_KILLER=y +CONFIG_SYNC=y +CONFIG_SW_SYNC=y +CONFIG_SW_SYNC_USER=y +CONFIG_ION=y +CONFIG_GOLDFISH_AUDIO=y +CONFIG_GOLDFISH=y +CONFIG_GOLDFISH_PIPE=y +CONFIG_ANDROID=y +CONFIG_ANDROID_BINDER_IPC=y +CONFIG_EXT4_FS=y +CONFIG_EXT4_FS_SECURITY=y +CONFIG_QUOTA=y +CONFIG_FUSE_FS=y +CONFIG_CUSE=y +CONFIG_MSDOS_FS=y +CONFIG_VFAT_FS=y +CONFIG_TMPFS=y +CONFIG_TMPFS_POSIX_ACL=y +CONFIG_PSTORE=y +CONFIG_PSTORE_CONSOLE=y +CONFIG_PSTORE_RAM=y +CONFIG_NFS_FS=y +CONFIG_ROOT_NFS=y +CONFIG_NLS_CODEPAGE_437=y +CONFIG_NLS_ISO8859_1=y +CONFIG_DEBUG_INFO=y +CONFIG_MAGIC_SYSRQ=y +CONFIG_DETECT_HUNG_TASK=y +CONFIG_PANIC_TIMEOUT=5 +# CONFIG_SCHED_DEBUG is not set +CONFIG_SCHEDSTATS=y +CONFIG_TIMER_STATS=y +CONFIG_ENABLE_DEFAULT_TRACERS=y +CONFIG_SECURITY=y +CONFIG_SECURITY_NETWORK=y +CONFIG_SECURITY_SELINUX=y +CONFIG_VIRTUALIZATION=y diff --git a/arch/arm/crypto/aes-ce-glue.c b/arch/arm/crypto/aes-ce-glue.c index aef022a87c5379..04410d9f5e72e0 100644 --- a/arch/arm/crypto/aes-ce-glue.c +++ b/arch/arm/crypto/aes-ce-glue.c @@ -88,8 +88,13 @@ static int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, u32 *rki = ctx->key_enc + (i * kwords); u32 *rko = rki + kwords; +#ifndef CONFIG_CPU_BIG_ENDIAN rko[0] = ror32(ce_aes_sub(rki[kwords - 1]), 8); rko[0] = rko[0] ^ rki[0] ^ rcon[i]; +#else + rko[0] = rol32(ce_aes_sub(rki[kwords - 1]), 8); + rko[0] = rko[0] ^ rki[0] ^ (rcon[i] << 24); +#endif rko[1] = rko[0] ^ rki[1]; rko[2] = rko[1] ^ rki[2]; rko[3] = rko[2] ^ rki[3]; diff --git a/arch/arm/include/asm/cputype.h b/arch/arm/include/asm/cputype.h index 522b5feb4eaa34..b62eaeb147aa9a 100644 --- a/arch/arm/include/asm/cputype.h +++ b/arch/arm/include/asm/cputype.h @@ -94,6 +94,9 @@ #define ARM_CPU_XSCALE_ARCH_V2 0x4000 #define ARM_CPU_XSCALE_ARCH_V3 0x6000 +/* Qualcomm implemented cores */ +#define ARM_CPU_PART_SCORPION 0x510002d0 + extern unsigned int processor_id; #ifdef CONFIG_CPU_CP15 diff --git a/arch/arm/include/asm/fiq_glue.h b/arch/arm/include/asm/fiq_glue.h new file mode 100644 index 00000000000000..a9e244f9f197c8 --- /dev/null +++ b/arch/arm/include/asm/fiq_glue.h @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2010 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef __ASM_FIQ_GLUE_H +#define __ASM_FIQ_GLUE_H + +struct fiq_glue_handler { + void (*fiq)(struct fiq_glue_handler *h, void *regs, void *svc_sp); + void (*resume)(struct fiq_glue_handler *h); +}; +typedef void (*fiq_return_handler_t)(void); + +int fiq_glue_register_handler(struct fiq_glue_handler *handler); +int fiq_glue_set_return_handler(fiq_return_handler_t fiq_return); +int fiq_glue_clear_return_handler(fiq_return_handler_t fiq_return); + +#ifdef CONFIG_FIQ_GLUE +void fiq_glue_resume(void); +#else +static inline void fiq_glue_resume(void) {} +#endif + +#endif diff --git a/arch/arm/include/asm/topology.h b/arch/arm/include/asm/topology.h index 370f7a732900ae..d0606412069492 100644 --- a/arch/arm/include/asm/topology.h +++ b/arch/arm/include/asm/topology.h @@ -3,6 +3,7 @@ #ifdef CONFIG_ARM_CPU_TOPOLOGY +#include #include struct cputopo_arm { @@ -24,6 +25,12 @@ void init_cpu_topology(void); void store_cpu_topology(unsigned int cpuid); const struct cpumask *cpu_coregroup_mask(int cpu); +#ifdef CONFIG_CPU_FREQ +#define arch_scale_freq_capacity cpufreq_scale_freq_capacity +#endif +#define arch_scale_cpu_capacity scale_cpu_capacity +extern unsigned long scale_cpu_capacity(struct sched_domain *sd, int cpu); + #else static inline void init_cpu_topology(void) { } diff --git a/arch/arm/kernel/hw_breakpoint.c b/arch/arm/kernel/hw_breakpoint.c index b8df45883cf78e..25538a9358741a 100644 --- a/arch/arm/kernel/hw_breakpoint.c +++ b/arch/arm/kernel/hw_breakpoint.c @@ -1066,6 +1066,22 @@ static int __init arch_hw_breakpoint_init(void) return 0; } + /* + * Scorpion CPUs (at least those in APQ8060) seem to set DBGPRSR.SPD + * whenever a WFI is issued, even if the core is not powered down, in + * violation of the architecture. When DBGPRSR.SPD is set, accesses to + * breakpoint and watchpoint registers are treated as undefined, so + * this results in boot time and runtime failures when these are + * accessed and we unexpectedly take a trap. + * + * It's not clear if/how this can be worked around, so we blacklist + * Scorpion CPUs to avoid these issues. + */ + if (read_cpuid_part() == ARM_CPU_PART_SCORPION) { + pr_info("Scorpion CPU detected. Hardware breakpoints and watchpoints disabled\n"); + return 0; + } + has_ossr = core_has_os_save_restore(); /* Determine how many BRPs/WRPs are available. */ diff --git a/arch/arm/kernel/kgdb.c b/arch/arm/kernel/kgdb.c index 9232caee70604c..f3c66229953135 100644 --- a/arch/arm/kernel/kgdb.c +++ b/arch/arm/kernel/kgdb.c @@ -140,6 +140,8 @@ int kgdb_arch_handle_exception(int exception_vector, int signo, static int kgdb_brk_fn(struct pt_regs *regs, unsigned int instr) { + if (user_mode(regs)) + return -1; kgdb_handle_exception(1, SIGTRAP, 0, regs); return 0; @@ -147,6 +149,8 @@ static int kgdb_brk_fn(struct pt_regs *regs, unsigned int instr) static int kgdb_compiled_brk_fn(struct pt_regs *regs, unsigned int instr) { + if (user_mode(regs)) + return -1; compiled_break = 1; kgdb_handle_exception(1, SIGTRAP, 0, regs); diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c index 91d2d5b014145d..c6324b534b9df2 100644 --- a/arch/arm/kernel/process.c +++ b/arch/arm/kernel/process.c @@ -80,6 +80,7 @@ void arch_cpu_idle_prepare(void) void arch_cpu_idle_enter(void) { + idle_notifier_call_chain(IDLE_START); ledtrig_cpu(CPU_LED_IDLE_START); #ifdef CONFIG_PL310_ERRATA_769419 wmb(); @@ -89,6 +90,78 @@ void arch_cpu_idle_enter(void) void arch_cpu_idle_exit(void) { ledtrig_cpu(CPU_LED_IDLE_END); + idle_notifier_call_chain(IDLE_END); +} + +/* + * dump a block of kernel memory from around the given address + */ +static void show_data(unsigned long addr, int nbytes, const char *name) +{ + int i, j; + int nlines; + u32 *p; + + /* + * don't attempt to dump non-kernel addresses or + * values that are probably just small negative numbers + */ + if (addr < PAGE_OFFSET || addr > -256UL) + return; + + printk("\n%s: %#lx:\n", name, addr); + + /* + * round address down to a 32 bit boundary + * and always dump a multiple of 32 bytes + */ + p = (u32 *)(addr & ~(sizeof(u32) - 1)); + nbytes += (addr & (sizeof(u32) - 1)); + nlines = (nbytes + 31) / 32; + + + for (i = 0; i < nlines; i++) { + /* + * just display low 16 bits of address to keep + * each line of the dump < 80 characters + */ + printk("%04lx ", (unsigned long)p & 0xffff); + for (j = 0; j < 8; j++) { + u32 data; + if (probe_kernel_address(p, data)) { + printk(" ********"); + } else { + printk(" %08x", data); + } + ++p; + } + printk("\n"); + } +} + +static void show_extra_register_data(struct pt_regs *regs, int nbytes) +{ + mm_segment_t fs; + + fs = get_fs(); + set_fs(KERNEL_DS); + show_data(regs->ARM_pc - nbytes, nbytes * 2, "PC"); + show_data(regs->ARM_lr - nbytes, nbytes * 2, "LR"); + show_data(regs->ARM_sp - nbytes, nbytes * 2, "SP"); + show_data(regs->ARM_ip - nbytes, nbytes * 2, "IP"); + show_data(regs->ARM_fp - nbytes, nbytes * 2, "FP"); + show_data(regs->ARM_r0 - nbytes, nbytes * 2, "R0"); + show_data(regs->ARM_r1 - nbytes, nbytes * 2, "R1"); + show_data(regs->ARM_r2 - nbytes, nbytes * 2, "R2"); + show_data(regs->ARM_r3 - nbytes, nbytes * 2, "R3"); + show_data(regs->ARM_r4 - nbytes, nbytes * 2, "R4"); + show_data(regs->ARM_r5 - nbytes, nbytes * 2, "R5"); + show_data(regs->ARM_r6 - nbytes, nbytes * 2, "R6"); + show_data(regs->ARM_r7 - nbytes, nbytes * 2, "R7"); + show_data(regs->ARM_r8 - nbytes, nbytes * 2, "R8"); + show_data(regs->ARM_r9 - nbytes, nbytes * 2, "R9"); + show_data(regs->ARM_r10 - nbytes, nbytes * 2, "R10"); + set_fs(fs); } void __show_regs(struct pt_regs *regs) @@ -182,6 +255,8 @@ void __show_regs(struct pt_regs *regs) printk("Control: %08x%s\n", ctrl, buf); } #endif + + show_extra_register_data(regs, 128); } void show_regs(struct pt_regs * regs) diff --git a/arch/arm/kernel/ptrace.c b/arch/arm/kernel/ptrace.c index ce131ed5939d5f..ae738a6319f6a3 100644 --- a/arch/arm/kernel/ptrace.c +++ b/arch/arm/kernel/ptrace.c @@ -600,7 +600,7 @@ static int gpr_set(struct task_struct *target, const void *kbuf, const void __user *ubuf) { int ret; - struct pt_regs newregs; + struct pt_regs newregs = *task_pt_regs(target); ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &newregs, diff --git a/arch/arm/kernel/reboot.c b/arch/arm/kernel/reboot.c index 3fa867a2aae672..d704df89a54673 100644 --- a/arch/arm/kernel/reboot.c +++ b/arch/arm/kernel/reboot.c @@ -6,6 +6,7 @@ * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. */ +#include #include #include #include @@ -122,6 +123,31 @@ void machine_power_off(void) pm_power_off(); } +#ifdef CONFIG_ARM_FLUSH_CONSOLE_ON_RESTART +void arm_machine_flush_console(void) +{ + printk("\n"); + pr_emerg("Restarting %s\n", linux_banner); + if (console_trylock()) { + console_unlock(); + return; + } + + mdelay(50); + + local_irq_disable(); + if (!console_trylock()) + pr_emerg("arm_restart: Console was locked! Busting\n"); + else + pr_emerg("arm_restart: Console was locked!\n"); + console_unlock(); +} +#else +void arm_machine_flush_console(void) +{ +} +#endif + /* * Restart requires that the secondary CPUs stop performing any activity * while the primary CPU resets the system. Systems with a single CPU can @@ -138,6 +164,10 @@ void machine_restart(char *cmd) local_irq_disable(); smp_send_stop(); + /* Flush the console to make sure all the relevant messages make it + * out to the console drivers */ + arm_machine_flush_console(); + if (arm_pm_restart) arm_pm_restart(reboot_mode, cmd); else diff --git a/arch/arm/kernel/smp_tlb.c b/arch/arm/kernel/smp_tlb.c index 22313cb5336257..9af0701f7094be 100644 --- a/arch/arm/kernel/smp_tlb.c +++ b/arch/arm/kernel/smp_tlb.c @@ -9,6 +9,7 @@ */ #include #include +#include #include #include @@ -40,8 +41,11 @@ static inline void ipi_flush_tlb_mm(void *arg) static inline void ipi_flush_tlb_page(void *arg) { struct tlb_args *ta = (struct tlb_args *)arg; + unsigned int __ua_flags = uaccess_save_and_enable(); local_flush_tlb_page(ta->ta_vma, ta->ta_start); + + uaccess_restore(__ua_flags); } static inline void ipi_flush_tlb_kernel_page(void *arg) @@ -54,8 +58,11 @@ static inline void ipi_flush_tlb_kernel_page(void *arg) static inline void ipi_flush_tlb_range(void *arg) { struct tlb_args *ta = (struct tlb_args *)arg; + unsigned int __ua_flags = uaccess_save_and_enable(); local_flush_tlb_range(ta->ta_vma, ta->ta_start, ta->ta_end); + + uaccess_restore(__ua_flags); } static inline void ipi_flush_tlb_kernel_range(void *arg) diff --git a/arch/arm/kernel/topology.c b/arch/arm/kernel/topology.c index ec279d161b3287..df3020071f3206 100644 --- a/arch/arm/kernel/topology.c +++ b/arch/arm/kernel/topology.c @@ -42,9 +42,15 @@ */ static DEFINE_PER_CPU(unsigned long, cpu_scale) = SCHED_CAPACITY_SCALE; -unsigned long arch_scale_cpu_capacity(struct sched_domain *sd, int cpu) +unsigned long scale_cpu_capacity(struct sched_domain *sd, int cpu) { +#ifdef CONFIG_CPU_FREQ + unsigned long max_freq_scale = cpufreq_scale_max_freq_capacity(cpu); + + return per_cpu(cpu_scale, cpu) * max_freq_scale >> SCHED_CAPACITY_SHIFT; +#else return per_cpu(cpu_scale, cpu); +#endif } static void set_capacity_scale(unsigned int cpu, unsigned long capacity) @@ -153,6 +159,8 @@ static void __init parse_dt_topology(void) } +static const struct sched_group_energy * const cpu_core_energy(int cpu); + /* * Look for a customed capacity of a CPU in the cpu_capacity table during the * boot. The update of all CPUs is in O(n^2) for heteregeneous system but the @@ -160,10 +168,14 @@ static void __init parse_dt_topology(void) */ static void update_cpu_capacity(unsigned int cpu) { - if (!cpu_capacity(cpu)) - return; + unsigned long capacity = SCHED_CAPACITY_SCALE; + + if (cpu_core_energy(cpu)) { + int max_cap_idx = cpu_core_energy(cpu)->nr_cap_states - 1; + capacity = cpu_core_energy(cpu)->cap_states[max_cap_idx].cap; + } - set_capacity_scale(cpu, cpu_capacity(cpu) / middle_capacity); + set_capacity_scale(cpu, capacity); pr_info("CPU%u: update cpu_capacity %lu\n", cpu, arch_scale_cpu_capacity(NULL, cpu)); @@ -275,17 +287,138 @@ void store_cpu_topology(unsigned int cpuid) cpu_topology[cpuid].socket_id, mpidr); } +/* + * ARM TC2 specific energy cost model data. There are no unit requirements for + * the data. Data can be normalized to any reference point, but the + * normalization must be consistent. That is, one bogo-joule/watt must be the + * same quantity for all data, but we don't care what it is. + */ +static struct idle_state idle_states_cluster_a7[] = { + { .power = 25 }, /* arch_cpu_idle() (active idle) = WFI */ + { .power = 25 }, /* WFI */ + { .power = 10 }, /* cluster-sleep-l */ + }; + +static struct idle_state idle_states_cluster_a15[] = { + { .power = 70 }, /* arch_cpu_idle() (active idle) = WFI */ + { .power = 70 }, /* WFI */ + { .power = 25 }, /* cluster-sleep-b */ + }; + +static struct capacity_state cap_states_cluster_a7[] = { + /* Cluster only power */ + { .cap = 150, .power = 2967, }, /* 350 MHz */ + { .cap = 172, .power = 2792, }, /* 400 MHz */ + { .cap = 215, .power = 2810, }, /* 500 MHz */ + { .cap = 258, .power = 2815, }, /* 600 MHz */ + { .cap = 301, .power = 2919, }, /* 700 MHz */ + { .cap = 344, .power = 2847, }, /* 800 MHz */ + { .cap = 387, .power = 3917, }, /* 900 MHz */ + { .cap = 430, .power = 4905, }, /* 1000 MHz */ + }; + +static struct capacity_state cap_states_cluster_a15[] = { + /* Cluster only power */ + { .cap = 426, .power = 7920, }, /* 500 MHz */ + { .cap = 512, .power = 8165, }, /* 600 MHz */ + { .cap = 597, .power = 8172, }, /* 700 MHz */ + { .cap = 682, .power = 8195, }, /* 800 MHz */ + { .cap = 768, .power = 8265, }, /* 900 MHz */ + { .cap = 853, .power = 8446, }, /* 1000 MHz */ + { .cap = 938, .power = 11426, }, /* 1100 MHz */ + { .cap = 1024, .power = 15200, }, /* 1200 MHz */ + }; + +static struct sched_group_energy energy_cluster_a7 = { + .nr_idle_states = ARRAY_SIZE(idle_states_cluster_a7), + .idle_states = idle_states_cluster_a7, + .nr_cap_states = ARRAY_SIZE(cap_states_cluster_a7), + .cap_states = cap_states_cluster_a7, +}; + +static struct sched_group_energy energy_cluster_a15 = { + .nr_idle_states = ARRAY_SIZE(idle_states_cluster_a15), + .idle_states = idle_states_cluster_a15, + .nr_cap_states = ARRAY_SIZE(cap_states_cluster_a15), + .cap_states = cap_states_cluster_a15, +}; + +static struct idle_state idle_states_core_a7[] = { + { .power = 0 }, /* arch_cpu_idle (active idle) = WFI */ + { .power = 0 }, /* WFI */ + { .power = 0 }, /* cluster-sleep-l */ + }; + +static struct idle_state idle_states_core_a15[] = { + { .power = 0 }, /* arch_cpu_idle (active idle) = WFI */ + { .power = 0 }, /* WFI */ + { .power = 0 }, /* cluster-sleep-b */ + }; + +static struct capacity_state cap_states_core_a7[] = { + /* Power per cpu */ + { .cap = 150, .power = 187, }, /* 350 MHz */ + { .cap = 172, .power = 275, }, /* 400 MHz */ + { .cap = 215, .power = 334, }, /* 500 MHz */ + { .cap = 258, .power = 407, }, /* 600 MHz */ + { .cap = 301, .power = 447, }, /* 700 MHz */ + { .cap = 344, .power = 549, }, /* 800 MHz */ + { .cap = 387, .power = 761, }, /* 900 MHz */ + { .cap = 430, .power = 1024, }, /* 1000 MHz */ + }; + +static struct capacity_state cap_states_core_a15[] = { + /* Power per cpu */ + { .cap = 426, .power = 2021, }, /* 500 MHz */ + { .cap = 512, .power = 2312, }, /* 600 MHz */ + { .cap = 597, .power = 2756, }, /* 700 MHz */ + { .cap = 682, .power = 3125, }, /* 800 MHz */ + { .cap = 768, .power = 3524, }, /* 900 MHz */ + { .cap = 853, .power = 3846, }, /* 1000 MHz */ + { .cap = 938, .power = 5177, }, /* 1100 MHz */ + { .cap = 1024, .power = 6997, }, /* 1200 MHz */ + }; + +static struct sched_group_energy energy_core_a7 = { + .nr_idle_states = ARRAY_SIZE(idle_states_core_a7), + .idle_states = idle_states_core_a7, + .nr_cap_states = ARRAY_SIZE(cap_states_core_a7), + .cap_states = cap_states_core_a7, +}; + +static struct sched_group_energy energy_core_a15 = { + .nr_idle_states = ARRAY_SIZE(idle_states_core_a15), + .idle_states = idle_states_core_a15, + .nr_cap_states = ARRAY_SIZE(cap_states_core_a15), + .cap_states = cap_states_core_a15, +}; + +/* sd energy functions */ +static inline +const struct sched_group_energy * const cpu_cluster_energy(int cpu) +{ + return cpu_topology[cpu].socket_id ? &energy_cluster_a7 : + &energy_cluster_a15; +} + +static inline +const struct sched_group_energy * const cpu_core_energy(int cpu) +{ + return cpu_topology[cpu].socket_id ? &energy_core_a7 : + &energy_core_a15; +} + static inline int cpu_corepower_flags(void) { - return SD_SHARE_PKG_RESOURCES | SD_SHARE_POWERDOMAIN; + return SD_SHARE_PKG_RESOURCES | SD_SHARE_POWERDOMAIN | \ + SD_SHARE_CAP_STATES; } static struct sched_domain_topology_level arm_topology[] = { #ifdef CONFIG_SCHED_MC - { cpu_corepower_mask, cpu_corepower_flags, SD_INIT_NAME(GMC) }, - { cpu_coregroup_mask, cpu_core_flags, SD_INIT_NAME(MC) }, + { cpu_coregroup_mask, cpu_corepower_flags, cpu_core_energy, SD_INIT_NAME(MC) }, #endif - { cpu_cpu_mask, SD_INIT_NAME(DIE) }, + { cpu_cpu_mask, NULL, cpu_cluster_energy, SD_INIT_NAME(DIE) }, { NULL, }, }; diff --git a/arch/arm/mach-davinci/da850.c b/arch/arm/mach-davinci/da850.c index ed3d0e9f72ac35..d7a43afdfac0e2 100644 --- a/arch/arm/mach-davinci/da850.c +++ b/arch/arm/mach-davinci/da850.c @@ -319,6 +319,16 @@ static struct clk emac_clk = { .gpsc = 1, }; +/* + * In order to avoid adding the emac_clk to the clock lookup table twice (and + * screwing up the linked list in the process) create a separate clock for + * mdio inheriting the rate from emac_clk. + */ +static struct clk mdio_clk = { + .name = "mdio", + .parent = &emac_clk, +}; + static struct clk mcasp_clk = { .name = "mcasp", .parent = &async3_clk, @@ -494,7 +504,7 @@ static struct clk_lookup da850_clks[] = { CLK(NULL, "arm", &arm_clk), CLK(NULL, "rmii", &rmii_clk), CLK("davinci_emac.1", NULL, &emac_clk), - CLK("davinci_mdio.0", "fck", &emac_clk), + CLK("davinci_mdio.0", "fck", &mdio_clk), CLK("davinci-mcasp.0", NULL, &mcasp_clk), CLK("davinci-mcbsp.0", NULL, &mcbsp0_clk), CLK("davinci-mcbsp.1", NULL, &mcbsp1_clk), diff --git a/arch/arm/mach-omap2/Makefile b/arch/arm/mach-omap2/Makefile index 5b37ec29996e9d..e37ceb81a379cd 100644 --- a/arch/arm/mach-omap2/Makefile +++ b/arch/arm/mach-omap2/Makefile @@ -80,7 +80,7 @@ endif # Power Management omap-4-5-pm-common = omap-mpuss-lowpower.o obj-$(CONFIG_ARCH_OMAP4) += $(omap-4-5-pm-common) -obj-$(CONFIG_ARCH_OMAP5) += $(omap-4-5-pm-common) +obj-$(CONFIG_SOC_OMAP5) += $(omap-4-5-pm-common) obj-$(CONFIG_OMAP_PM_NOOP) += omap-pm-noop.o ifeq ($(CONFIG_PM),y) diff --git a/arch/arm/mach-omap2/board-generic.c b/arch/arm/mach-omap2/board-generic.c index bab814d2f37dcc..60f5e838a3bccc 100644 --- a/arch/arm/mach-omap2/board-generic.c +++ b/arch/arm/mach-omap2/board-generic.c @@ -306,7 +306,7 @@ DT_MACHINE_START(AM43_DT, "Generic AM43 (Flattened Device Tree)") .init_late = am43xx_init_late, .init_irq = omap_gic_of_init, .init_machine = omap_generic_init, - .init_time = omap4_local_timer_init, + .init_time = omap3_gptimer_timer_init, .dt_compat = am43_boards_compat, .restart = omap44xx_restart, MACHINE_END diff --git a/arch/arm/mach-omap2/common.h b/arch/arm/mach-omap2/common.h index deed42e1dd9c90..6dcca2957e23ef 100644 --- a/arch/arm/mach-omap2/common.h +++ b/arch/arm/mach-omap2/common.h @@ -262,8 +262,6 @@ extern void __iomem *omap4_get_sar_ram_base(void); extern void omap4_mpuss_early_init(void); extern void omap_do_wfi(void); -extern void omap4_secondary_startup(void); -extern void omap4460_secondary_startup(void); #ifdef CONFIG_SMP /* Needed for secondary core boot */ @@ -275,16 +273,11 @@ extern void omap4_cpu_die(unsigned int cpu); extern int omap4_cpu_kill(unsigned int cpu); extern const struct smp_operations omap4_smp_ops; - -extern void omap5_secondary_startup(void); -extern void omap5_secondary_hyp_startup(void); #endif #if defined(CONFIG_SMP) && defined(CONFIG_PM) extern int omap4_mpuss_init(void); extern int omap4_enter_lowpower(unsigned int cpu, unsigned int power_state); -extern int omap4_finish_suspend(unsigned long cpu_state); -extern void omap4_cpu_resume(void); extern int omap4_hotplug_cpu(unsigned int cpu, unsigned int power_state); #else static inline int omap4_enter_lowpower(unsigned int cpu, @@ -305,14 +298,41 @@ static inline int omap4_mpuss_init(void) return 0; } +#endif + +#ifdef CONFIG_ARCH_OMAP4 +void omap4_secondary_startup(void); +void omap4460_secondary_startup(void); +int omap4_finish_suspend(unsigned long cpu_state); +void omap4_cpu_resume(void); +#else +static inline void omap4_secondary_startup(void) +{ +} + +static inline void omap4460_secondary_startup(void) +{ +} static inline int omap4_finish_suspend(unsigned long cpu_state) { return 0; } - static inline void omap4_cpu_resume(void) -{} +{ +} +#endif +#if defined(CONFIG_SOC_OMAP5) || defined(CONFIG_SOC_DRA7XX) +void omap5_secondary_startup(void); +void omap5_secondary_hyp_startup(void); +#else +static inline void omap5_secondary_startup(void) +{ +} + +static inline void omap5_secondary_hyp_startup(void) +{ +} #endif void pdata_quirks_init(const struct of_device_id *); diff --git a/arch/arm/mach-omap2/io.c b/arch/arm/mach-omap2/io.c index 0e9acdd95d707d..f0da5259762aa4 100644 --- a/arch/arm/mach-omap2/io.c +++ b/arch/arm/mach-omap2/io.c @@ -717,10 +717,11 @@ void __init omap5_init_early(void) OMAP2_L4_IO_ADDRESS(OMAP54XX_SCM_BASE)); omap2_set_globals_prcm_mpu(OMAP2_L4_IO_ADDRESS(OMAP54XX_PRCM_MPU_BASE)); omap2_control_base_init(); - omap4_pm_init_early(); omap2_prcm_base_init(); omap5xxx_check_revision(); omap4_sar_ram_init(); + omap4_mpuss_early_init(); + omap4_pm_init_early(); omap54xx_voltagedomains_init(); omap54xx_powerdomains_init(); omap54xx_clockdomains_init(); diff --git a/arch/arm/mach-omap2/omap-mpuss-lowpower.c b/arch/arm/mach-omap2/omap-mpuss-lowpower.c index ad982465efd0d0..7d62ad48c7c9dd 100644 --- a/arch/arm/mach-omap2/omap-mpuss-lowpower.c +++ b/arch/arm/mach-omap2/omap-mpuss-lowpower.c @@ -48,6 +48,7 @@ #include #include #include +#include #include #include "soc.h" @@ -244,10 +245,9 @@ int omap4_enter_lowpower(unsigned int cpu, unsigned int power_state) save_state = 1; break; case PWRDM_POWER_RET: - if (IS_PM44XX_ERRATUM(PM_OMAP4_CPU_OSWR_DISABLE)) { + if (IS_PM44XX_ERRATUM(PM_OMAP4_CPU_OSWR_DISABLE)) save_state = 0; - break; - } + break; default: /* * CPUx CSWR is invalid hardware state. Also CPUx OSWR @@ -371,8 +371,12 @@ int __init omap4_mpuss_init(void) pm_info = &per_cpu(omap4_pm_info, 0x0); if (sar_base) { pm_info->scu_sar_addr = sar_base + SCU_OFFSET0; - pm_info->wkup_sar_addr = sar_base + - CPU0_WAKEUP_NS_PA_ADDR_OFFSET; + if (cpu_is_omap44xx()) + pm_info->wkup_sar_addr = sar_base + + CPU0_WAKEUP_NS_PA_ADDR_OFFSET; + else + pm_info->wkup_sar_addr = sar_base + + OMAP5_CPU0_WAKEUP_NS_PA_ADDR_OFFSET; pm_info->l2x0_sar_addr = sar_base + L2X0_SAVE_OFFSET0; } pm_info->pwrdm = pwrdm_lookup("cpu0_pwrdm"); @@ -391,8 +395,12 @@ int __init omap4_mpuss_init(void) pm_info = &per_cpu(omap4_pm_info, 0x1); if (sar_base) { pm_info->scu_sar_addr = sar_base + SCU_OFFSET1; - pm_info->wkup_sar_addr = sar_base + - CPU1_WAKEUP_NS_PA_ADDR_OFFSET; + if (cpu_is_omap44xx()) + pm_info->wkup_sar_addr = sar_base + + CPU1_WAKEUP_NS_PA_ADDR_OFFSET; + else + pm_info->wkup_sar_addr = sar_base + + OMAP5_CPU1_WAKEUP_NS_PA_ADDR_OFFSET; pm_info->l2x0_sar_addr = sar_base + L2X0_SAVE_OFFSET1; } @@ -453,15 +461,24 @@ void __init omap4_mpuss_early_init(void) { unsigned long startup_pa; - if (!cpu_is_omap44xx()) + if (!(cpu_is_omap44xx() || soc_is_omap54xx())) return; sar_base = omap4_get_sar_ram_base(); if (cpu_is_omap443x()) startup_pa = virt_to_phys(omap4_secondary_startup); - else + else if (cpu_is_omap446x()) startup_pa = virt_to_phys(omap4460_secondary_startup); + else if ((__boot_cpu_mode & MODE_MASK) == HYP_MODE) + startup_pa = virt_to_phys(omap5_secondary_hyp_startup); + else + startup_pa = virt_to_phys(omap5_secondary_startup); - writel_relaxed(startup_pa, sar_base + CPU1_WAKEUP_NS_PA_ADDR_OFFSET); + if (cpu_is_omap44xx()) + writel_relaxed(startup_pa, sar_base + + CPU1_WAKEUP_NS_PA_ADDR_OFFSET); + else + writel_relaxed(startup_pa, sar_base + + OMAP5_CPU1_WAKEUP_NS_PA_ADDR_OFFSET); } diff --git a/arch/arm/mach-omap2/omap4-sar-layout.h b/arch/arm/mach-omap2/omap4-sar-layout.h index 792b1069f724ff..5b2966a0f73308 100644 --- a/arch/arm/mach-omap2/omap4-sar-layout.h +++ b/arch/arm/mach-omap2/omap4-sar-layout.h @@ -31,6 +31,8 @@ /* CPUx Wakeup Non-Secure Physical Address offsets in SAR_BANK3 */ #define CPU0_WAKEUP_NS_PA_ADDR_OFFSET 0xa04 #define CPU1_WAKEUP_NS_PA_ADDR_OFFSET 0xa08 +#define OMAP5_CPU0_WAKEUP_NS_PA_ADDR_OFFSET 0xe00 +#define OMAP5_CPU1_WAKEUP_NS_PA_ADDR_OFFSET 0xe04 #define SAR_BACKUP_STATUS_OFFSET (SAR_BANK3_OFFSET + 0x500) #define SAR_SECURE_RAM_SIZE_OFFSET (SAR_BANK3_OFFSET + 0x504) diff --git a/arch/arm/mach-omap2/timer.c b/arch/arm/mach-omap2/timer.c index 5e2e2218a4023a..b2f2448bfa6dc0 100644 --- a/arch/arm/mach-omap2/timer.c +++ b/arch/arm/mach-omap2/timer.c @@ -510,18 +510,19 @@ void __init omap3_secure_sync32k_timer_init(void) } #endif /* CONFIG_ARCH_OMAP3 */ -#if defined(CONFIG_ARCH_OMAP3) || defined(CONFIG_SOC_AM33XX) +#if defined(CONFIG_ARCH_OMAP3) || defined(CONFIG_SOC_AM33XX) || \ + defined(CONFIG_SOC_AM43XX) void __init omap3_gptimer_timer_init(void) { __omap_sync32k_timer_init(2, "timer_sys_ck", NULL, 1, "timer_sys_ck", "ti,timer-alwon", true); - - clocksource_probe(); + if (of_have_populated_dt()) + clocksource_probe(); } #endif #if defined(CONFIG_ARCH_OMAP4) || defined(CONFIG_SOC_OMAP5) || \ - defined(CONFIG_SOC_DRA7XX) || defined(CONFIG_SOC_AM43XX) + defined(CONFIG_SOC_DRA7XX) static void __init omap4_sync32k_timer_init(void) { __omap_sync32k_timer_init(1, "timer_32k_ck", "ti,timer-alwon", diff --git a/arch/arm/mach-pxa/pxa25x.c b/arch/arm/mach-pxa/pxa25x.c index 12b94357fbc11f..c725baf119e113 100644 --- a/arch/arm/mach-pxa/pxa25x.c +++ b/arch/arm/mach-pxa/pxa25x.c @@ -156,7 +156,7 @@ static int __init __init pxa25x_dt_init_irq(struct device_node *node, struct device_node *parent) { pxa_dt_irq_init(pxa25x_set_wake); - set_handle_irq(ichp_handle_irq); + set_handle_irq(icip_handle_irq); return 0; } diff --git a/arch/arm/mach-ux500/pm.c b/arch/arm/mach-ux500/pm.c index 8538910db202ab..a970e7fcba9e02 100644 --- a/arch/arm/mach-ux500/pm.c +++ b/arch/arm/mach-ux500/pm.c @@ -134,8 +134,8 @@ bool prcmu_pending_irq(void) */ bool prcmu_is_cpu_in_wfi(int cpu) { - return readl(PRCM_ARM_WFI_STANDBY) & cpu ? PRCM_ARM_WFI_STANDBY_WFI1 : - PRCM_ARM_WFI_STANDBY_WFI0; + return readl(PRCM_ARM_WFI_STANDBY) & + (cpu ? PRCM_ARM_WFI_STANDBY_WFI1 : PRCM_ARM_WFI_STANDBY_WFI0); } /* diff --git a/arch/arm/mach-zynq/common.c b/arch/arm/mach-zynq/common.c index d12002cd63bc6f..ed118648313f63 100644 --- a/arch/arm/mach-zynq/common.c +++ b/arch/arm/mach-zynq/common.c @@ -59,7 +59,7 @@ void __iomem *zynq_scu_base; static void __init zynq_memory_init(void) { if (!__pa(PAGE_OFFSET)) - memblock_reserve(__pa(PAGE_OFFSET), __pa(swapper_pg_dir)); + memblock_reserve(__pa(PAGE_OFFSET), 0x80000); } static struct platform_device zynq_cpuidle_device = { diff --git a/arch/arm/mm/cache-v6.S b/arch/arm/mm/cache-v6.S index 24659952c2784d..11da0f50a1fef8 100644 --- a/arch/arm/mm/cache-v6.S +++ b/arch/arm/mm/cache-v6.S @@ -270,6 +270,11 @@ v6_dma_clean_range: * - end - virtual end address of region */ ENTRY(v6_dma_flush_range) +#ifdef CONFIG_CACHE_FLUSH_RANGE_LIMIT + sub r2, r1, r0 + cmp r2, #CONFIG_CACHE_FLUSH_RANGE_LIMIT + bhi v6_dma_flush_dcache_all +#endif #ifdef CONFIG_DMA_CACHE_RWFO ldrb r2, [r0] @ read for ownership strb r2, [r0] @ write for ownership @@ -292,6 +297,18 @@ ENTRY(v6_dma_flush_range) mcr p15, 0, r0, c7, c10, 4 @ drain write buffer ret lr +#ifdef CONFIG_CACHE_FLUSH_RANGE_LIMIT +v6_dma_flush_dcache_all: + mov r0, #0 +#ifdef HARVARD_CACHE + mcr p15, 0, r0, c7, c14, 0 @ D cache clean+invalidate +#else + mcr p15, 0, r0, c7, c15, 0 @ Cache clean+invalidate +#endif + mcr p15, 0, r0, c7, c10, 4 @ drain write buffer + mov pc, lr +#endif + /* * dma_map_area(start, size, dir) * - start - kernel virtual start address diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c index 3a2e678b8d30ca..aec74bf85b3963 100644 --- a/arch/arm/mm/fault.c +++ b/arch/arm/mm/fault.c @@ -273,10 +273,10 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) local_irq_enable(); /* - * If we're in an interrupt or have no user + * If we're in an interrupt, or have no irqs, or have no user * context, we must not take the fault.. */ - if (faulthandler_disabled() || !mm) + if (faulthandler_disabled() || irqs_disabled() || !mm) goto no_context; if (user_mode(regs)) @@ -610,9 +610,9 @@ static int __init early_abort_handler(unsigned long addr, unsigned int fsr, void __init early_abt_enable(void) { - fsr_info[22].fn = early_abort_handler; + fsr_info[FSR_FS_AEA].fn = early_abort_handler; local_abt_enable(); - fsr_info[22].fn = do_bad; + fsr_info[FSR_FS_AEA].fn = do_bad; } #ifndef CONFIG_ARM_LPAE diff --git a/arch/arm/mm/fault.h b/arch/arm/mm/fault.h index 67532f24227105..afc1f84e763b24 100644 --- a/arch/arm/mm/fault.h +++ b/arch/arm/mm/fault.h @@ -11,11 +11,15 @@ #define FSR_FS5_0 (0x3f) #ifdef CONFIG_ARM_LPAE +#define FSR_FS_AEA 17 + static inline int fsr_fs(unsigned int fsr) { return fsr & FSR_FS5_0; } #else +#define FSR_FS_AEA 22 + static inline int fsr_fs(unsigned int fsr) { return (fsr & FSR_FS3_0) | (fsr & FSR_FS4) >> 6; diff --git a/arch/arm/xen/enlighten.c b/arch/arm/xen/enlighten.c index f193414d0f6ff7..4986dc0c1dff05 100644 --- a/arch/arm/xen/enlighten.c +++ b/arch/arm/xen/enlighten.c @@ -372,8 +372,7 @@ static int __init xen_guest_init(void) * for secondary CPUs as they are brought up. * For uniformity we use VCPUOP_register_vcpu_info even on cpu0. */ - xen_vcpu_info = __alloc_percpu(sizeof(struct vcpu_info), - sizeof(struct vcpu_info)); + xen_vcpu_info = alloc_percpu(struct vcpu_info); if (xen_vcpu_info == NULL) return -ENOMEM; diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 969ef880d234e3..9d0edb767fd845 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -949,6 +949,23 @@ config CMDLINE entering them here. As a minimum, you should specify the the root device (e.g. root=/dev/nfs). +choice + prompt "Kernel command line type" if CMDLINE != "" + default CMDLINE_FROM_BOOTLOADER + +config CMDLINE_FROM_BOOTLOADER + bool "Use bootloader kernel arguments if available" + help + Uses the command-line options passed by the boot loader. If + the boot loader doesn't provide any, the default kernel command + string provided in CMDLINE will be used. + +config CMDLINE_EXTEND + bool "Extend bootloader kernel arguments" + help + The command-line arguments provided by the boot loader will be + appended to the default kernel command string. + config CMDLINE_FORCE bool "Always use the default kernel command string" help @@ -956,6 +973,7 @@ config CMDLINE_FORCE loader passes other arguments to the kernel. This is useful if you cannot or don't want to change the command-line options your boot loader passes to the kernel. +endchoice config EFI_STUB bool @@ -988,6 +1006,21 @@ config DMI However, even with this option, the resultant kernel should continue to boot on existing non-UEFI platforms. +config BUILD_ARM64_APPENDED_DTB_IMAGE + bool "Build a concatenated Image.gz/dtb by default" + depends on OF + help + Enabling this option will cause a concatenated Image.gz and list of + DTBs to be built by default (instead of a standalone Image.gz.) + The image will built in arch/arm64/boot/Image.gz-dtb + +config BUILD_ARM64_APPENDED_DTB_IMAGE_NAMES + string "Default dtb names" + depends on BUILD_ARM64_APPENDED_DTB_IMAGE + help + Space separated list of names of dtbs to append when + building a concatenated Image.gz-dtb. + endmenu menu "Userspace binary formats" diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile index 3635b866272456..8aeb4bd41f19be 100644 --- a/arch/arm64/Makefile +++ b/arch/arm64/Makefile @@ -40,6 +40,7 @@ endif KBUILD_CFLAGS += -mgeneral-regs-only $(lseinstr) KBUILD_CFLAGS += -fno-asynchronous-unwind-tables KBUILD_CFLAGS += $(call cc-option, -mpc-relative-literal-loads) +KBUILD_CFLAGS += -fno-pic KBUILD_AFLAGS += $(lseinstr) ifeq ($(CONFIG_CPU_BIG_ENDIAN), y) @@ -94,7 +95,12 @@ libs-y := arch/arm64/lib/ $(libs-y) core-$(CONFIG_EFI_STUB) += $(objtree)/drivers/firmware/efi/libstub/lib.a # Default target when executing plain make +ifeq ($(CONFIG_BUILD_ARM64_APPENDED_DTB_IMAGE),y) +KBUILD_IMAGE := Image-dtb +else KBUILD_IMAGE := Image.gz +endif + KBUILD_DTBS := dtbs all: $(KBUILD_IMAGE) $(KBUILD_DTBS) @@ -121,6 +127,9 @@ dtbs: prepare scripts dtbs_install: $(Q)$(MAKE) $(dtbinst)=$(boot)/dts +Image-dtb Image.gz-dtb: vmlinux scripts dtbs + $(Q)$(MAKE) $(build)=$(boot) $(boot)/$@ + PHONY += vdso_install vdso_install: $(Q)$(MAKE) $(build)=arch/arm64/kernel/vdso $@ diff --git a/arch/arm64/boot/.gitignore b/arch/arm64/boot/.gitignore index 8dab0bb6ae667c..34e35209fc2ed7 100644 --- a/arch/arm64/boot/.gitignore +++ b/arch/arm64/boot/.gitignore @@ -1,2 +1,4 @@ Image +Image-dtb Image.gz +Image.gz-dtb diff --git a/arch/arm64/boot/Makefile b/arch/arm64/boot/Makefile index 1f012c50643436..2c8cb864315e30 100644 --- a/arch/arm64/boot/Makefile +++ b/arch/arm64/boot/Makefile @@ -14,16 +14,29 @@ # Based on the ia64 boot/Makefile. # +include $(srctree)/arch/arm64/boot/dts/Makefile + OBJCOPYFLAGS_Image :=-O binary -R .note -R .note.gnu.build-id -R .comment -S targets := Image Image.gz +DTB_NAMES := $(subst $\",,$(CONFIG_BUILD_ARM64_APPENDED_DTB_IMAGE_NAMES)) +ifneq ($(DTB_NAMES),) +DTB_LIST := $(addsuffix .dtb,$(DTB_NAMES)) +else +DTB_LIST := $(dtb-y) +endif +DTB_OBJS := $(addprefix $(obj)/dts/,$(DTB_LIST)) + $(obj)/Image: vmlinux FORCE $(call if_changed,objcopy) $(obj)/Image.bz2: $(obj)/Image FORCE $(call if_changed,bzip2) +$(obj)/Image-dtb: $(obj)/Image $(DTB_OBJS) FORCE + $(call if_changed,cat) + $(obj)/Image.gz: $(obj)/Image FORCE $(call if_changed,gzip) @@ -36,6 +49,9 @@ $(obj)/Image.lzma: $(obj)/Image FORCE $(obj)/Image.lzo: $(obj)/Image FORCE $(call if_changed,lzo) +$(obj)/Image.gz-dtb: $(obj)/Image.gz $(DTB_OBJS) FORCE + $(call if_changed,cat) + install: $(CONFIG_SHELL) $(srctree)/$(src)/install.sh $(KERNELRELEASE) \ $(obj)/Image System.map "$(INSTALL_PATH)" diff --git a/arch/arm64/boot/dts/Makefile b/arch/arm64/boot/dts/Makefile index 6684f97c27223f..7ad2cf0a607b0f 100644 --- a/arch/arm64/boot/dts/Makefile +++ b/arch/arm64/boot/dts/Makefile @@ -28,3 +28,17 @@ dtstree := $(srctree)/$(src) dtb-$(CONFIG_OF_ALL_DTBS) := $(patsubst $(dtstree)/%.dts,%.dtb, $(foreach d,$(dts-dirs), $(wildcard $(dtstree)/$(d)/*.dts))) always := $(dtb-y) + +targets += dtbs + +DTB_NAMES := $(subst $\",,$(CONFIG_BUILD_ARM64_APPENDED_DTB_IMAGE_NAMES)) +ifneq ($(DTB_NAMES),) +DTB_LIST := $(addsuffix .dtb,$(DTB_NAMES)) +else +DTB_LIST := $(dtb-y) +endif +targets += $(DTB_LIST) + +dtbs: $(addprefix $(obj)/, $(DTB_LIST)) + +clean-files := dts/*.dtb *.dtb diff --git a/arch/arm64/boot/dts/broadcom/bcm2837-rpi-3-b.dts b/arch/arm64/boot/dts/broadcom/bcm2837-rpi-3-b.dts index 7841b724e340e1..4617759670da80 100644 --- a/arch/arm64/boot/dts/broadcom/bcm2837-rpi-3-b.dts +++ b/arch/arm64/boot/dts/broadcom/bcm2837-rpi-3-b.dts @@ -15,13 +15,6 @@ act { gpios = <&gpio 47 0>; }; - - pwr { - label = "PWR"; - gpios = <&gpio 35 0>; - default-state = "keep"; - linux,default-trigger = "default-on"; - }; }; }; diff --git a/arch/arm64/boot/dts/broadcom/bcm2837.dtsi b/arch/arm64/boot/dts/broadcom/bcm2837.dtsi index 8216bbb29fe075..c1f719b7097a60 100644 --- a/arch/arm64/boot/dts/broadcom/bcm2837.dtsi +++ b/arch/arm64/boot/dts/broadcom/bcm2837.dtsi @@ -1,7 +1,7 @@ #include "bcm283x.dtsi" / { - compatible = "brcm,bcm2836"; + compatible = "brcm,bcm2837"; soc { ranges = <0x7e000000 0x3f000000 0x1000000>, diff --git a/arch/arm64/boot/dts/hisilicon/hi6220-hikey.dts b/arch/arm64/boot/dts/hisilicon/hi6220-hikey.dts index dba3c131c62c09..12f0bed2efe5a4 100644 --- a/arch/arm64/boot/dts/hisilicon/hi6220-hikey.dts +++ b/arch/arm64/boot/dts/hisilicon/hi6220-hikey.dts @@ -19,12 +19,101 @@ serial1 = &uart1; /* BT UART */ serial2 = &uart2; /* LS Expansion UART0 */ serial3 = &uart3; /* LS Expansion UART1 */ + spi0 = &spi0; }; chosen { stdout-path = "serial3:115200n8"; }; + fiq-debugger { + compatible = "android,irq-hi6220-uart"; + reg = <0x0 0xf7113000 0x0 0x1000>; + interrupts = ; + interrupt-names = "fiq", "signal"; + }; + + uart_overlay@0 { + fragment@0 { + target-path="/soc/uart@f7113000"; + __overlay__ { + status = "disabled"; + }; + }; + }; + + overlay_mgr { + compatible = "linux,overlay_manager"; + hardware_cfg_innolux_panel { + overlay_0 { + fragment@0 { + target-path="/soc/dsi@f4107800"; + __overlay__ { /* For panel reg's value should >= 1 */ + #address-cells = <1>; + #size-cells = <0>; + ports { + #address-cells = <1>; + #size-cells = <0>; + /* 1 for output port */ + port@1 { + #address-cells = <1>; + #size-cells = <0>; + reg = <1>; + dsi_out1: endpoint@1 { + reg = <1>; + remote-endpoint = <&panel0_in>; + }; + }; + }; + panel@1 { + compatible = "innolux,n070icn-pb1"; + reg = <1>; + power-on-delay= <50>; + reset-delay = <100>; + init-delay = <100>; + panel-width-mm = <58>; + panel-height-mm = <103>; + pwr-en-gpio = <&gpio2 1 0>; + bl-en-gpio = <&gpio2 3 0>; + pwm-gpio = <&gpio12 7 0>; + + port { + panel0_in: endpoint { + remote-endpoint = <&dsi_out1>; + }; + }; + }; + }; + }; + }; + }; + hardware_cfg_spidev0 { + overlay_0 { + fragment@0 { + target-path="/soc/spi@f7106000/spidev@0"; + __overlay__ { + status = "ok"; + }; + }; + }; + }; + hardware_cfg_cs_sd_qb { + overlay_0 { + fragment@0 { /* Select SD from Quick Bus */ + target-path="/soc/gpio@f7020000"; + __overlay__ { + sd_sel { + gpio-hog; + gpios = <4 0>; + output-high; + line-name = "gpio_sd_sel"; + }; + }; + }; + }; + }; + }; + /* * Reserve below regions from memory node: * @@ -322,6 +411,27 @@ }; }; + kim { + compatible = "kim"; + pinctrl-names = "default"; + pinctrl-0 = <>; /* FIXME: add BT PCM pinctrl here */ + /* + * FIXME: The following is complete CRAP since + * the vendor driver doesn't follow the gpio + * binding. Passing in a magic Linux gpio number + * here until we fix the vendor driver. + */ + /* BT_EN: BT_REG_ON_GPIO1_7 */ + nshutdown_gpio = <503>; + dev_name = "/dev/ttyAMA1"; + flow_cntrl = <1>; + baud_rate = <3000000>; + }; + + btwilink { + compatible = "btwilink"; + }; + pmic: pmic@f8000000 { compatible = "hisilicon,hi655x-pmic"; reg = <0x0 0xf8000000 0x0 0x1000>; @@ -406,6 +516,30 @@ }; }; }; + + sound { + compatible = "simple-audio-card"; + simple-audio-card,name = "hikey-hdmi"; + simple-audio-card,format = "i2s"; + + simple-audio-card,bitclock-master = <&sound_master>; + simple-audio-card,frame-master = <&sound_master>; + + sound_master: simple-audio-card,cpu { + sound-dai = <&i2s0>; + }; + + simple-audio-card,codec { + sound-dai = <&adv7533>; + }; + }; + + firmware { + optee { + compatible = "linaro,optee-tz"; + method = "smc"; + }; + }; }; &uart2 { @@ -446,6 +580,7 @@ interrupts = <1 2>; pd-gpio = <&gpio0 4 0>; adi,dsi-lanes = <4>; + #sound-dai-cells = <0>; port { adv7533_in: endpoint { @@ -453,4 +588,17 @@ }; }; }; + }; + +&spi0 { + #address-cells = <1>; + #size-cells = <0>; + status = "ok"; + + spidev@0 { + compatible = "linux,spidev"; + spi-max-frequency = <500000>; + reg = <0>; + status = "disabled"; + }; }; diff --git a/arch/arm64/boot/dts/hisilicon/hi6220-sched-energy.dtsi b/arch/arm64/boot/dts/hisilicon/hi6220-sched-energy.dtsi new file mode 100644 index 00000000000000..6dfc49332b4f1c --- /dev/null +++ b/arch/arm64/boot/dts/hisilicon/hi6220-sched-energy.dtsi @@ -0,0 +1,69 @@ +/* + * Hikey specific energy cost model data. + */ + +/* static struct idle_state idle_states_cluster_a53[] = { */ +/* { .power = 47 }, /\* arch_cpu_idle() (active idle) = WFI *\/ */ +/* { .power = 47 }, /\* WFI *\/ */ +/* { .power = 47 }, /\* cpu-sleep-0 *\/ */ +/* { .power = 0 }, /\* cluster-sleep-0 *\/ */ +/* }; */ + +/* static struct capacity_state cap_states_cluster_a53[] = { */ +/* /\* Power per cluster *\/ */ +/* { .cap = 178, .power = 16, }, /\* 200 MHz *\/ */ +/* { .cap = 369, .power = 29, }, /\* 432 MHz *\/ */ +/* { .cap = 622, .power = 47, }, /\* 729 MHz *\/ */ +/* { .cap = 819, .power = 75, }, /\* 960 MHz *\/ */ +/* { .cap = 1024, .power = 112, }, /\* 1200 Mhz *\/ */ +/* }; */ + +/* static struct idle_state idle_states_core_a53[] = { */ +/* { .power = 15 }, /\* arch_cpu_idle() (active idle) = WFI *\/ */ +/* { .power = 15 }, /\* WFI *\/ */ +/* { .power = 0 }, /\* cpu-sleep-0 *\/ */ +/* { .power = 0 }, /\* cluster-sleep-0 *\/ */ +/* }; */ + +/* static struct capacity_state cap_states_core_a53[] = { */ +/* /\* Power per cpu *\/ */ +/* { .cap = 178, .power = 69, }, /\* 200 MHz *\/ */ +/* { .cap = 369, .power = 124, }, /\* 432 MHz *\/ */ +/* { .cap = 622, .power = 224, }, /\* 729 MHz *\/ */ +/* { .cap = 819, .power = 367, }, /\* 960 MHz *\/ */ +/* { .cap = 1024, .power = 670, }, /\* 1200 Mhz *\/ */ +/* }; */ + +energy-costs { + CPU_COST: core-cost { + busy-cost-data = < + 178 69 + 369 124 + 622 224 + 819 367 + 1024 670 + >; + idle-cost-data = < + 15 + 15 + 0 + 0 + >; + }; + + CLUSTER_COST: cluster-cost { + busy-cost-data = < + 178 16 + 369 29 + 622 47 + 819 75 + 1024 112 + >; + idle-cost-data = < + 47 + 47 + 47 + 0 + >; + }; +}; diff --git a/arch/arm64/boot/dts/hisilicon/hi6220.dtsi b/arch/arm64/boot/dts/hisilicon/hi6220.dtsi index 17839db585d5c8..b9fddbed520528 100644 --- a/arch/arm64/boot/dts/hisilicon/hi6220.dtsi +++ b/arch/arm64/boot/dts/hisilicon/hi6220.dtsi @@ -91,6 +91,7 @@ cooling-max-level = <0>; #cooling-cells = <2>; /* min followed by max */ cpu-idle-states = <&CPU_SLEEP &CLUSTER_SLEEP>; + sched-energy-costs = <&CPU_COST &CLUSTER_COST>; dynamic-power-coefficient = <311>; }; @@ -102,6 +103,7 @@ next-level-cache = <&CLUSTER0_L2>; operating-points-v2 = <&cpu_opp_table>; cpu-idle-states = <&CPU_SLEEP &CLUSTER_SLEEP>; + sched-energy-costs = <&CPU_COST &CLUSTER_COST>; }; cpu2: cpu@2 { @@ -112,6 +114,7 @@ next-level-cache = <&CLUSTER0_L2>; operating-points-v2 = <&cpu_opp_table>; cpu-idle-states = <&CPU_SLEEP &CLUSTER_SLEEP>; + sched-energy-costs = <&CPU_COST &CLUSTER_COST>; }; cpu3: cpu@3 { @@ -122,6 +125,7 @@ next-level-cache = <&CLUSTER0_L2>; operating-points-v2 = <&cpu_opp_table>; cpu-idle-states = <&CPU_SLEEP &CLUSTER_SLEEP>; + sched-energy-costs = <&CPU_COST &CLUSTER_COST>; }; cpu4: cpu@100 { @@ -132,6 +136,7 @@ next-level-cache = <&CLUSTER1_L2>; operating-points-v2 = <&cpu_opp_table>; cpu-idle-states = <&CPU_SLEEP &CLUSTER_SLEEP>; + sched-energy-costs = <&CPU_COST &CLUSTER_COST>; }; cpu5: cpu@101 { @@ -142,6 +147,7 @@ next-level-cache = <&CLUSTER1_L2>; operating-points-v2 = <&cpu_opp_table>; cpu-idle-states = <&CPU_SLEEP &CLUSTER_SLEEP>; + sched-energy-costs = <&CPU_COST &CLUSTER_COST>; }; cpu6: cpu@102 { @@ -152,6 +158,7 @@ next-level-cache = <&CLUSTER1_L2>; operating-points-v2 = <&cpu_opp_table>; cpu-idle-states = <&CPU_SLEEP &CLUSTER_SLEEP>; + sched-energy-costs = <&CPU_COST &CLUSTER_COST>; }; cpu7: cpu@103 { @@ -162,6 +169,7 @@ next-level-cache = <&CLUSTER1_L2>; operating-points-v2 = <&cpu_opp_table>; cpu-idle-states = <&CPU_SLEEP &CLUSTER_SLEEP>; + sched-energy-costs = <&CPU_COST &CLUSTER_COST>; }; CLUSTER0_L2: l2-cache0 { @@ -171,6 +179,8 @@ CLUSTER1_L2: l2-cache1 { compatible = "cache"; }; + + /include/ "hi6220-sched-energy.dtsi" }; cpu_opp_table: cpu_opp_table { @@ -332,6 +342,19 @@ status = "disabled"; }; + dma0: dma@f7370000 { + compatible = "hisilicon,k3-dma-1.0"; + reg = <0x0 0xf7370000 0x0 0x1000>; + #dma-cells = <1>; + dma-channels = <15>; + dma-requests = <32>; + interrupts = <0 84 4>; + clocks = <&sys_ctrl HI6220_EDMAC_ACLK>; + dma-no-cci; + dma-type = "hi6220_dma"; + status = "ok"; + }; + dual_timer0: timer@f8008000 { compatible = "arm,sp804", "arm,primecell"; reg = <0x0 0xf8008000 0x0 0x1000>; @@ -679,7 +702,9 @@ reg = <0x0 0xf7106000 0x0 0x1000>; interrupts = <0 50 4>; bus-id = <0>; - enable-dma = <0>; + enable-dma = <1>; + dmas = <&dma0 12 &dma0 13>; + dma-names = "rx", "tx"; clocks = <&sys_ctrl HI6220_SPI_CLK>; clock-names = "apb_pclk"; pinctrl-names = "default"; @@ -732,6 +757,16 @@ regulator-always-on; }; + usb_vbus: usb-vbus { + compatible = "linux,extcon-usb-gpio"; + id-gpio = <&gpio2 6 1>; + }; + + usb_id: usb-id { + compatible = "linux,extcon-usb-gpio"; + id-gpio = <&gpio2 5 1>; + }; + usb_phy: usbphy { compatible = "hisilicon,hi6220-usb-phy"; #phy-cells = <0>; @@ -745,6 +780,7 @@ phys = <&usb_phy>; phy-names = "usb2-phy"; clocks = <&sys_ctrl HI6220_USBOTG_HCLK>; + extcon = <&usb_vbus>, <&usb_id>; clock-names = "otg"; dr_mode = "otg"; g-use-dma; @@ -829,6 +865,19 @@ #thermal-sensor-cells = <1>; }; + i2s0: hi6210_i2s { + compatible = "hisilicon,hi6210-i2s"; + reg = <0x0 0xf7118000 0x0 0x8000>; /* i2s unit */ + interrupts = ; /* 155 "DigACodec_intr"-32 */ + clocks = <&sys_ctrl HI6220_DACODEC_PCLK>, + <&sys_ctrl HI6220_BBPPLL0_DIV>; + clock-names = "dacodec", "i2s-base"; + dmas = <&dma0 15 &dma0 14>; + dma-names = "rx", "tx"; + hisilicon,sysctrl-syscon = <&sys_ctrl>; + #sound-dai-cells = <0>; + }; + thermal-zones { cls0: cls0 { @@ -911,5 +960,43 @@ }; }; }; + + mali:mali@f4080000 { + compatible = "arm,mali-450", "arm,mali-utgard"; + reg = <0x0 0x3f100000 0x0 0x00708000>; + clocks = <&media_ctrl HI6220_G3D_CLK>, + <&media_ctrl HI6220_G3D_PCLK>; + clock-names = "clk_g3d", "pclk_g3d"; + mali_def_freq = <500>; + pclk_freq = <144>; + dfs_steps = <2>; + dfs_lockprf = <1>; + dfs_limit_max_prf = <1>; + dfs_profile_num = <2>; + dfs_profiles = <250 3 0>, <500 1 0>; + mali_type = <2>; + + interrupt-parent = <&gic>; + interrupts = <1 126 4>, /*gp*/ + <1 126 4>, /*gp mmu*/ + <1 126 4>, /*pp bc*/ + <1 126 4>, /*pmu*/ + <1 126 4>, /*pp0*/ + <1 126 4>, + <1 126 4>, /*pp1*/ + <1 126 4>, + <1 126 4>, /*pp2*/ + <1 126 4>, + <1 126 4>, /*pp4*/ + <1 126 4>, + <1 126 4>, /*pp5*/ + <1 126 4>, + <1 126 4>, /*pp6*/ + <1 126 4>; + interrupt-names = "IRQGP", "IRQGPMMU", "IRQPP", "IRQPMU", + "IRQPP0", "IRQPPMMU0", "IRQPP1", "IRQPPMMU1", + "IRQPP2", "IRQPPMMU2","IRQPP4", "IRQPPMMU4", + "IRQPP5", "IRQPPMMU5", "IRQPP6", "IRQPPMMU6"; + }; }; }; diff --git a/arch/arm64/boot/dts/hisilicon/hip06.dtsi b/arch/arm64/boot/dts/hisilicon/hip06.dtsi index b548763366dd65..af450413b9ddbe 100644 --- a/arch/arm64/boot/dts/hisilicon/hip06.dtsi +++ b/arch/arm64/boot/dts/hisilicon/hip06.dtsi @@ -322,7 +322,7 @@ compatible = "generic-ohci"; reg = <0x0 0xa7030000 0x0 0x10000>; interrupt-parent = <&mbigen_usb>; - interrupts = <64 4>; + interrupts = <640 4>; dma-coherent; status = "disabled"; }; @@ -331,7 +331,7 @@ compatible = "generic-ehci"; reg = <0x0 0xa7020000 0x0 0x10000>; interrupt-parent = <&mbigen_usb>; - interrupts = <65 4>; + interrupts = <641 4>; dma-coherent; status = "disabled"; }; diff --git a/arch/arm64/boot/dts/mediatek/mt8173.dtsi b/arch/arm64/boot/dts/mediatek/mt8173.dtsi index 1c71e256601dc4..6c03c1702bb32f 100644 --- a/arch/arm64/boot/dts/mediatek/mt8173.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt8173.dtsi @@ -450,6 +450,9 @@ auxadc: auxadc@11001000 { compatible = "mediatek,mt8173-auxadc"; reg = <0 0x11001000 0 0x1000>; + clocks = <&pericfg CLK_PERI_AUXADC>; + clock-names = "main"; + #io-channel-cells = <1>; }; uart0: serial@11002000 { diff --git a/arch/arm64/boot/dts/nvidia/tegra210-p2180.dtsi b/arch/arm64/boot/dts/nvidia/tegra210-p2180.dtsi index 5fda583351d766..906fb836d24188 100644 --- a/arch/arm64/boot/dts/nvidia/tegra210-p2180.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra210-p2180.dtsi @@ -21,6 +21,10 @@ reg = <0x0 0x80000000 0x1 0x0>; }; + gpu@57000000 { + vdd-supply = <&vdd_gpu>; + }; + /* debug port */ serial@70006000 { status = "okay"; @@ -291,4 +295,18 @@ clock-frequency = <32768>; }; }; + + regulators { + vdd_gpu: regulator@100 { + compatible = "pwm-regulator"; + reg = <100>; + pwms = <&pwm 1 4880>; + regulator-name = "VDD_GPU"; + regulator-min-microvolt = <710000>; + regulator-max-microvolt = <1320000>; + enable-gpios = <&pmic 6 GPIO_ACTIVE_HIGH>; + regulator-ramp-delay = <80>; + regulator-enable-ramp-delay = <1000>; + }; + }; }; diff --git a/arch/arm64/configs/hikey_defconfig b/arch/arm64/configs/hikey_defconfig new file mode 100644 index 00000000000000..191a398ac52abd --- /dev/null +++ b/arch/arm64/configs/hikey_defconfig @@ -0,0 +1,530 @@ +CONFIG_POSIX_MQUEUE=y +CONFIG_AUDIT=y +CONFIG_NO_HZ=y +CONFIG_HIGH_RES_TIMERS=y +CONFIG_SCHED_WALT=y +CONFIG_BSD_PROCESS_ACCT=y +CONFIG_BSD_PROCESS_ACCT_V3=y +CONFIG_TASKSTATS=y +CONFIG_TASK_DELAY_ACCT=y +CONFIG_TASK_XACCT=y +CONFIG_TASK_IO_ACCOUNTING=y +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y +CONFIG_CGROUP_DEBUG=y +CONFIG_CGROUP_FREEZER=y +CONFIG_CGROUP_DEVICE=y +CONFIG_CPUSETS=y +CONFIG_CGROUP_CPUACCT=y +CONFIG_CGROUP_SCHEDTUNE=y +CONFIG_MEMCG=y +CONFIG_MEMCG_SWAP=y +CONFIG_BLK_CGROUP=y +CONFIG_RT_GROUP_SCHED=y +CONFIG_CGROUP_HUGETLB=y +CONFIG_NAMESPACES=y +CONFIG_USER_NS=y +CONFIG_SCHED_AUTOGROUP=y +CONFIG_SCHED_TUNE=y +CONFIG_DEFAULT_USE_ENERGY_AWARE=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_KALLSYMS_ALL=y +CONFIG_EMBEDDED=y +# CONFIG_COMPAT_BRK is not set +CONFIG_SLAB=y +CONFIG_PROFILING=y +CONFIG_JUMP_LABEL=y +# CONFIG_BLK_DEV_BSG is not set +# CONFIG_IOSCHED_DEADLINE is not set +CONFIG_CFQ_GROUP_IOSCHED=y +CONFIG_ARCH_HISI=y +CONFIG_ARCH_VEXPRESS=y +CONFIG_ARCH_XGENE=y +CONFIG_PCI=y +CONFIG_PCI_XGENE=y +CONFIG_SCHED_MC=y +CONFIG_PREEMPT=y +CONFIG_KSM=y +CONFIG_TRANSPARENT_HUGEPAGE=y +CONFIG_CMA=y +CONFIG_SECCOMP=y +CONFIG_ARMV8_DEPRECATED=y +CONFIG_SWP_EMULATION=y +CONFIG_CP15_BARRIER_EMULATION=y +CONFIG_SETEND_EMULATION=y +CONFIG_RANDOMIZE_BASE=y +CONFIG_CMDLINE="console=ttyAMA0" +CONFIG_BUILD_ARM64_APPENDED_DTB_IMAGE=y +CONFIG_BUILD_ARM64_APPENDED_DTB_IMAGE_NAMES="hisilicon/hi6220-hikey" +# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set +CONFIG_BINFMT_MISC=y +CONFIG_COMPAT=y +CONFIG_PM_AUTOSLEEP=y +CONFIG_PM_WAKELOCKS=y +CONFIG_WQ_POWER_EFFICIENT_DEFAULT=y +CONFIG_CPU_IDLE=y +CONFIG_CPU_IDLE_GOV_LADDER=y +CONFIG_ARM_CPUIDLE=y +CONFIG_CPU_FREQ=y +CONFIG_CPU_FREQ_DEFAULT_GOV_SCHED=y +CONFIG_CPU_FREQ_GOV_PERFORMANCE=y +CONFIG_CPU_FREQ_GOV_USERSPACE=y +CONFIG_CPU_FREQ_GOV_ONDEMAND=y +CONFIG_CPU_FREQ_GOV_SCHED=y +CONFIG_CPU_FREQ_GOV_INTERACTIVE=y +CONFIG_CPUFREQ_DT=y +CONFIG_NET=y +CONFIG_PACKET=y +CONFIG_UNIX=y +CONFIG_XFRM_USER=y +CONFIG_NET_KEY=y +CONFIG_NET_KEY_MIGRATE=y +CONFIG_INET=y +CONFIG_IP_MULTICAST=y +CONFIG_IP_ADVANCED_ROUTER=y +CONFIG_IP_MULTIPLE_TABLES=y +CONFIG_IP_PNP=y +CONFIG_IP_PNP_DHCP=y +CONFIG_IP_PNP_BOOTP=y +CONFIG_IP_PNP_RARP=y +CONFIG_INET_ESP=y +CONFIG_INET_DIAG_DESTROY=y +CONFIG_IPV6_ROUTER_PREF=y +CONFIG_IPV6_ROUTE_INFO=y +CONFIG_IPV6_OPTIMISTIC_DAD=y +CONFIG_INET6_AH=y +CONFIG_INET6_ESP=y +CONFIG_INET6_IPCOMP=y +CONFIG_IPV6_MIP6=y +CONFIG_IPV6_MULTIPLE_TABLES=y +CONFIG_NETLABEL=y +CONFIG_NETFILTER=y +CONFIG_NF_CONNTRACK=y +CONFIG_NF_CONNTRACK_SECMARK=y +CONFIG_NF_CONNTRACK_EVENTS=y +CONFIG_NF_CT_PROTO_DCCP=y +CONFIG_NF_CT_PROTO_SCTP=y +CONFIG_NF_CT_PROTO_UDPLITE=y +CONFIG_NF_CONNTRACK_AMANDA=y +CONFIG_NF_CONNTRACK_FTP=y +CONFIG_NF_CONNTRACK_H323=y +CONFIG_NF_CONNTRACK_IRC=y +CONFIG_NF_CONNTRACK_NETBIOS_NS=y +CONFIG_NF_CONNTRACK_PPTP=y +CONFIG_NF_CONNTRACK_SANE=y +CONFIG_NF_CONNTRACK_TFTP=y +CONFIG_NF_CT_NETLINK=y +CONFIG_NETFILTER_XT_TARGET_CLASSIFY=y +CONFIG_NETFILTER_XT_TARGET_CONNMARK=y +CONFIG_NETFILTER_XT_TARGET_CONNSECMARK=y +CONFIG_NETFILTER_XT_TARGET_IDLETIMER=y +CONFIG_NETFILTER_XT_TARGET_MARK=y +CONFIG_NETFILTER_XT_TARGET_NFLOG=y +CONFIG_NETFILTER_XT_TARGET_NFQUEUE=y +CONFIG_NETFILTER_XT_TARGET_TPROXY=y +CONFIG_NETFILTER_XT_TARGET_TRACE=y +CONFIG_NETFILTER_XT_TARGET_SECMARK=y +CONFIG_NETFILTER_XT_TARGET_TCPMSS=y +CONFIG_NETFILTER_XT_MATCH_COMMENT=y +CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=y +CONFIG_NETFILTER_XT_MATCH_CONNMARK=y +CONFIG_NETFILTER_XT_MATCH_CONNTRACK=y +CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=y +CONFIG_NETFILTER_XT_MATCH_HELPER=y +CONFIG_NETFILTER_XT_MATCH_IPRANGE=y +CONFIG_NETFILTER_XT_MATCH_LENGTH=y +CONFIG_NETFILTER_XT_MATCH_LIMIT=y +CONFIG_NETFILTER_XT_MATCH_MAC=y +CONFIG_NETFILTER_XT_MATCH_MARK=y +CONFIG_NETFILTER_XT_MATCH_POLICY=y +CONFIG_NETFILTER_XT_MATCH_PKTTYPE=y +CONFIG_NETFILTER_XT_MATCH_QTAGUID=y +CONFIG_NETFILTER_XT_MATCH_QUOTA=y +CONFIG_NETFILTER_XT_MATCH_QUOTA2=y +CONFIG_NETFILTER_XT_MATCH_SOCKET=y +CONFIG_NETFILTER_XT_MATCH_STATE=y +CONFIG_NETFILTER_XT_MATCH_STATISTIC=y +CONFIG_NETFILTER_XT_MATCH_STRING=y +CONFIG_NETFILTER_XT_MATCH_TIME=y +CONFIG_NETFILTER_XT_MATCH_U32=y +CONFIG_NF_CONNTRACK_IPV4=y +CONFIG_IP_NF_IPTABLES=y +CONFIG_IP_NF_MATCH_AH=y +CONFIG_IP_NF_MATCH_ECN=y +CONFIG_IP_NF_MATCH_TTL=y +CONFIG_IP_NF_FILTER=y +CONFIG_IP_NF_TARGET_REJECT=y +CONFIG_IP_NF_NAT=y +CONFIG_IP_NF_TARGET_MASQUERADE=y +CONFIG_IP_NF_MANGLE=y +CONFIG_IP_NF_RAW=y +CONFIG_IP_NF_SECURITY=y +CONFIG_IP_NF_ARPTABLES=y +CONFIG_IP_NF_ARPFILTER=y +CONFIG_IP_NF_ARP_MANGLE=y +CONFIG_NF_CONNTRACK_IPV6=y +CONFIG_IP6_NF_IPTABLES=y +CONFIG_IP6_NF_MATCH_AH=y +CONFIG_IP6_NF_MATCH_EUI64=y +CONFIG_IP6_NF_MATCH_FRAG=y +CONFIG_IP6_NF_MATCH_OPTS=y +CONFIG_IP6_NF_MATCH_HL=y +CONFIG_IP6_NF_MATCH_IPV6HEADER=y +CONFIG_IP6_NF_MATCH_MH=y +CONFIG_IP6_NF_FILTER=y +CONFIG_IP6_NF_TARGET_REJECT=y +CONFIG_IP6_NF_MANGLE=y +CONFIG_IP6_NF_RAW=y +CONFIG_IP6_NF_NAT=y +CONFIG_IP6_NF_TARGET_MASQUERADE=y +CONFIG_BRIDGE=y +CONFIG_BRIDGE_VLAN_FILTERING=y +CONFIG_VLAN_8021Q=y +CONFIG_NET_SCHED=y +CONFIG_NET_SCH_HTB=y +CONFIG_NET_CLS_U32=y +CONFIG_NET_EMATCH=y +CONFIG_NET_EMATCH_U32=y +CONFIG_NET_CLS_ACT=y +CONFIG_BT=y +CONFIG_BT_RFCOMM=y +CONFIG_BT_RFCOMM_TTY=y +CONFIG_BT_HIDP=y +CONFIG_BT_LEDS=y +CONFIG_BT_HCIUART=y +CONFIG_BT_HCIUART_H4=y +CONFIG_BT_WILINK=y +CONFIG_CFG80211=y +CONFIG_NL80211_TESTMODE=y +CONFIG_CFG80211_WEXT=y +CONFIG_MAC80211=y +CONFIG_MAC80211_MESH=y +CONFIG_MAC80211_LEDS=y +CONFIG_RFKILL=y +CONFIG_RFKILL_REGULATOR=y +CONFIG_RFKILL_GPIO=y +CONFIG_NET_9P=y +CONFIG_NET_9P_VIRTIO=y +CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" +CONFIG_FW_LOADER_USER_HELPER_FALLBACK=y +CONFIG_DMA_CMA=y +CONFIG_CMA_SIZE_MBYTES=64 +CONFIG_CONNECTOR=y +CONFIG_OF_OVERLAY_MGR=y +CONFIG_BLK_DEV_LOOP=y +CONFIG_BLK_DEV_RAM=y +CONFIG_BLK_DEV_RAM_SIZE=16384 +CONFIG_VIRTIO_BLK=y +CONFIG_UID_CPUTIME=y +CONFIG_TI_ST=y +CONFIG_ST_HCI=y +# CONFIG_SCSI_PROC_FS is not set +CONFIG_BLK_DEV_SD=y +# CONFIG_SCSI_LOWLEVEL is not set +CONFIG_ATA=y +CONFIG_SATA_AHCI=y +CONFIG_SATA_AHCI_PLATFORM=y +CONFIG_AHCI_XGENE=y +CONFIG_PATA_PLATFORM=y +CONFIG_PATA_OF_PLATFORM=y +CONFIG_MD=y +CONFIG_BLK_DEV_DM=y +CONFIG_DM_CRYPT=y +CONFIG_DM_VERITY=y +CONFIG_NETDEVICES=y +CONFIG_MACVLAN=y +CONFIG_TUN=y +CONFIG_VETH=y +CONFIG_VIRTIO_NET=y +CONFIG_NET_XGENE=y +CONFIG_SMC91X=y +CONFIG_SMSC911X=y +CONFIG_PPP=y +CONFIG_PPP_BSDCOMP=y +CONFIG_PPP_DEFLATE=y +CONFIG_PPP_MPPE=y +CONFIG_USB_PEGASUS=y +CONFIG_USB_RTL8150=y +CONFIG_USB_RTL8152=y +CONFIG_USB_USBNET=y +CONFIG_USB_NET_DM9601=y +CONFIG_USB_NET_SR9800=y +CONFIG_USB_NET_SMSC75XX=y +CONFIG_USB_NET_SMSC95XX=y +CONFIG_USB_NET_PLUSB=y +CONFIG_USB_NET_MCS7830=y +CONFIG_WL18XX=y +CONFIG_WLCORE_SDIO=y +CONFIG_INPUT_EVDEV=y +CONFIG_KEYBOARD_GPIO=y +CONFIG_INPUT_MISC=y +CONFIG_INPUT_UINPUT=y +CONFIG_INPUT_HISI_POWERKEY=y +# CONFIG_SERIO_SERPORT is not set +CONFIG_SERIO_AMBAKMI=y +CONFIG_VT_HW_CONSOLE_BINDING=y +CONFIG_LEGACY_PTY_COUNT=16 +# CONFIG_DEVMEM is not set +# CONFIG_DEVKMEM is not set +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_8250_CONSOLE=y +CONFIG_SERIAL_OF_PLATFORM=y +CONFIG_SERIAL_AMBA_PL011=y +CONFIG_SERIAL_AMBA_PL011_CONSOLE=y +CONFIG_VIRTIO_CONSOLE=y +# CONFIG_HW_RANDOM is not set +CONFIG_I2C_CHARDEV=y +CONFIG_I2C_DESIGNWARE_PLATFORM=y +CONFIG_SPI=y +CONFIG_SPI_PL022=y +CONFIG_SPI_SPIDEV=y +CONFIG_PINCTRL_SINGLE=y +CONFIG_GPIO_SYSFS=y +CONFIG_GPIO_PL061=y +CONFIG_GPIO_XGENE=y +CONFIG_POWER_RESET_HISI=y +CONFIG_POWER_RESET_SYSCON=y +CONFIG_SYSCON_REBOOT_MODE=y +CONFIG_THERMAL=y +CONFIG_THERMAL_DEFAULT_GOV_POWER_ALLOCATOR=y +CONFIG_THERMAL_GOV_STEP_WISE=y +CONFIG_CPU_THERMAL=y +CONFIG_HISI_THERMAL=y +CONFIG_MFD_HI655X_PMIC=y +CONFIG_REGULATOR=y +CONFIG_REGULATOR_FIXED_VOLTAGE=y +CONFIG_REGULATOR_HI655X=y +CONFIG_MEDIA_SUPPORT=y +CONFIG_MEDIA_ANALOG_TV_SUPPORT=y +# CONFIG_MEDIA_SUBDRV_AUTOSELECT is not set +CONFIG_MALI400=y +CONFIG_MALI450=y +CONFIG_MALI400_DEBUG=y +# CONFIG_MALI400_PROFILING is not set +# CONFIG_MALI_DVFS is not set +CONFIG_MALI_SHARED_INTERRUPTS=y +CONFIG_MALI_DT=y +CONFIG_MALI_PLAT_SPECIFIC_DT=y +CONFIG_DRM=y +CONFIG_DRM_CMA_FBDEV_BUFFER_NUM=2 +CONFIG_DRM_I2C_ADV7511=y +CONFIG_DRM_I2C_ADV7511_AUDIO=y +CONFIG_DRM_HISI_KIRIN=y +CONFIG_FB_ARMCLCD=y +CONFIG_FB_SIMPLE=y +CONFIG_SOUND=y +CONFIG_SND=y +CONFIG_SND_VERBOSE_PRINTK=y +CONFIG_SND_DEBUG=y +CONFIG_SND_DEBUG_VERBOSE=y +CONFIG_SND_USB_AUDIO=y +CONFIG_SND_SOC=y +CONFIG_SND_I2S_HI6210_I2S=y +CONFIG_SND_SIMPLE_CARD=y +CONFIG_HIDRAW=y +CONFIG_UHID=y +CONFIG_HID_A4TECH=y +CONFIG_HID_ACRUX=y +CONFIG_HID_ACRUX_FF=y +CONFIG_HID_APPLE=y +CONFIG_HID_BELKIN=y +CONFIG_HID_CHERRY=y +CONFIG_HID_CHICONY=y +CONFIG_HID_PRODIKEYS=y +CONFIG_HID_CYPRESS=y +CONFIG_HID_DRAGONRISE=y +CONFIG_DRAGONRISE_FF=y +CONFIG_HID_EMS_FF=y +CONFIG_HID_ELECOM=y +CONFIG_HID_EZKEY=y +CONFIG_HID_KEYTOUCH=y +CONFIG_HID_KYE=y +CONFIG_HID_WALTOP=y +CONFIG_HID_GYRATION=y +CONFIG_HID_TWINHAN=y +CONFIG_HID_KENSINGTON=y +CONFIG_HID_LCPOWER=y +CONFIG_HID_LOGITECH=y +CONFIG_HID_LOGITECH_DJ=y +CONFIG_LOGITECH_FF=y +CONFIG_LOGIRUMBLEPAD2_FF=y +CONFIG_LOGIG940_FF=y +CONFIG_HID_MAGICMOUSE=y +CONFIG_HID_MICROSOFT=y +CONFIG_HID_MONTEREY=y +CONFIG_HID_MULTITOUCH=y +CONFIG_HID_ORTEK=y +CONFIG_HID_PANTHERLORD=y +CONFIG_PANTHERLORD_FF=y +CONFIG_HID_PETALYNX=y +CONFIG_HID_PICOLCD=y +CONFIG_HID_PRIMAX=y +CONFIG_HID_SAITEK=y +CONFIG_HID_SAMSUNG=y +CONFIG_HID_SPEEDLINK=y +CONFIG_HID_SUNPLUS=y +CONFIG_HID_GREENASIA=y +CONFIG_GREENASIA_FF=y +CONFIG_HID_SMARTJOYPLUS=y +CONFIG_SMARTJOYPLUS_FF=y +CONFIG_HID_TIVO=y +CONFIG_HID_TOPSEED=y +CONFIG_HID_THRUSTMASTER=y +CONFIG_HID_WACOM=y +CONFIG_HID_WIIMOTE=y +CONFIG_HID_ZEROPLUS=y +CONFIG_HID_ZYDACRON=y +CONFIG_USB=y +CONFIG_USB_EHCI_HCD=y +CONFIG_USB_EHCI_HCD_PLATFORM=y +CONFIG_USB_OHCI_HCD=y +CONFIG_USB_OHCI_HCD_PLATFORM=y +CONFIG_USB_ACM=y +CONFIG_USB_STORAGE=y +CONFIG_USB_DWC2=y +CONFIG_USB_SERIAL=y +CONFIG_USB_SERIAL_GENERIC=y +CONFIG_USB_SERIAL_SIMPLE=y +CONFIG_USB_SERIAL_FTDI_SIO=y +CONFIG_USB_SERIAL_PL2303=y +CONFIG_USB_SERIAL_QUALCOMM=y +CONFIG_USB_SERIAL_OPTION=y +CONFIG_USB_OTG_WAKELOCK=y +CONFIG_USB_ULPI=y +CONFIG_USB_GADGET=y +CONFIG_USB_CONFIGFS=y +CONFIG_USB_CONFIGFS_SERIAL=y +CONFIG_USB_CONFIGFS_ACM=y +CONFIG_USB_CONFIGFS_RNDIS=y +CONFIG_USB_CONFIGFS_F_FS=y +CONFIG_USB_CONFIGFS_F_MTP=y +CONFIG_USB_CONFIGFS_F_PTP=y +CONFIG_USB_CONFIGFS_F_ACC=y +CONFIG_USB_CONFIGFS_F_AUDIO_SRC=y +CONFIG_USB_CONFIGFS_UEVENT=y +CONFIG_USB_CONFIGFS_F_MIDI=y +CONFIG_USB_GADGETFS=y +CONFIG_MMC=y +CONFIG_MMC_PARANOID_SD_INIT=y +CONFIG_MMC_BLOCK_MINORS=64 +CONFIG_MMC_ARMMMCI=y +CONFIG_MMC_SDHCI=y +CONFIG_MMC_SDHCI_PLTFM=y +CONFIG_MMC_SPI=y +CONFIG_MMC_DW=y +CONFIG_MMC_DW_EXYNOS=y +CONFIG_MMC_DW_K3=y +CONFIG_LEDS_GPIO=y +CONFIG_LEDS_REGULATOR=y +CONFIG_LEDS_SYSCON=y +CONFIG_LEDS_TRIGGER_HEARTBEAT=y +CONFIG_LEDS_TRIGGER_CPU=y +CONFIG_RTC_CLASS=y +CONFIG_RTC_DRV_EFI=y +CONFIG_RTC_DRV_PL031=y +CONFIG_RTC_DRV_XGENE=y +CONFIG_DMADEVICES=y +CONFIG_K3_DMA=y +CONFIG_SYNC_FILE=y +CONFIG_VIRTIO_BALLOON=y +CONFIG_VIRTIO_MMIO=y +CONFIG_STAGING=y +CONFIG_ASHMEM=y +CONFIG_ANDROID_TIMED_GPIO=y +CONFIG_ANDROID_LOW_MEMORY_KILLER=y +CONFIG_SYNC=y +CONFIG_ION=y +CONFIG_ION_DUMMY=y +CONFIG_ION_OF=y +CONFIG_FIQ_DEBUGGER_NO_SLEEP=y +CONFIG_FIQ_DEBUGGER_CONSOLE=y +CONFIG_FIQ_DEBUGGER_CONSOLE_DEFAULT_ENABLE=y +CONFIG_FIQ_DEBUGGER_UART_OVERLAY=y +CONFIG_HISI_FIQ_DEBUGGER=y +CONFIG_STUB_CLK_HI6220=y +CONFIG_MAILBOX=y +CONFIG_HI6220_MBOX=y +# CONFIG_IOMMU_SUPPORT is not set +CONFIG_EXTCON=y +CONFIG_EXTCON_GPIO=y +CONFIG_EXTCON_USB_GPIO=y +CONFIG_PHY_HI6220_USB=y +CONFIG_PHY_XGENE=y +CONFIG_ANDROID=y +CONFIG_ANDROID_BINDER_IPC=y +CONFIG_ANDROID_BINDER_DEVICES="binder,hwbinder" +CONFIG_TEE=y +CONFIG_OPTEE=y +CONFIG_EFI_VARS=y +CONFIG_EFI_VARS_PSTORE_DEFAULT_DISABLE=y +CONFIG_EXT4_FS=y +CONFIG_EXT4_FS_SECURITY=y +CONFIG_EXT4_ENCRYPTION=y +CONFIG_BTRFS_FS=y +CONFIG_FANOTIFY=y +CONFIG_FANOTIFY_ACCESS_PERMISSIONS=y +CONFIG_QUOTA=y +CONFIG_QFMT_V2=y +CONFIG_AUTOFS4_FS=y +CONFIG_FUSE_FS=y +CONFIG_CUSE=y +CONFIG_OVERLAY_FS=y +CONFIG_MSDOS_FS=y +CONFIG_VFAT_FS=y +CONFIG_TMPFS=y +CONFIG_TMPFS_POSIX_ACL=y +CONFIG_HUGETLBFS=y +CONFIG_SQUASHFS=y +CONFIG_SQUASHFS_FILE_DIRECT=y +CONFIG_SQUASHFS_DECOMP_MULTI_PERCPU=y +CONFIG_SQUASHFS_XATTR=y +CONFIG_SQUASHFS_LZ4=y +CONFIG_SQUASHFS_XZ=y +CONFIG_PSTORE=y +CONFIG_PSTORE_CONSOLE=y +CONFIG_PSTORE_PMSG=y +CONFIG_PSTORE_FTRACE=y +CONFIG_PSTORE_RAM=y +CONFIG_NFS_FS=y +# CONFIG_NFS_V2 is not set +CONFIG_NFS_V3_ACL=y +CONFIG_NFS_V4=y +CONFIG_ROOT_NFS=y +CONFIG_9P_FS=y +CONFIG_NLS_CODEPAGE_437=y +CONFIG_NLS_ISO8859_1=y +CONFIG_VIRTUALIZATION=y +CONFIG_KVM=y +CONFIG_PRINTK_TIME=y +CONFIG_DEBUG_INFO=y +CONFIG_MAGIC_SYSRQ=y +CONFIG_LOCKUP_DETECTOR=y +CONFIG_PANIC_ON_OOPS=y +CONFIG_PANIC_TIMEOUT=5 +# CONFIG_SCHED_DEBUG is not set +CONFIG_SCHEDSTATS=y +CONFIG_TIMER_STATS=y +CONFIG_FUNCTION_TRACER=y +CONFIG_IRQSOFF_TRACER=y +CONFIG_PREEMPT_TRACER=y +CONFIG_SCHED_TRACER=y +CONFIG_FTRACE_SYSCALLS=y +CONFIG_BLK_DEV_IO_TRACE=y +CONFIG_FUNCTION_PROFILER=y +CONFIG_SECURITY=y +CONFIG_SECURITY_NETWORK=y +CONFIG_SECURITY_NETWORK_XFRM=y +CONFIG_LSM_MMAP_MIN_ADDR=4096 +CONFIG_SECURITY_SELINUX=y +CONFIG_SECURITY_SELINUX_BOOTPARAM=y +CONFIG_SECURITY_SELINUX_DISABLE=y +CONFIG_CRYPTO_MICHAEL_MIC=y +CONFIG_CRYPTO_TWOFISH=y +CONFIG_ARM64_CRYPTO=y +CONFIG_CRYPTO_SHA1_ARM64_CE=y +CONFIG_CRYPTO_SHA2_ARM64_CE=y +CONFIG_CRYPTO_GHASH_ARM64_CE=y +CONFIG_CRYPTO_AES_ARM64_CE_CCM=y +CONFIG_CRYPTO_AES_ARM64_CE_BLK=y +CONFIG_CRYPTO_AES_ARM64_NEON_BLK=y +CONFIG_CRC_T10DIF=y diff --git a/arch/arm64/configs/ranchu64_defconfig b/arch/arm64/configs/ranchu64_defconfig new file mode 100644 index 00000000000000..fc55008d8c4c97 --- /dev/null +++ b/arch/arm64/configs/ranchu64_defconfig @@ -0,0 +1,312 @@ +# CONFIG_LOCALVERSION_AUTO is not set +# CONFIG_SWAP is not set +CONFIG_POSIX_MQUEUE=y +CONFIG_AUDIT=y +CONFIG_NO_HZ=y +CONFIG_HIGH_RES_TIMERS=y +CONFIG_BSD_PROCESS_ACCT=y +CONFIG_BSD_PROCESS_ACCT_V3=y +CONFIG_TASKSTATS=y +CONFIG_TASK_DELAY_ACCT=y +CONFIG_TASK_XACCT=y +CONFIG_TASK_IO_ACCOUNTING=y +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y +CONFIG_LOG_BUF_SHIFT=14 +CONFIG_CGROUP_DEBUG=y +CONFIG_CGROUP_FREEZER=y +CONFIG_CGROUP_CPUACCT=y +CONFIG_RT_GROUP_SCHED=y +CONFIG_SCHED_AUTOGROUP=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_KALLSYMS_ALL=y +CONFIG_EMBEDDED=y +# CONFIG_COMPAT_BRK is not set +CONFIG_PROFILING=y +CONFIG_ARCH_MMAP_RND_BITS=24 +CONFIG_ARCH_MMAP_RND_COMPAT_BITS=16 +# CONFIG_BLK_DEV_BSG is not set +# CONFIG_IOSCHED_DEADLINE is not set +CONFIG_ARCH_VEXPRESS=y +CONFIG_NR_CPUS=4 +CONFIG_PREEMPT=y +CONFIG_KSM=y +CONFIG_SECCOMP=y +CONFIG_ARMV8_DEPRECATED=y +CONFIG_SWP_EMULATION=y +CONFIG_CP15_BARRIER_EMULATION=y +CONFIG_SETEND_EMULATION=y +CONFIG_CMDLINE="console=ttyAMA0" +# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set +CONFIG_COMPAT=y +CONFIG_PM_AUTOSLEEP=y +CONFIG_PM_WAKELOCKS=y +CONFIG_PM_WAKELOCKS_LIMIT=0 +# CONFIG_PM_WAKELOCKS_GC is not set +CONFIG_PM_DEBUG=y +CONFIG_NET=y +CONFIG_PACKET=y +CONFIG_UNIX=y +CONFIG_XFRM_USER=y +CONFIG_NET_KEY=y +CONFIG_INET=y +CONFIG_INET_DIAG_DESTROY=y +CONFIG_IP_MULTICAST=y +CONFIG_IP_ADVANCED_ROUTER=y +CONFIG_IP_MULTIPLE_TABLES=y +CONFIG_IP_PNP=y +CONFIG_IP_PNP_DHCP=y +CONFIG_IP_PNP_BOOTP=y +CONFIG_INET_ESP=y +# CONFIG_INET_LRO is not set +CONFIG_IPV6_ROUTER_PREF=y +CONFIG_IPV6_ROUTE_INFO=y +CONFIG_IPV6_OPTIMISTIC_DAD=y +CONFIG_INET6_AH=y +CONFIG_INET6_ESP=y +CONFIG_INET6_IPCOMP=y +CONFIG_IPV6_MIP6=y +CONFIG_IPV6_MULTIPLE_TABLES=y +CONFIG_NETFILTER=y +CONFIG_NF_CONNTRACK=y +CONFIG_NF_CONNTRACK_SECMARK=y +CONFIG_NF_CONNTRACK_EVENTS=y +CONFIG_NF_CT_PROTO_DCCP=y +CONFIG_NF_CT_PROTO_SCTP=y +CONFIG_NF_CT_PROTO_UDPLITE=y +CONFIG_NF_CONNTRACK_AMANDA=y +CONFIG_NF_CONNTRACK_FTP=y +CONFIG_NF_CONNTRACK_H323=y +CONFIG_NF_CONNTRACK_IRC=y +CONFIG_NF_CONNTRACK_NETBIOS_NS=y +CONFIG_NF_CONNTRACK_PPTP=y +CONFIG_NF_CONNTRACK_SANE=y +CONFIG_NF_CONNTRACK_TFTP=y +CONFIG_NF_CT_NETLINK=y +CONFIG_NETFILTER_XT_TARGET_CLASSIFY=y +CONFIG_NETFILTER_XT_TARGET_CONNMARK=y +CONFIG_NETFILTER_XT_TARGET_CONNSECMARK=y +CONFIG_NETFILTER_XT_TARGET_IDLETIMER=y +CONFIG_NETFILTER_XT_TARGET_MARK=y +CONFIG_NETFILTER_XT_TARGET_NFLOG=y +CONFIG_NETFILTER_XT_TARGET_NFQUEUE=y +CONFIG_NETFILTER_XT_TARGET_TPROXY=y +CONFIG_NETFILTER_XT_TARGET_TRACE=y +CONFIG_NETFILTER_XT_TARGET_SECMARK=y +CONFIG_NETFILTER_XT_TARGET_TCPMSS=y +CONFIG_NETFILTER_XT_MATCH_COMMENT=y +CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=y +CONFIG_NETFILTER_XT_MATCH_CONNMARK=y +CONFIG_NETFILTER_XT_MATCH_CONNTRACK=y +CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=y +CONFIG_NETFILTER_XT_MATCH_HELPER=y +CONFIG_NETFILTER_XT_MATCH_IPRANGE=y +CONFIG_NETFILTER_XT_MATCH_LENGTH=y +CONFIG_NETFILTER_XT_MATCH_LIMIT=y +CONFIG_NETFILTER_XT_MATCH_MAC=y +CONFIG_NETFILTER_XT_MATCH_MARK=y +CONFIG_NETFILTER_XT_MATCH_POLICY=y +CONFIG_NETFILTER_XT_MATCH_PKTTYPE=y +CONFIG_NETFILTER_XT_MATCH_QTAGUID=y +CONFIG_NETFILTER_XT_MATCH_QUOTA=y +CONFIG_NETFILTER_XT_MATCH_QUOTA2=y +CONFIG_NETFILTER_XT_MATCH_SOCKET=y +CONFIG_NETFILTER_XT_MATCH_STATE=y +CONFIG_NETFILTER_XT_MATCH_STATISTIC=y +CONFIG_NETFILTER_XT_MATCH_STRING=y +CONFIG_NETFILTER_XT_MATCH_TIME=y +CONFIG_NETFILTER_XT_MATCH_U32=y +CONFIG_NF_CONNTRACK_IPV4=y +CONFIG_IP_NF_IPTABLES=y +CONFIG_IP_NF_MATCH_AH=y +CONFIG_IP_NF_MATCH_ECN=y +CONFIG_IP_NF_MATCH_RPFILTER=y +CONFIG_IP_NF_MATCH_TTL=y +CONFIG_IP_NF_FILTER=y +CONFIG_IP_NF_TARGET_REJECT=y +CONFIG_IP_NF_MANGLE=y +CONFIG_IP_NF_TARGET_ECN=y +CONFIG_IP_NF_TARGET_TTL=y +CONFIG_IP_NF_RAW=y +CONFIG_IP_NF_SECURITY=y +CONFIG_IP_NF_ARPTABLES=y +CONFIG_IP_NF_ARPFILTER=y +CONFIG_IP_NF_ARP_MANGLE=y +CONFIG_NF_CONNTRACK_IPV6=y +CONFIG_IP6_NF_IPTABLES=y +CONFIG_IP6_NF_MATCH_AH=y +CONFIG_IP6_NF_MATCH_EUI64=y +CONFIG_IP6_NF_MATCH_FRAG=y +CONFIG_IP6_NF_MATCH_OPTS=y +CONFIG_IP6_NF_MATCH_HL=y +CONFIG_IP6_NF_MATCH_IPV6HEADER=y +CONFIG_IP6_NF_MATCH_MH=y +CONFIG_IP6_NF_MATCH_RT=y +CONFIG_IP6_NF_TARGET_HL=y +CONFIG_IP6_NF_FILTER=y +CONFIG_IP6_NF_TARGET_REJECT=y +CONFIG_IP6_NF_MANGLE=y +CONFIG_IP6_NF_RAW=y +CONFIG_BRIDGE=y +CONFIG_NET_SCHED=y +CONFIG_NET_SCH_HTB=y +CONFIG_NET_CLS_U32=y +CONFIG_NET_EMATCH=y +CONFIG_NET_EMATCH_U32=y +CONFIG_NET_CLS_ACT=y +# CONFIG_WIRELESS is not set +CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" +CONFIG_BLK_DEV_LOOP=y +CONFIG_BLK_DEV_RAM=y +CONFIG_BLK_DEV_RAM_SIZE=8192 +CONFIG_VIRTIO_BLK=y +CONFIG_SCSI=y +# CONFIG_SCSI_PROC_FS is not set +CONFIG_BLK_DEV_SD=y +# CONFIG_SCSI_LOWLEVEL is not set +CONFIG_MD=y +CONFIG_BLK_DEV_DM=y +CONFIG_DM_CRYPT=y +CONFIG_DM_UEVENT=y +CONFIG_DM_VERITY=y +CONFIG_DM_VERITY_FEC=y +CONFIG_NETDEVICES=y +CONFIG_TUN=y +CONFIG_VIRTIO_NET=y +CONFIG_SMC91X=y +CONFIG_PPP=y +CONFIG_PPP_BSDCOMP=y +CONFIG_PPP_DEFLATE=y +CONFIG_PPP_MPPE=y +CONFIG_PPPOLAC=y +CONFIG_PPPOPNS=y +# CONFIG_WLAN is not set +CONFIG_INPUT_EVDEV=y +CONFIG_INPUT_KEYRESET=y +CONFIG_KEYBOARD_GOLDFISH_EVENTS=y +# CONFIG_INPUT_MOUSE is not set +CONFIG_INPUT_JOYSTICK=y +CONFIG_INPUT_TABLET=y +CONFIG_INPUT_MISC=y +CONFIG_INPUT_KEYCHORD=y +CONFIG_INPUT_UINPUT=y +CONFIG_INPUT_GPIO=y +# CONFIG_SERIO_SERPORT is not set +# CONFIG_VT is not set +# CONFIG_LEGACY_PTYS is not set +# CONFIG_DEVMEM is not set +# CONFIG_DEVKMEM is not set +CONFIG_SERIAL_AMBA_PL011=y +CONFIG_SERIAL_AMBA_PL011_CONSOLE=y +CONFIG_VIRTIO_CONSOLE=y +# CONFIG_HW_RANDOM is not set +CONFIG_BATTERY_GOLDFISH=y +# CONFIG_HWMON is not set +CONFIG_MEDIA_SUPPORT=y +CONFIG_FB=y +CONFIG_FB_GOLDFISH=y +CONFIG_FB_SIMPLE=y +CONFIG_BACKLIGHT_LCD_SUPPORT=y +CONFIG_LOGO=y +# CONFIG_LOGO_LINUX_MONO is not set +# CONFIG_LOGO_LINUX_VGA16 is not set +CONFIG_SOUND=y +CONFIG_SND=y +CONFIG_HIDRAW=y +CONFIG_UHID=y +CONFIG_HID_A4TECH=y +CONFIG_HID_ACRUX=y +CONFIG_HID_ACRUX_FF=y +CONFIG_HID_APPLE=y +CONFIG_HID_BELKIN=y +CONFIG_HID_CHERRY=y +CONFIG_HID_CHICONY=y +CONFIG_HID_PRODIKEYS=y +CONFIG_HID_CYPRESS=y +CONFIG_HID_DRAGONRISE=y +CONFIG_DRAGONRISE_FF=y +CONFIG_HID_EMS_FF=y +CONFIG_HID_ELECOM=y +CONFIG_HID_EZKEY=y +CONFIG_HID_KEYTOUCH=y +CONFIG_HID_KYE=y +CONFIG_HID_WALTOP=y +CONFIG_HID_GYRATION=y +CONFIG_HID_TWINHAN=y +CONFIG_HID_KENSINGTON=y +CONFIG_HID_LCPOWER=y +CONFIG_HID_LOGITECH=y +CONFIG_HID_LOGITECH_DJ=y +CONFIG_LOGITECH_FF=y +CONFIG_LOGIRUMBLEPAD2_FF=y +CONFIG_LOGIG940_FF=y +CONFIG_HID_MAGICMOUSE=y +CONFIG_HID_MICROSOFT=y +CONFIG_HID_MONTEREY=y +CONFIG_HID_MULTITOUCH=y +CONFIG_HID_ORTEK=y +CONFIG_HID_PANTHERLORD=y +CONFIG_PANTHERLORD_FF=y +CONFIG_HID_PETALYNX=y +CONFIG_HID_PICOLCD=y +CONFIG_HID_PRIMAX=y +CONFIG_HID_SAITEK=y +CONFIG_HID_SAMSUNG=y +CONFIG_HID_SPEEDLINK=y +CONFIG_HID_SUNPLUS=y +CONFIG_HID_GREENASIA=y +CONFIG_GREENASIA_FF=y +CONFIG_HID_SMARTJOYPLUS=y +CONFIG_SMARTJOYPLUS_FF=y +CONFIG_HID_TIVO=y +CONFIG_HID_TOPSEED=y +CONFIG_HID_THRUSTMASTER=y +CONFIG_HID_WACOM=y +CONFIG_HID_WIIMOTE=y +CONFIG_HID_ZEROPLUS=y +CONFIG_HID_ZYDACRON=y +# CONFIG_USB_SUPPORT is not set +CONFIG_RTC_CLASS=y +CONFIG_VIRTIO_MMIO=y +CONFIG_STAGING=y +CONFIG_ASHMEM=y +CONFIG_ANDROID_TIMED_GPIO=y +CONFIG_ANDROID_LOW_MEMORY_KILLER=y +CONFIG_SYNC=y +CONFIG_SW_SYNC=y +CONFIG_SW_SYNC_USER=y +CONFIG_ION=y +CONFIG_GOLDFISH_AUDIO=y +CONFIG_GOLDFISH=y +CONFIG_GOLDFISH_PIPE=y +# CONFIG_IOMMU_SUPPORT is not set +CONFIG_ANDROID=y +CONFIG_ANDROID_BINDER_IPC=y +CONFIG_EXT2_FS=y +CONFIG_EXT4_FS=y +CONFIG_EXT4_FS_SECURITY=y +CONFIG_QUOTA=y +CONFIG_FUSE_FS=y +CONFIG_CUSE=y +CONFIG_MSDOS_FS=y +CONFIG_VFAT_FS=y +CONFIG_TMPFS=y +CONFIG_TMPFS_POSIX_ACL=y +# CONFIG_MISC_FILESYSTEMS is not set +CONFIG_NFS_FS=y +CONFIG_ROOT_NFS=y +CONFIG_NLS_CODEPAGE_437=y +CONFIG_NLS_ISO8859_1=y +CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_FS=y +CONFIG_MAGIC_SYSRQ=y +CONFIG_PANIC_TIMEOUT=5 +# CONFIG_SCHED_DEBUG is not set +CONFIG_SCHEDSTATS=y +CONFIG_TIMER_STATS=y +# CONFIG_FTRACE is not set +CONFIG_ATOMIC64_SELFTEST=y +CONFIG_DEBUG_RODATA=y +CONFIG_SECURITY=y +CONFIG_SECURITY_NETWORK=y +CONFIG_SECURITY_SELINUX=y diff --git a/arch/arm64/crypto/aes-ce-ccm-core.S b/arch/arm64/crypto/aes-ce-ccm-core.S index a2a7fbcacc141e..3363560c79b7e6 100644 --- a/arch/arm64/crypto/aes-ce-ccm-core.S +++ b/arch/arm64/crypto/aes-ce-ccm-core.S @@ -9,6 +9,7 @@ */ #include +#include .text .arch armv8-a+crypto @@ -19,7 +20,7 @@ */ ENTRY(ce_aes_ccm_auth_data) ldr w8, [x3] /* leftover from prev round? */ - ld1 {v0.2d}, [x0] /* load mac */ + ld1 {v0.16b}, [x0] /* load mac */ cbz w8, 1f sub w8, w8, #16 eor v1.16b, v1.16b, v1.16b @@ -31,7 +32,7 @@ ENTRY(ce_aes_ccm_auth_data) beq 8f /* out of input? */ cbnz w8, 0b eor v0.16b, v0.16b, v1.16b -1: ld1 {v3.2d}, [x4] /* load first round key */ +1: ld1 {v3.16b}, [x4] /* load first round key */ prfm pldl1strm, [x1] cmp w5, #12 /* which key size? */ add x6, x4, #16 @@ -41,17 +42,17 @@ ENTRY(ce_aes_ccm_auth_data) mov v5.16b, v3.16b b 4f 2: mov v4.16b, v3.16b - ld1 {v5.2d}, [x6], #16 /* load 2nd round key */ + ld1 {v5.16b}, [x6], #16 /* load 2nd round key */ 3: aese v0.16b, v4.16b aesmc v0.16b, v0.16b -4: ld1 {v3.2d}, [x6], #16 /* load next round key */ +4: ld1 {v3.16b}, [x6], #16 /* load next round key */ aese v0.16b, v5.16b aesmc v0.16b, v0.16b -5: ld1 {v4.2d}, [x6], #16 /* load next round key */ +5: ld1 {v4.16b}, [x6], #16 /* load next round key */ subs w7, w7, #3 aese v0.16b, v3.16b aesmc v0.16b, v0.16b - ld1 {v5.2d}, [x6], #16 /* load next round key */ + ld1 {v5.16b}, [x6], #16 /* load next round key */ bpl 3b aese v0.16b, v4.16b subs w2, w2, #16 /* last data? */ @@ -60,7 +61,7 @@ ENTRY(ce_aes_ccm_auth_data) ld1 {v1.16b}, [x1], #16 /* load next input block */ eor v0.16b, v0.16b, v1.16b /* xor with mac */ bne 1b -6: st1 {v0.2d}, [x0] /* store mac */ +6: st1 {v0.16b}, [x0] /* store mac */ beq 10f adds w2, w2, #16 beq 10f @@ -79,7 +80,7 @@ ENTRY(ce_aes_ccm_auth_data) adds w7, w7, #1 bne 9b eor v0.16b, v0.16b, v1.16b - st1 {v0.2d}, [x0] + st1 {v0.16b}, [x0] 10: str w8, [x3] ret ENDPROC(ce_aes_ccm_auth_data) @@ -89,27 +90,27 @@ ENDPROC(ce_aes_ccm_auth_data) * u32 rounds); */ ENTRY(ce_aes_ccm_final) - ld1 {v3.2d}, [x2], #16 /* load first round key */ - ld1 {v0.2d}, [x0] /* load mac */ + ld1 {v3.16b}, [x2], #16 /* load first round key */ + ld1 {v0.16b}, [x0] /* load mac */ cmp w3, #12 /* which key size? */ sub w3, w3, #2 /* modified # of rounds */ - ld1 {v1.2d}, [x1] /* load 1st ctriv */ + ld1 {v1.16b}, [x1] /* load 1st ctriv */ bmi 0f bne 3f mov v5.16b, v3.16b b 2f 0: mov v4.16b, v3.16b -1: ld1 {v5.2d}, [x2], #16 /* load next round key */ +1: ld1 {v5.16b}, [x2], #16 /* load next round key */ aese v0.16b, v4.16b aesmc v0.16b, v0.16b aese v1.16b, v4.16b aesmc v1.16b, v1.16b -2: ld1 {v3.2d}, [x2], #16 /* load next round key */ +2: ld1 {v3.16b}, [x2], #16 /* load next round key */ aese v0.16b, v5.16b aesmc v0.16b, v0.16b aese v1.16b, v5.16b aesmc v1.16b, v1.16b -3: ld1 {v4.2d}, [x2], #16 /* load next round key */ +3: ld1 {v4.16b}, [x2], #16 /* load next round key */ subs w3, w3, #3 aese v0.16b, v3.16b aesmc v0.16b, v0.16b @@ -120,47 +121,47 @@ ENTRY(ce_aes_ccm_final) aese v1.16b, v4.16b /* final round key cancels out */ eor v0.16b, v0.16b, v1.16b /* en-/decrypt the mac */ - st1 {v0.2d}, [x0] /* store result */ + st1 {v0.16b}, [x0] /* store result */ ret ENDPROC(ce_aes_ccm_final) .macro aes_ccm_do_crypt,enc ldr x8, [x6, #8] /* load lower ctr */ - ld1 {v0.2d}, [x5] /* load mac */ - rev x8, x8 /* keep swabbed ctr in reg */ + ld1 {v0.16b}, [x5] /* load mac */ +CPU_LE( rev x8, x8 ) /* keep swabbed ctr in reg */ 0: /* outer loop */ - ld1 {v1.1d}, [x6] /* load upper ctr */ + ld1 {v1.8b}, [x6] /* load upper ctr */ prfm pldl1strm, [x1] add x8, x8, #1 rev x9, x8 cmp w4, #12 /* which key size? */ sub w7, w4, #2 /* get modified # of rounds */ ins v1.d[1], x9 /* no carry in lower ctr */ - ld1 {v3.2d}, [x3] /* load first round key */ + ld1 {v3.16b}, [x3] /* load first round key */ add x10, x3, #16 bmi 1f bne 4f mov v5.16b, v3.16b b 3f 1: mov v4.16b, v3.16b - ld1 {v5.2d}, [x10], #16 /* load 2nd round key */ + ld1 {v5.16b}, [x10], #16 /* load 2nd round key */ 2: /* inner loop: 3 rounds, 2x interleaved */ aese v0.16b, v4.16b aesmc v0.16b, v0.16b aese v1.16b, v4.16b aesmc v1.16b, v1.16b -3: ld1 {v3.2d}, [x10], #16 /* load next round key */ +3: ld1 {v3.16b}, [x10], #16 /* load next round key */ aese v0.16b, v5.16b aesmc v0.16b, v0.16b aese v1.16b, v5.16b aesmc v1.16b, v1.16b -4: ld1 {v4.2d}, [x10], #16 /* load next round key */ +4: ld1 {v4.16b}, [x10], #16 /* load next round key */ subs w7, w7, #3 aese v0.16b, v3.16b aesmc v0.16b, v0.16b aese v1.16b, v3.16b aesmc v1.16b, v1.16b - ld1 {v5.2d}, [x10], #16 /* load next round key */ + ld1 {v5.16b}, [x10], #16 /* load next round key */ bpl 2b aese v0.16b, v4.16b aese v1.16b, v4.16b @@ -177,14 +178,14 @@ ENDPROC(ce_aes_ccm_final) eor v0.16b, v0.16b, v2.16b /* xor mac with pt ^ rk[last] */ st1 {v1.16b}, [x0], #16 /* write output block */ bne 0b - rev x8, x8 - st1 {v0.2d}, [x5] /* store mac */ +CPU_LE( rev x8, x8 ) + st1 {v0.16b}, [x5] /* store mac */ str x8, [x6, #8] /* store lsb end of ctr (BE) */ 5: ret 6: eor v0.16b, v0.16b, v5.16b /* final round mac */ eor v1.16b, v1.16b, v5.16b /* final round enc */ - st1 {v0.2d}, [x5] /* store mac */ + st1 {v0.16b}, [x5] /* store mac */ add w2, w2, #16 /* process partial tail block */ 7: ldrb w9, [x1], #1 /* get 1 byte of input */ umov w6, v1.b[0] /* get top crypted ctr byte */ diff --git a/arch/arm64/crypto/aes-ce-cipher.c b/arch/arm64/crypto/aes-ce-cipher.c index f7bd9bf0bbb398..50d9fe11d0c862 100644 --- a/arch/arm64/crypto/aes-ce-cipher.c +++ b/arch/arm64/crypto/aes-ce-cipher.c @@ -47,24 +47,24 @@ static void aes_cipher_encrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[]) kernel_neon_begin_partial(4); __asm__(" ld1 {v0.16b}, %[in] ;" - " ld1 {v1.2d}, [%[key]], #16 ;" + " ld1 {v1.16b}, [%[key]], #16 ;" " cmp %w[rounds], #10 ;" " bmi 0f ;" " bne 3f ;" " mov v3.16b, v1.16b ;" " b 2f ;" "0: mov v2.16b, v1.16b ;" - " ld1 {v3.2d}, [%[key]], #16 ;" + " ld1 {v3.16b}, [%[key]], #16 ;" "1: aese v0.16b, v2.16b ;" " aesmc v0.16b, v0.16b ;" - "2: ld1 {v1.2d}, [%[key]], #16 ;" + "2: ld1 {v1.16b}, [%[key]], #16 ;" " aese v0.16b, v3.16b ;" " aesmc v0.16b, v0.16b ;" - "3: ld1 {v2.2d}, [%[key]], #16 ;" + "3: ld1 {v2.16b}, [%[key]], #16 ;" " subs %w[rounds], %w[rounds], #3 ;" " aese v0.16b, v1.16b ;" " aesmc v0.16b, v0.16b ;" - " ld1 {v3.2d}, [%[key]], #16 ;" + " ld1 {v3.16b}, [%[key]], #16 ;" " bpl 1b ;" " aese v0.16b, v2.16b ;" " eor v0.16b, v0.16b, v3.16b ;" @@ -92,24 +92,24 @@ static void aes_cipher_decrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[]) kernel_neon_begin_partial(4); __asm__(" ld1 {v0.16b}, %[in] ;" - " ld1 {v1.2d}, [%[key]], #16 ;" + " ld1 {v1.16b}, [%[key]], #16 ;" " cmp %w[rounds], #10 ;" " bmi 0f ;" " bne 3f ;" " mov v3.16b, v1.16b ;" " b 2f ;" "0: mov v2.16b, v1.16b ;" - " ld1 {v3.2d}, [%[key]], #16 ;" + " ld1 {v3.16b}, [%[key]], #16 ;" "1: aesd v0.16b, v2.16b ;" " aesimc v0.16b, v0.16b ;" - "2: ld1 {v1.2d}, [%[key]], #16 ;" + "2: ld1 {v1.16b}, [%[key]], #16 ;" " aesd v0.16b, v3.16b ;" " aesimc v0.16b, v0.16b ;" - "3: ld1 {v2.2d}, [%[key]], #16 ;" + "3: ld1 {v2.16b}, [%[key]], #16 ;" " subs %w[rounds], %w[rounds], #3 ;" " aesd v0.16b, v1.16b ;" " aesimc v0.16b, v0.16b ;" - " ld1 {v3.2d}, [%[key]], #16 ;" + " ld1 {v3.16b}, [%[key]], #16 ;" " bpl 1b ;" " aesd v0.16b, v2.16b ;" " eor v0.16b, v0.16b, v3.16b ;" @@ -173,7 +173,12 @@ int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, u32 *rki = ctx->key_enc + (i * kwords); u32 *rko = rki + kwords; +#ifndef CONFIG_CPU_BIG_ENDIAN rko[0] = ror32(aes_sub(rki[kwords - 1]), 8) ^ rcon[i] ^ rki[0]; +#else + rko[0] = rol32(aes_sub(rki[kwords - 1]), 8) ^ (rcon[i] << 24) ^ + rki[0]; +#endif rko[1] = rko[0] ^ rki[1]; rko[2] = rko[1] ^ rki[2]; rko[3] = rko[2] ^ rki[3]; diff --git a/arch/arm64/crypto/aes-ce.S b/arch/arm64/crypto/aes-ce.S index 78f3cfe92c0872..b46093d567e544 100644 --- a/arch/arm64/crypto/aes-ce.S +++ b/arch/arm64/crypto/aes-ce.S @@ -10,6 +10,7 @@ */ #include +#include #define AES_ENTRY(func) ENTRY(ce_ ## func) #define AES_ENDPROC(func) ENDPROC(ce_ ## func) diff --git a/arch/arm64/crypto/aes-modes.S b/arch/arm64/crypto/aes-modes.S index f6e372c528eb43..838dad5c209fae 100644 --- a/arch/arm64/crypto/aes-modes.S +++ b/arch/arm64/crypto/aes-modes.S @@ -193,15 +193,16 @@ AES_ENTRY(aes_cbc_encrypt) cbz w6, .Lcbcencloop ld1 {v0.16b}, [x5] /* get iv */ - enc_prepare w3, x2, x5 + enc_prepare w3, x2, x6 .Lcbcencloop: ld1 {v1.16b}, [x1], #16 /* get next pt block */ eor v0.16b, v0.16b, v1.16b /* ..and xor with iv */ - encrypt_block v0, w3, x2, x5, w6 + encrypt_block v0, w3, x2, x6, w7 st1 {v0.16b}, [x0], #16 subs w4, w4, #1 bne .Lcbcencloop + st1 {v0.16b}, [x5] /* return iv */ ret AES_ENDPROC(aes_cbc_encrypt) @@ -211,7 +212,7 @@ AES_ENTRY(aes_cbc_decrypt) cbz w6, .LcbcdecloopNx ld1 {v7.16b}, [x5] /* get iv */ - dec_prepare w3, x2, x5 + dec_prepare w3, x2, x6 .LcbcdecloopNx: #if INTERLEAVE >= 2 @@ -248,7 +249,7 @@ AES_ENTRY(aes_cbc_decrypt) .Lcbcdecloop: ld1 {v1.16b}, [x1], #16 /* get next ct block */ mov v0.16b, v1.16b /* ...and copy to v0 */ - decrypt_block v0, w3, x2, x5, w6 + decrypt_block v0, w3, x2, x6, w7 eor v0.16b, v0.16b, v7.16b /* xor with iv => pt */ mov v7.16b, v1.16b /* ct is next iv */ st1 {v0.16b}, [x0], #16 @@ -256,6 +257,7 @@ AES_ENTRY(aes_cbc_decrypt) bne .Lcbcdecloop .Lcbcdecout: FRAME_POP + st1 {v7.16b}, [x5] /* return iv */ ret AES_ENDPROC(aes_cbc_decrypt) @@ -267,24 +269,15 @@ AES_ENDPROC(aes_cbc_decrypt) AES_ENTRY(aes_ctr_encrypt) FRAME_PUSH - cbnz w6, .Lctrfirst /* 1st time around? */ - umov x5, v4.d[1] /* keep swabbed ctr in reg */ - rev x5, x5 -#if INTERLEAVE >= 2 - cmn w5, w4 /* 32 bit overflow? */ - bcs .Lctrinc - add x5, x5, #1 /* increment BE ctr */ - b .LctrincNx -#else - b .Lctrinc -#endif -.Lctrfirst: + cbz w6, .Lctrnotfirst /* 1st time around? */ enc_prepare w3, x2, x6 ld1 {v4.16b}, [x5] - umov x5, v4.d[1] /* keep swabbed ctr in reg */ - rev x5, x5 + +.Lctrnotfirst: + umov x8, v4.d[1] /* keep swabbed ctr in reg */ + rev x8, x8 #if INTERLEAVE >= 2 - cmn w5, w4 /* 32 bit overflow? */ + cmn w8, w4 /* 32 bit overflow? */ bcs .Lctrloop .LctrloopNx: subs w4, w4, #INTERLEAVE @@ -292,11 +285,11 @@ AES_ENTRY(aes_ctr_encrypt) #if INTERLEAVE == 2 mov v0.8b, v4.8b mov v1.8b, v4.8b - rev x7, x5 - add x5, x5, #1 + rev x7, x8 + add x8, x8, #1 ins v0.d[1], x7 - rev x7, x5 - add x5, x5, #1 + rev x7, x8 + add x8, x8, #1 ins v1.d[1], x7 ld1 {v2.16b-v3.16b}, [x1], #32 /* get 2 input blocks */ do_encrypt_block2x @@ -305,7 +298,7 @@ AES_ENTRY(aes_ctr_encrypt) st1 {v0.16b-v1.16b}, [x0], #32 #else ldr q8, =0x30000000200000001 /* addends 1,2,3[,0] */ - dup v7.4s, w5 + dup v7.4s, w8 mov v0.16b, v4.16b add v7.4s, v7.4s, v8.4s mov v1.16b, v4.16b @@ -323,18 +316,12 @@ AES_ENTRY(aes_ctr_encrypt) eor v2.16b, v7.16b, v2.16b eor v3.16b, v5.16b, v3.16b st1 {v0.16b-v3.16b}, [x0], #64 - add x5, x5, #INTERLEAVE + add x8, x8, #INTERLEAVE #endif - cbz w4, .LctroutNx -.LctrincNx: - rev x7, x5 + rev x7, x8 ins v4.d[1], x7 + cbz w4, .Lctrout b .LctrloopNx -.LctroutNx: - sub x5, x5, #1 - rev x7, x5 - ins v4.d[1], x7 - b .Lctrout .Lctr1x: adds w4, w4, #INTERLEAVE beq .Lctrout @@ -342,30 +329,39 @@ AES_ENTRY(aes_ctr_encrypt) .Lctrloop: mov v0.16b, v4.16b encrypt_block v0, w3, x2, x6, w7 + + adds x8, x8, #1 /* increment BE ctr */ + rev x7, x8 + ins v4.d[1], x7 + bcs .Lctrcarry /* overflow? */ + +.Lctrcarrydone: subs w4, w4, #1 bmi .Lctrhalfblock /* blocks < 0 means 1/2 block */ ld1 {v3.16b}, [x1], #16 eor v3.16b, v0.16b, v3.16b st1 {v3.16b}, [x0], #16 - beq .Lctrout -.Lctrinc: - adds x5, x5, #1 /* increment BE ctr */ - rev x7, x5 - ins v4.d[1], x7 - bcc .Lctrloop /* no overflow? */ - umov x7, v4.d[0] /* load upper word of ctr */ - rev x7, x7 /* ... to handle the carry */ - add x7, x7, #1 - rev x7, x7 - ins v4.d[0], x7 - b .Lctrloop + bne .Lctrloop + +.Lctrout: + st1 {v4.16b}, [x5] /* return next CTR value */ + FRAME_POP + ret + .Lctrhalfblock: ld1 {v3.8b}, [x1] eor v3.8b, v0.8b, v3.8b st1 {v3.8b}, [x0] -.Lctrout: FRAME_POP ret + +.Lctrcarry: + umov x7, v4.d[0] /* load upper word of ctr */ + rev x7, x7 /* ... to handle the carry */ + add x7, x7, #1 + rev x7, x7 + ins v4.d[0], x7 + b .Lctrcarrydone AES_ENDPROC(aes_ctr_encrypt) .ltorg @@ -386,7 +382,8 @@ AES_ENDPROC(aes_ctr_encrypt) .endm .Lxts_mul_x: - .word 1, 0, 0x87, 0 +CPU_LE( .quad 1, 0x87 ) +CPU_BE( .quad 0x87, 1 ) AES_ENTRY(aes_xts_encrypt) FRAME_PUSH diff --git a/arch/arm64/crypto/aes-neon.S b/arch/arm64/crypto/aes-neon.S index b93170e1cc9338..85f07ead7c5c27 100644 --- a/arch/arm64/crypto/aes-neon.S +++ b/arch/arm64/crypto/aes-neon.S @@ -9,6 +9,7 @@ */ #include +#include #define AES_ENTRY(func) ENTRY(neon_ ## func) #define AES_ENDPROC(func) ENDPROC(neon_ ## func) @@ -83,13 +84,13 @@ .endm .macro do_block, enc, in, rounds, rk, rkp, i - ld1 {v15.16b}, [\rk] + ld1 {v15.4s}, [\rk] add \rkp, \rk, #16 mov \i, \rounds 1111: eor \in\().16b, \in\().16b, v15.16b /* ^round key */ tbl \in\().16b, {\in\().16b}, v13.16b /* ShiftRows */ sub_bytes \in - ld1 {v15.16b}, [\rkp], #16 + ld1 {v15.4s}, [\rkp], #16 subs \i, \i, #1 beq 2222f .if \enc == 1 @@ -229,7 +230,7 @@ .endm .macro do_block_2x, enc, in0, in1 rounds, rk, rkp, i - ld1 {v15.16b}, [\rk] + ld1 {v15.4s}, [\rk] add \rkp, \rk, #16 mov \i, \rounds 1111: eor \in0\().16b, \in0\().16b, v15.16b /* ^round key */ @@ -237,7 +238,7 @@ sub_bytes_2x \in0, \in1 tbl \in0\().16b, {\in0\().16b}, v13.16b /* ShiftRows */ tbl \in1\().16b, {\in1\().16b}, v13.16b /* ShiftRows */ - ld1 {v15.16b}, [\rkp], #16 + ld1 {v15.4s}, [\rkp], #16 subs \i, \i, #1 beq 2222f .if \enc == 1 @@ -254,7 +255,7 @@ .endm .macro do_block_4x, enc, in0, in1, in2, in3, rounds, rk, rkp, i - ld1 {v15.16b}, [\rk] + ld1 {v15.4s}, [\rk] add \rkp, \rk, #16 mov \i, \rounds 1111: eor \in0\().16b, \in0\().16b, v15.16b /* ^round key */ @@ -266,7 +267,7 @@ tbl \in1\().16b, {\in1\().16b}, v13.16b /* ShiftRows */ tbl \in2\().16b, {\in2\().16b}, v13.16b /* ShiftRows */ tbl \in3\().16b, {\in3\().16b}, v13.16b /* ShiftRows */ - ld1 {v15.16b}, [\rkp], #16 + ld1 {v15.4s}, [\rkp], #16 subs \i, \i, #1 beq 2222f .if \enc == 1 @@ -306,12 +307,16 @@ .text .align 4 .LForward_ShiftRows: - .byte 0x0, 0x5, 0xa, 0xf, 0x4, 0x9, 0xe, 0x3 - .byte 0x8, 0xd, 0x2, 0x7, 0xc, 0x1, 0x6, 0xb +CPU_LE( .byte 0x0, 0x5, 0xa, 0xf, 0x4, 0x9, 0xe, 0x3 ) +CPU_LE( .byte 0x8, 0xd, 0x2, 0x7, 0xc, 0x1, 0x6, 0xb ) +CPU_BE( .byte 0xb, 0x6, 0x1, 0xc, 0x7, 0x2, 0xd, 0x8 ) +CPU_BE( .byte 0x3, 0xe, 0x9, 0x4, 0xf, 0xa, 0x5, 0x0 ) .LReverse_ShiftRows: - .byte 0x0, 0xd, 0xa, 0x7, 0x4, 0x1, 0xe, 0xb - .byte 0x8, 0x5, 0x2, 0xf, 0xc, 0x9, 0x6, 0x3 +CPU_LE( .byte 0x0, 0xd, 0xa, 0x7, 0x4, 0x1, 0xe, 0xb ) +CPU_LE( .byte 0x8, 0x5, 0x2, 0xf, 0xc, 0x9, 0x6, 0x3 ) +CPU_BE( .byte 0x3, 0x6, 0x9, 0xc, 0xf, 0x2, 0x5, 0x8 ) +CPU_BE( .byte 0xb, 0xe, 0x1, 0x4, 0x7, 0xa, 0xd, 0x0 ) .LForward_Sbox: .byte 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5 diff --git a/arch/arm64/crypto/ghash-ce-core.S b/arch/arm64/crypto/ghash-ce-core.S index dc457015884e04..f0bb9f0b524fce 100644 --- a/arch/arm64/crypto/ghash-ce-core.S +++ b/arch/arm64/crypto/ghash-ce-core.S @@ -29,8 +29,8 @@ * struct ghash_key const *k, const char *head) */ ENTRY(pmull_ghash_update) - ld1 {SHASH.16b}, [x3] - ld1 {XL.16b}, [x1] + ld1 {SHASH.2d}, [x3] + ld1 {XL.2d}, [x1] movi MASK.16b, #0xe1 ext SHASH2.16b, SHASH.16b, SHASH.16b, #8 shl MASK.2d, MASK.2d, #57 @@ -74,6 +74,6 @@ CPU_LE( rev64 T1.16b, T1.16b ) cbnz w0, 0b - st1 {XL.16b}, [x1] + st1 {XL.2d}, [x1] ret ENDPROC(pmull_ghash_update) diff --git a/arch/arm64/crypto/sha1-ce-core.S b/arch/arm64/crypto/sha1-ce-core.S index 033aae6d732a14..c98e7e849f06f4 100644 --- a/arch/arm64/crypto/sha1-ce-core.S +++ b/arch/arm64/crypto/sha1-ce-core.S @@ -78,7 +78,7 @@ ENTRY(sha1_ce_transform) ld1r {k3.4s}, [x6] /* load state */ - ldr dga, [x0] + ld1 {dgav.4s}, [x0] ldr dgb, [x0, #16] /* load sha1_ce_state::finalize */ @@ -144,7 +144,7 @@ CPU_LE( rev32 v11.16b, v11.16b ) b 1b /* store new state */ -3: str dga, [x0] +3: st1 {dgav.4s}, [x0] str dgb, [x0, #16] ret ENDPROC(sha1_ce_transform) diff --git a/arch/arm64/crypto/sha2-ce-core.S b/arch/arm64/crypto/sha2-ce-core.S index 5df9d9d470adb5..01cfee066837cd 100644 --- a/arch/arm64/crypto/sha2-ce-core.S +++ b/arch/arm64/crypto/sha2-ce-core.S @@ -85,7 +85,7 @@ ENTRY(sha2_ce_transform) ld1 {v12.4s-v15.4s}, [x8] /* load state */ - ldp dga, dgb, [x0] + ld1 {dgav.4s, dgbv.4s}, [x0] /* load sha256_ce_state::finalize */ ldr w4, [x0, #:lo12:sha256_ce_offsetof_finalize] @@ -148,6 +148,6 @@ CPU_LE( rev32 v19.16b, v19.16b ) b 1b /* store new state */ -3: stp dga, dgb, [x0] +3: st1 {dgav.4s, dgbv.4s}, [x0] ret ENDPROC(sha2_ce_transform) diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h index b71086d251954f..53211a0acf0f07 100644 --- a/arch/arm64/include/asm/memory.h +++ b/arch/arm64/include/asm/memory.h @@ -217,7 +217,7 @@ static inline void *phys_to_virt(phys_addr_t x) #define _virt_addr_valid(kaddr) pfn_valid(__pa(kaddr) >> PAGE_SHIFT) #else #define __virt_to_pgoff(kaddr) (((u64)(kaddr) & ~PAGE_OFFSET) / PAGE_SIZE * sizeof(struct page)) -#define __page_to_voff(page) (((u64)(page) & ~VMEMMAP_START) * PAGE_SIZE / sizeof(struct page)) +#define __page_to_voff(kaddr) (((u64)(kaddr) & ~VMEMMAP_START) * PAGE_SIZE / sizeof(struct page)) #define page_to_virt(page) ((void *)((__page_to_voff(page)) | PAGE_OFFSET)) #define virt_to_page(vaddr) ((struct page *)((__virt_to_pgoff(vaddr)) | VMEMMAP_START)) diff --git a/arch/arm64/include/asm/topology.h b/arch/arm64/include/asm/topology.h index 8b57339823e9b1..7ec84d0191c8ef 100644 --- a/arch/arm64/include/asm/topology.h +++ b/arch/arm64/include/asm/topology.h @@ -31,6 +31,14 @@ int pcibus_to_node(struct pci_bus *bus); cpumask_of_node(pcibus_to_node(bus))) #endif /* CONFIG_NUMA */ +struct sched_domain; +#ifdef CONFIG_CPU_FREQ +#define arch_scale_freq_capacity cpufreq_scale_freq_capacity +extern unsigned long cpufreq_scale_freq_capacity(struct sched_domain *sd, int cpu); +extern unsigned long cpufreq_scale_max_freq_capacity(int cpu); +#endif +#define arch_scale_cpu_capacity scale_cpu_capacity +extern unsigned long scale_cpu_capacity(struct sched_domain *sd, int cpu); #include diff --git a/arch/arm64/include/uapi/asm/ptrace.h b/arch/arm64/include/uapi/asm/ptrace.h index b5c3933ed44163..d1ff83dfe5deae 100644 --- a/arch/arm64/include/uapi/asm/ptrace.h +++ b/arch/arm64/include/uapi/asm/ptrace.h @@ -77,6 +77,7 @@ struct user_fpsimd_state { __uint128_t vregs[32]; __u32 fpsr; __u32 fpcr; + __u32 __reserved[2]; }; struct user_hwdebug_state { diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 223d54a4d66b44..79b0fe24d5b753 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -624,7 +624,7 @@ el0_inv: mov x0, sp mov x1, #BAD_SYNC mov x2, x25 - bl bad_mode + bl bad_el0_sync b ret_to_user ENDPROC(el0_sync) diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 01753cd7d3f01d..2a474166d38456 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -166,6 +166,70 @@ void machine_restart(char *cmd) while (1); } +/* + * dump a block of kernel memory from around the given address + */ +static void show_data(unsigned long addr, int nbytes, const char *name) +{ + int i, j; + int nlines; + u32 *p; + + /* + * don't attempt to dump non-kernel addresses or + * values that are probably just small negative numbers + */ + if (addr < PAGE_OFFSET || addr > -256UL) + return; + + printk("\n%s: %#lx:\n", name, addr); + + /* + * round address down to a 32 bit boundary + * and always dump a multiple of 32 bytes + */ + p = (u32 *)(addr & ~(sizeof(u32) - 1)); + nbytes += (addr & (sizeof(u32) - 1)); + nlines = (nbytes + 31) / 32; + + + for (i = 0; i < nlines; i++) { + /* + * just display low 16 bits of address to keep + * each line of the dump < 80 characters + */ + printk("%04lx ", (unsigned long)p & 0xffff); + for (j = 0; j < 8; j++) { + u32 data; + if (probe_kernel_address(p, data)) { + printk(" ********"); + } else { + printk(" %08x", data); + } + ++p; + } + printk("\n"); + } +} + +static void show_extra_register_data(struct pt_regs *regs, int nbytes) +{ + mm_segment_t fs; + unsigned int i; + + fs = get_fs(); + set_fs(KERNEL_DS); + show_data(regs->pc - nbytes, nbytes * 2, "PC"); + show_data(regs->regs[30] - nbytes, nbytes * 2, "LR"); + show_data(regs->sp - nbytes, nbytes * 2, "SP"); + for (i = 0; i < 30; i++) { + char name[4]; + snprintf(name, sizeof(name), "X%u", i); + show_data(regs->regs[i] - nbytes, nbytes * 2, name); + } + set_fs(fs); +} + void __show_regs(struct pt_regs *regs) { int i, top_reg; @@ -201,6 +265,8 @@ void __show_regs(struct pt_regs *regs) pr_cont("\n"); } + if (!user_mode(regs)) + show_extra_register_data(regs, 128); printk("\n"); } diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index e0c81da60f7627..8eedeef375d606 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -550,6 +550,8 @@ static int hw_break_set(struct task_struct *target, /* (address, ctrl) registers */ limit = regset->n * regset->size; while (count && offset < limit) { + if (count < PTRACE_HBP_ADDR_SZ) + return -EINVAL; ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &addr, offset, offset + PTRACE_HBP_ADDR_SZ); if (ret) @@ -559,6 +561,8 @@ static int hw_break_set(struct task_struct *target, return ret; offset += PTRACE_HBP_ADDR_SZ; + if (!count) + break; ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &ctrl, offset, offset + PTRACE_HBP_CTRL_SZ); if (ret) @@ -595,7 +599,7 @@ static int gpr_set(struct task_struct *target, const struct user_regset *regset, const void *kbuf, const void __user *ubuf) { int ret; - struct user_pt_regs newregs; + struct user_pt_regs newregs = task_pt_regs(target)->user_regs; ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &newregs, 0, -1); if (ret) @@ -625,7 +629,8 @@ static int fpr_set(struct task_struct *target, const struct user_regset *regset, const void *kbuf, const void __user *ubuf) { int ret; - struct user_fpsimd_state newstate; + struct user_fpsimd_state newstate = + target->thread.fpsimd_state.user_fpsimd; ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &newstate, 0, -1); if (ret) @@ -649,7 +654,7 @@ static int tls_set(struct task_struct *target, const struct user_regset *regset, const void *kbuf, const void __user *ubuf) { int ret; - unsigned long tls; + unsigned long tls = target->thread.tp_value; ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &tls, 0, -1); if (ret) @@ -675,7 +680,8 @@ static int system_call_set(struct task_struct *target, unsigned int pos, unsigned int count, const void *kbuf, const void __user *ubuf) { - int syscallno, ret; + int syscallno = task_pt_regs(target)->syscallno; + int ret; ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &syscallno, 0, -1); if (ret) @@ -947,7 +953,7 @@ static int compat_tls_set(struct task_struct *target, const void __user *ubuf) { int ret; - compat_ulong_t tls; + compat_ulong_t tls = target->thread.tp_value; ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &tls, 0, -1); if (ret) diff --git a/arch/arm64/kernel/topology.c b/arch/arm64/kernel/topology.c index 694f6deedbab89..5b2c67a510d8a5 100644 --- a/arch/arm64/kernel/topology.c +++ b/arch/arm64/kernel/topology.c @@ -19,10 +19,30 @@ #include #include #include +#include +#include #include #include +static DEFINE_PER_CPU(unsigned long, cpu_scale) = SCHED_CAPACITY_SCALE; + +unsigned long scale_cpu_capacity(struct sched_domain *sd, int cpu) +{ +#ifdef CONFIG_CPU_FREQ + unsigned long max_freq_scale = cpufreq_scale_max_freq_capacity(cpu); + + return per_cpu(cpu_scale, cpu) * max_freq_scale >> SCHED_CAPACITY_SHIFT; +#else + return per_cpu(cpu_scale, cpu); +#endif +} + +static void set_capacity_scale(unsigned int cpu, unsigned long capacity) +{ + per_cpu(cpu_scale, cpu) = capacity; +} + static int __init get_cpu_for_node(struct device_node *node) { struct device_node *cpu_node; @@ -206,11 +226,67 @@ static int __init parse_dt_topology(void) struct cpu_topology cpu_topology[NR_CPUS]; EXPORT_SYMBOL_GPL(cpu_topology); +/* sd energy functions */ +static inline +const struct sched_group_energy * const cpu_cluster_energy(int cpu) +{ + struct sched_group_energy *sge = sge_array[cpu][SD_LEVEL1]; + + if (!sge) { + pr_warn("Invalid sched_group_energy for Cluster%d\n", cpu); + return NULL; + } + + return sge; +} + +static inline +const struct sched_group_energy * const cpu_core_energy(int cpu) +{ + struct sched_group_energy *sge = sge_array[cpu][SD_LEVEL0]; + + if (!sge) { + pr_warn("Invalid sched_group_energy for CPU%d\n", cpu); + return NULL; + } + + return sge; +} + const struct cpumask *cpu_coregroup_mask(int cpu) { return &cpu_topology[cpu].core_sibling; } +static inline int cpu_corepower_flags(void) +{ + return SD_SHARE_PKG_RESOURCES | SD_SHARE_POWERDOMAIN | \ + SD_SHARE_CAP_STATES; +} + +static struct sched_domain_topology_level arm64_topology[] = { +#ifdef CONFIG_SCHED_MC + { cpu_coregroup_mask, cpu_corepower_flags, cpu_core_energy, SD_INIT_NAME(MC) }, +#endif + { cpu_cpu_mask, NULL, cpu_cluster_energy, SD_INIT_NAME(DIE) }, + { NULL, }, +}; + +static void update_cpu_capacity(unsigned int cpu) +{ + unsigned long capacity = SCHED_CAPACITY_SCALE; + + if (cpu_core_energy(cpu)) { + int max_cap_idx = cpu_core_energy(cpu)->nr_cap_states - 1; + capacity = cpu_core_energy(cpu)->cap_states[max_cap_idx].cap; + } + + set_capacity_scale(cpu, capacity); + + pr_info("CPU%d: update cpu_capacity %lu\n", + cpu, arch_scale_cpu_capacity(NULL, cpu)); +} + static void update_siblings_masks(unsigned int cpuid) { struct cpu_topology *cpu_topo, *cpuid_topo = &cpu_topology[cpuid]; @@ -272,6 +348,7 @@ void store_cpu_topology(unsigned int cpuid) topology_populated: update_siblings_masks(cpuid); + update_cpu_capacity(cpuid); } static void __init reset_cpu_topology(void) @@ -302,4 +379,8 @@ void __init init_cpu_topology(void) */ if (of_have_populated_dt() && parse_dt_topology()) reset_cpu_topology(); + else + set_sched_topology(arm64_topology); + + init_sched_energy_costs(); } diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c index c9986b3e0a96f9..11e5eae088abac 100644 --- a/arch/arm64/kernel/traps.c +++ b/arch/arm64/kernel/traps.c @@ -596,17 +596,34 @@ const char *esr_get_class_string(u32 esr) } /* - * bad_mode handles the impossible case in the exception vector. + * bad_mode handles the impossible case in the exception vector. This is always + * fatal. */ asmlinkage void bad_mode(struct pt_regs *regs, int reason, unsigned int esr) { - siginfo_t info; - void __user *pc = (void __user *)instruction_pointer(regs); console_verbose(); pr_crit("Bad mode in %s handler detected on CPU%d, code 0x%08x -- %s\n", handler[reason], smp_processor_id(), esr, esr_get_class_string(esr)); + + die("Oops - bad mode", regs, 0); + local_irq_disable(); + panic("bad mode"); +} + +/* + * bad_el0_sync handles unexpected, but potentially recoverable synchronous + * exceptions taken from EL0. Unlike bad_mode, this returns. + */ +asmlinkage void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr) +{ + siginfo_t info; + void __user *pc = (void __user *)instruction_pointer(regs); + console_verbose(); + + pr_crit("Bad EL0 synchronous exception detected on CPU%d, code 0x%08x -- %s\n", + smp_processor_id(), esr, esr_get_class_string(esr)); __show_regs(regs); info.si_signo = SIGILL; @@ -614,7 +631,10 @@ asmlinkage void bad_mode(struct pt_regs *regs, int reason, unsigned int esr) info.si_code = ILL_ILLOPC; info.si_addr = pc; - arm64_notify_die("Oops - bad mode", regs, &info, 0); + current->thread.fault_address = 0; + current->thread.fault_code = 0; + + force_sig_info(info.si_signo, &info, current); } void __pte_error(const char *file, int line, unsigned long val) diff --git a/arch/arm64/kvm/hyp/switch.c b/arch/arm64/kvm/hyp/switch.c index 83037cd62d013a..0c848c18ca4470 100644 --- a/arch/arm64/kvm/hyp/switch.c +++ b/arch/arm64/kvm/hyp/switch.c @@ -85,7 +85,13 @@ static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu) write_sysreg(val, hcr_el2); /* Trap on AArch32 cp15 c15 accesses (EL1 or EL0) */ write_sysreg(1 << 15, hstr_el2); - /* Make sure we trap PMU access from EL0 to EL2 */ + /* + * Make sure we trap PMU access from EL0 to EL2. Also sanitize + * PMSELR_EL0 to make sure it never contains the cycle + * counter, which could make a PMXEVCNTR_EL0 access UNDEF at + * EL1 instead of being trapped to EL2. + */ + write_sysreg(0, pmselr_el0); write_sysreg(ARMV8_PMU_USERENR_MASK, pmuserenr_el0); write_sysreg(vcpu->arch.mdcr_el2, mdcr_el2); __activate_traps_arch()(); diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c index 3f74d0d98de633..9fb1263e053398 100644 --- a/arch/arm64/mm/dma-mapping.c +++ b/arch/arm64/mm/dma-mapping.c @@ -174,7 +174,7 @@ static void *__dma_alloc(struct device *dev, size_t size, /* create a coherent mapping */ page = virt_to_page(ptr); coherent_ptr = dma_common_contiguous_remap(page, size, VM_USERMAP, - prot, NULL); + prot, __builtin_return_address(0)); if (!coherent_ptr) goto no_map; @@ -524,7 +524,8 @@ EXPORT_SYMBOL(dummy_dma_ops); static int __init arm64_dma_init(void) { - if (swiotlb_force || max_pfn > (arm64_dma_phys_limit >> PAGE_SHIFT)) + if (swiotlb_force == SWIOTLB_FORCE || + max_pfn > (arm64_dma_phys_limit >> PAGE_SHIFT)) swiotlb = 1; return atomic_pool_init(); diff --git a/arch/arm64/mm/hugetlbpage.c b/arch/arm64/mm/hugetlbpage.c index 2e49bd252fe764..45bec627bae3ec 100644 --- a/arch/arm64/mm/hugetlbpage.c +++ b/arch/arm64/mm/hugetlbpage.c @@ -51,20 +51,8 @@ static int find_num_contig(struct mm_struct *mm, unsigned long addr, *pgsize = PAGE_SIZE; if (!pte_cont(pte)) return 1; - if (!pgd_present(*pgd)) { - VM_BUG_ON(!pgd_present(*pgd)); - return 1; - } pud = pud_offset(pgd, addr); - if (!pud_present(*pud)) { - VM_BUG_ON(!pud_present(*pud)); - return 1; - } pmd = pmd_offset(pud, addr); - if (!pmd_present(*pmd)) { - VM_BUG_ON(!pmd_present(*pmd)); - return 1; - } if ((pte_t *)pmd == ptep) { *pgsize = PMD_SIZE; return CONT_PMDS; @@ -212,7 +200,7 @@ pte_t huge_ptep_get_and_clear(struct mm_struct *mm, ncontig = find_num_contig(mm, addr, cpte, *cpte, &pgsize); /* save the 1st pte to return */ pte = ptep_get_and_clear(mm, addr, cpte); - for (i = 1; i < ncontig; ++i) { + for (i = 1, addr += pgsize; i < ncontig; ++i, addr += pgsize) { /* * If HW_AFDBM is enabled, then the HW could * turn on the dirty bit for any of the page @@ -250,8 +238,8 @@ int huge_ptep_set_access_flags(struct vm_area_struct *vma, pfn = pte_pfn(*cpte); ncontig = find_num_contig(vma->vm_mm, addr, cpte, *cpte, &pgsize); - for (i = 0; i < ncontig; ++i, ++cpte) { - changed = ptep_set_access_flags(vma, addr, cpte, + for (i = 0; i < ncontig; ++i, ++cpte, addr += pgsize) { + changed |= ptep_set_access_flags(vma, addr, cpte, pfn_pte(pfn, hugeprot), dirty); @@ -273,7 +261,7 @@ void huge_ptep_set_wrprotect(struct mm_struct *mm, cpte = huge_pte_offset(mm, addr); ncontig = find_num_contig(mm, addr, cpte, *cpte, &pgsize); - for (i = 0; i < ncontig; ++i, ++cpte) + for (i = 0; i < ncontig; ++i, ++cpte, addr += pgsize) ptep_set_wrprotect(mm, addr, cpte); } else { ptep_set_wrprotect(mm, addr, ptep); @@ -291,7 +279,7 @@ void huge_ptep_clear_flush(struct vm_area_struct *vma, cpte = huge_pte_offset(vma->vm_mm, addr); ncontig = find_num_contig(vma->vm_mm, addr, cpte, *cpte, &pgsize); - for (i = 0; i < ncontig; ++i, ++cpte) + for (i = 0; i < ncontig; ++i, ++cpte, addr += pgsize) ptep_clear_flush(vma, addr, cpte); } else { ptep_clear_flush(vma, addr, ptep); diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index 212c4d1e2f26df..0b9492e85ffce4 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -145,9 +145,11 @@ static void __init zone_sizes_init(unsigned long min, unsigned long max) #endif /* CONFIG_NUMA */ #ifdef CONFIG_HAVE_ARCH_PFN_VALID +#define PFN_MASK ((1UL << (64 - PAGE_SHIFT)) - 1) + int pfn_valid(unsigned long pfn) { - return memblock_is_map_memory(pfn << PAGE_SHIFT); + return (pfn & PFN_MASK) == pfn && memblock_is_map_memory(pfn << PAGE_SHIFT); } EXPORT_SYMBOL(pfn_valid); #endif @@ -401,8 +403,11 @@ static void __init free_unused_memmap(void) */ void __init mem_init(void) { - if (swiotlb_force || max_pfn > (arm64_dma_phys_limit >> PAGE_SHIFT)) + if (swiotlb_force == SWIOTLB_FORCE || + max_pfn > (arm64_dma_phys_limit >> PAGE_SHIFT)) swiotlb_init(1); + else + swiotlb_force = SWIOTLB_NO_FORCE; set_max_mapnr(pfn_to_page(max_pfn) - mem_map); diff --git a/arch/blackfin/kernel/ptrace.c b/arch/blackfin/kernel/ptrace.c index 8d79286ee4e878..360d996451633c 100644 --- a/arch/blackfin/kernel/ptrace.c +++ b/arch/blackfin/kernel/ptrace.c @@ -270,7 +270,7 @@ long arch_ptrace(struct task_struct *child, long request, switch (bfin_mem_access_type(addr, to_copy)) { case BFIN_MEM_ACCESS_CORE: case BFIN_MEM_ACCESS_CORE_ONLY: - copied = access_process_vm(child, addr, &tmp, + copied = ptrace_access_vm(child, addr, &tmp, to_copy, FOLL_FORCE); if (copied) break; @@ -323,7 +323,7 @@ long arch_ptrace(struct task_struct *child, long request, switch (bfin_mem_access_type(addr, to_copy)) { case BFIN_MEM_ACCESS_CORE: case BFIN_MEM_ACCESS_CORE_ONLY: - copied = access_process_vm(child, addr, &data, + copied = ptrace_access_vm(child, addr, &data, to_copy, FOLL_FORCE | FOLL_WRITE); break; diff --git a/arch/cris/arch-v32/kernel/ptrace.c b/arch/cris/arch-v32/kernel/ptrace.c index f0df654ac6fc5c..fe1f9cf7b391e5 100644 --- a/arch/cris/arch-v32/kernel/ptrace.c +++ b/arch/cris/arch-v32/kernel/ptrace.c @@ -147,7 +147,7 @@ long arch_ptrace(struct task_struct *child, long request, /* The trampoline page is globally mapped, no page table to traverse.*/ tmp = *(unsigned long*)addr; } else { - copied = access_process_vm(child, addr, &tmp, sizeof(tmp), FOLL_FORCE); + copied = ptrace_access_vm(child, addr, &tmp, sizeof(tmp), FOLL_FORCE); if (copied != sizeof(tmp)) break; diff --git a/arch/cris/boot/rescue/Makefile b/arch/cris/boot/rescue/Makefile index 52bd0bd1dd22f2..d98edbb30a18e8 100644 --- a/arch/cris/boot/rescue/Makefile +++ b/arch/cris/boot/rescue/Makefile @@ -10,6 +10,9 @@ asflags-y += $(LINUXINCLUDE) ccflags-y += -O2 $(LINUXINCLUDE) + +ifdef CONFIG_ETRAX_AXISFLASHMAP + arch-$(CONFIG_ETRAX_ARCH_V10) = v10 arch-$(CONFIG_ETRAX_ARCH_V32) = v32 @@ -28,6 +31,11 @@ $(obj)/rescue.bin: $(obj)/rescue.o FORCE $(call if_changed,objcopy) cp -p $(obj)/rescue.bin $(objtree) +else +$(obj)/rescue.bin: + +endif + $(obj)/testrescue.bin: $(obj)/testrescue.o $(OBJCOPY) $(OBJCOPYFLAGS) $(obj)/testrescue.o tr.bin # Pad it to 784 bytes diff --git a/arch/ia64/kernel/ptrace.c b/arch/ia64/kernel/ptrace.c index 31aa8c0f68e14a..36f660da81242f 100644 --- a/arch/ia64/kernel/ptrace.c +++ b/arch/ia64/kernel/ptrace.c @@ -1159,7 +1159,7 @@ arch_ptrace (struct task_struct *child, long request, case PTRACE_PEEKTEXT: case PTRACE_PEEKDATA: /* read word at location addr */ - if (access_process_vm(child, addr, &data, sizeof(data), + if (ptrace_access_vm(child, addr, &data, sizeof(data), FOLL_FORCE) != sizeof(data)) return -EIO; diff --git a/arch/mips/kernel/ptrace32.c b/arch/mips/kernel/ptrace32.c index 7e71a4e0281ba9..5fcbdcd7abd0bc 100644 --- a/arch/mips/kernel/ptrace32.c +++ b/arch/mips/kernel/ptrace32.c @@ -69,7 +69,7 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request, if (get_user(addrOthers, (u32 __user * __user *) (unsigned long) addr) != 0) break; - copied = access_process_vm(child, (u64)addrOthers, &tmp, + copied = ptrace_access_vm(child, (u64)addrOthers, &tmp, sizeof(tmp), FOLL_FORCE); if (copied != sizeof(tmp)) break; @@ -178,7 +178,7 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request, if (get_user(addrOthers, (u32 __user * __user *) (unsigned long) addr) != 0) break; ret = 0; - if (access_process_vm(child, (u64)addrOthers, &data, + if (ptrace_access_vm(child, (u64)addrOthers, &data, sizeof(data), FOLL_FORCE | FOLL_WRITE) == sizeof(data)) break; diff --git a/arch/mips/kvm/entry.c b/arch/mips/kvm/entry.c index 6a02b3a3fa6513..e92fb190e2d628 100644 --- a/arch/mips/kvm/entry.c +++ b/arch/mips/kvm/entry.c @@ -521,6 +521,9 @@ void *kvm_mips_build_exit(void *addr) uasm_i_and(&p, V0, V0, AT); uasm_i_lui(&p, AT, ST0_CU0 >> 16); uasm_i_or(&p, V0, V0, AT); +#ifdef CONFIG_64BIT + uasm_i_ori(&p, V0, V0, ST0_SX | ST0_UX); +#endif uasm_i_mtc0(&p, V0, C0_STATUS); uasm_i_ehb(&p); @@ -643,7 +646,7 @@ static void *kvm_mips_build_ret_to_guest(void *addr) /* Setup status register for running guest in UM */ uasm_i_ori(&p, V1, V1, ST0_EXL | KSU_USER | ST0_IE); - UASM_i_LA(&p, AT, ~(ST0_CU0 | ST0_MX)); + UASM_i_LA(&p, AT, ~(ST0_CU0 | ST0_MX | ST0_SX | ST0_UX)); uasm_i_and(&p, V1, V1, AT); uasm_i_mtc0(&p, V1, C0_STATUS); uasm_i_ehb(&p); diff --git a/arch/mips/kvm/mips.c b/arch/mips/kvm/mips.c index 06a60b19acfb53..29ec9ab3fd5570 100644 --- a/arch/mips/kvm/mips.c +++ b/arch/mips/kvm/mips.c @@ -360,8 +360,8 @@ struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id) dump_handler("kvm_exit", gebase + 0x2000, vcpu->arch.vcpu_run); /* Invalidate the icache for these ranges */ - local_flush_icache_range((unsigned long)gebase, - (unsigned long)gebase + ALIGN(size, PAGE_SIZE)); + flush_icache_range((unsigned long)gebase, + (unsigned long)gebase + ALIGN(size, PAGE_SIZE)); /* * Allocate comm page for guest kernel, a TLB will be reserved for diff --git a/arch/parisc/include/asm/bitops.h b/arch/parisc/include/asm/bitops.h index 3f9406d9b9d675..da87943328a510 100644 --- a/arch/parisc/include/asm/bitops.h +++ b/arch/parisc/include/asm/bitops.h @@ -6,7 +6,7 @@ #endif #include -#include /* for BITS_PER_LONG/SHIFT_PER_LONG */ +#include #include #include #include @@ -17,6 +17,12 @@ * to include/asm-i386/bitops.h or kerneldoc */ +#if __BITS_PER_LONG == 64 +#define SHIFT_PER_LONG 6 +#else +#define SHIFT_PER_LONG 5 +#endif + #define CHOP_SHIFTCOUNT(x) (((unsigned long) (x)) & (BITS_PER_LONG - 1)) diff --git a/arch/parisc/include/uapi/asm/bitsperlong.h b/arch/parisc/include/uapi/asm/bitsperlong.h index e0a23c7bdd432a..07fa7e50bdc069 100644 --- a/arch/parisc/include/uapi/asm/bitsperlong.h +++ b/arch/parisc/include/uapi/asm/bitsperlong.h @@ -3,10 +3,8 @@ #if defined(__LP64__) #define __BITS_PER_LONG 64 -#define SHIFT_PER_LONG 6 #else #define __BITS_PER_LONG 32 -#define SHIFT_PER_LONG 5 #endif #include diff --git a/arch/parisc/include/uapi/asm/swab.h b/arch/parisc/include/uapi/asm/swab.h index e78403b129ef92..928e1bbac98fef 100644 --- a/arch/parisc/include/uapi/asm/swab.h +++ b/arch/parisc/include/uapi/asm/swab.h @@ -1,6 +1,7 @@ #ifndef _PARISC_SWAB_H #define _PARISC_SWAB_H +#include #include #include @@ -38,7 +39,7 @@ static inline __attribute_const__ __u32 __arch_swab32(__u32 x) } #define __arch_swab32 __arch_swab32 -#if BITS_PER_LONG > 32 +#if __BITS_PER_LONG > 32 /* ** From "PA-RISC 2.0 Architecture", HP Professional Books. ** See Appendix I page 8 , "Endian Byte Swapping". @@ -61,6 +62,6 @@ static inline __attribute_const__ __u64 __arch_swab64(__u64 x) return x; } #define __arch_swab64 __arch_swab64 -#endif /* BITS_PER_LONG > 32 */ +#endif /* __BITS_PER_LONG > 32 */ #endif /* _PARISC_SWAB_H */ diff --git a/arch/parisc/kernel/time.c b/arch/parisc/kernel/time.c index 325f30d82b6434..47ef8fdcd382cb 100644 --- a/arch/parisc/kernel/time.c +++ b/arch/parisc/kernel/time.c @@ -289,9 +289,26 @@ void __init time_init(void) cr16_hz = 100 * PAGE0->mem_10msec; /* Hz */ - /* register at clocksource framework */ - clocksource_register_hz(&clocksource_cr16, cr16_hz); - /* register as sched_clock source */ sched_clock_register(read_cr16_sched_clock, BITS_PER_LONG, cr16_hz); } + +static int __init init_cr16_clocksource(void) +{ + /* + * The cr16 interval timers are not syncronized across CPUs, so mark + * them unstable and lower rating on SMP systems. + */ + if (num_online_cpus() > 1) { + clocksource_cr16.flags = CLOCK_SOURCE_UNSTABLE; + clocksource_cr16.rating = 0; + } + + /* register at clocksource framework */ + clocksource_register_hz(&clocksource_cr16, + 100 * PAGE0->mem_10msec); + + return 0; +} + +device_initcall(init_cr16_clocksource); diff --git a/arch/parisc/mm/fault.c b/arch/parisc/mm/fault.c index 8ff9253930af77..1a0b4f63f0e90f 100644 --- a/arch/parisc/mm/fault.c +++ b/arch/parisc/mm/fault.c @@ -234,7 +234,7 @@ show_signal_msg(struct pt_regs *regs, unsigned long code, tsk->comm, code, address); print_vma_addr(KERN_CONT " in ", regs->iaoq[0]); - pr_cont(" trap #%lu: %s%c", code, trap_name(code), + pr_cont("\ntrap #%lu: %s%c", code, trap_name(code), vma ? ',':'\n'); if (vma) diff --git a/arch/powerpc/boot/ps3-head.S b/arch/powerpc/boot/ps3-head.S index b6fcbaf5027bed..3dc44b05fb9724 100644 --- a/arch/powerpc/boot/ps3-head.S +++ b/arch/powerpc/boot/ps3-head.S @@ -57,11 +57,6 @@ __system_reset_overlay: bctr 1: - /* Save the value at addr zero for a null pointer write check later. */ - - li r4, 0 - lwz r3, 0(r4) - /* Primary delays then goes to _zimage_start in wrapper. */ or 31, 31, 31 /* db16cyc */ diff --git a/arch/powerpc/boot/ps3.c b/arch/powerpc/boot/ps3.c index 4ec2d86d3c5057..a05558a7e51ad3 100644 --- a/arch/powerpc/boot/ps3.c +++ b/arch/powerpc/boot/ps3.c @@ -119,13 +119,12 @@ void ps3_copy_vectors(void) flush_cache((void *)0x100, 512); } -void platform_init(unsigned long null_check) +void platform_init(void) { const u32 heapsize = 0x1000000 - (u32)_end; /* 16MiB */ void *chosen; unsigned long ft_addr; u64 rm_size; - unsigned long val; console_ops.write = ps3_console_write; platform_ops.exit = ps3_exit; @@ -153,11 +152,6 @@ void platform_init(unsigned long null_check) printf(" flat tree at 0x%lx\n\r", ft_addr); - val = *(unsigned long *)0; - - if (val != null_check) - printf("null check failed: %lx != %lx\n\r", val, null_check); - ((kernel_entry_t)0)(ft_addr, 0, NULL); ps3_exit(); diff --git a/arch/powerpc/boot/wrapper b/arch/powerpc/boot/wrapper index 404b3aabdb4dcd..76fe3ccfd381d6 100755 --- a/arch/powerpc/boot/wrapper +++ b/arch/powerpc/boot/wrapper @@ -181,6 +181,28 @@ case "$elfformat" in elf32-powerpc) format=elf32ppc ;; esac +ld_version() +{ + # Poached from scripts/ld-version.sh, but we don't want to call that because + # this script (wrapper) is distributed separately from the kernel source. + # Extract linker version number from stdin and turn into single number. + awk '{ + gsub(".*\\)", ""); + gsub(".*version ", ""); + gsub("-.*", ""); + split($1,a, "."); + print a[1]*100000000 + a[2]*1000000 + a[3]*10000; + exit + }' +} + +# Do not include PT_INTERP segment when linking pie. Non-pie linking +# just ignores this option. +LD_VERSION=$(${CROSS}ld --version | ld_version) +LD_NO_DL_MIN_VERSION=$(echo 2.26 | ld_version) +if [ "$LD_VERSION" -ge "$LD_NO_DL_MIN_VERSION" ] ; then + nodl="--no-dynamic-linker" +fi platformo=$object/"$platform".o lds=$object/zImage.lds @@ -446,7 +468,7 @@ if [ "$platform" != "miboot" ]; then text_start="-Ttext $link_address" fi #link everything - ${CROSS}ld -m $format -T $lds $text_start $pie -o "$ofile" \ + ${CROSS}ld -m $format -T $lds $text_start $pie $nodl -o "$ofile" \ $platformo $tmp $object/wrapper.a rm $tmp fi diff --git a/arch/powerpc/include/asm/book3s/64/mmu-hash.h b/arch/powerpc/include/asm/book3s/64/mmu-hash.h index e407af2b733350..2e6a823fa50272 100644 --- a/arch/powerpc/include/asm/book3s/64/mmu-hash.h +++ b/arch/powerpc/include/asm/book3s/64/mmu-hash.h @@ -70,7 +70,9 @@ #define HPTE_V_SSIZE_SHIFT 62 #define HPTE_V_AVPN_SHIFT 7 +#define HPTE_V_COMMON_BITS ASM_CONST(0x000fffffffffffff) #define HPTE_V_AVPN ASM_CONST(0x3fffffffffffff80) +#define HPTE_V_AVPN_3_0 ASM_CONST(0x000fffffffffff80) #define HPTE_V_AVPN_VAL(x) (((x) & HPTE_V_AVPN) >> HPTE_V_AVPN_SHIFT) #define HPTE_V_COMPARE(x,y) (!(((x) ^ (y)) & 0xffffffffffffff80UL)) #define HPTE_V_BOLTED ASM_CONST(0x0000000000000010) @@ -80,14 +82,16 @@ #define HPTE_V_VALID ASM_CONST(0x0000000000000001) /* - * ISA 3.0 have a different HPTE format. + * ISA 3.0 has a different HPTE format. */ #define HPTE_R_3_0_SSIZE_SHIFT 58 +#define HPTE_R_3_0_SSIZE_MASK (3ull << HPTE_R_3_0_SSIZE_SHIFT) #define HPTE_R_PP0 ASM_CONST(0x8000000000000000) #define HPTE_R_TS ASM_CONST(0x4000000000000000) #define HPTE_R_KEY_HI ASM_CONST(0x3000000000000000) #define HPTE_R_RPN_SHIFT 12 #define HPTE_R_RPN ASM_CONST(0x0ffffffffffff000) +#define HPTE_R_RPN_3_0 ASM_CONST(0x01fffffffffff000) #define HPTE_R_PP ASM_CONST(0x0000000000000003) #define HPTE_R_PPP ASM_CONST(0x8000000000000003) #define HPTE_R_N ASM_CONST(0x0000000000000004) @@ -316,11 +320,42 @@ static inline unsigned long hpte_encode_avpn(unsigned long vpn, int psize, */ v = (vpn >> (23 - VPN_SHIFT)) & ~(mmu_psize_defs[psize].avpnm); v <<= HPTE_V_AVPN_SHIFT; - if (!cpu_has_feature(CPU_FTR_ARCH_300)) - v |= ((unsigned long) ssize) << HPTE_V_SSIZE_SHIFT; + v |= ((unsigned long) ssize) << HPTE_V_SSIZE_SHIFT; return v; } +/* + * ISA v3.0 defines a new HPTE format, which differs from the old + * format in having smaller AVPN and ARPN fields, and the B field + * in the second dword instead of the first. + */ +static inline unsigned long hpte_old_to_new_v(unsigned long v) +{ + /* trim AVPN, drop B */ + return v & HPTE_V_COMMON_BITS; +} + +static inline unsigned long hpte_old_to_new_r(unsigned long v, unsigned long r) +{ + /* move B field from 1st to 2nd dword, trim ARPN */ + return (r & ~HPTE_R_3_0_SSIZE_MASK) | + (((v) >> HPTE_V_SSIZE_SHIFT) << HPTE_R_3_0_SSIZE_SHIFT); +} + +static inline unsigned long hpte_new_to_old_v(unsigned long v, unsigned long r) +{ + /* insert B field */ + return (v & HPTE_V_COMMON_BITS) | + ((r & HPTE_R_3_0_SSIZE_MASK) << + (HPTE_V_SSIZE_SHIFT - HPTE_R_3_0_SSIZE_SHIFT)); +} + +static inline unsigned long hpte_new_to_old_r(unsigned long r) +{ + /* clear out B field */ + return r & ~HPTE_R_3_0_SSIZE_MASK; +} + /* * This function sets the AVPN and L fields of the HPTE appropriately * using the base page size and actual page size. @@ -341,12 +376,8 @@ static inline unsigned long hpte_encode_v(unsigned long vpn, int base_psize, * aligned for the requested page size */ static inline unsigned long hpte_encode_r(unsigned long pa, int base_psize, - int actual_psize, int ssize) + int actual_psize) { - - if (cpu_has_feature(CPU_FTR_ARCH_300)) - pa |= ((unsigned long) ssize) << HPTE_R_3_0_SSIZE_SHIFT; - /* A 4K page needs no special encoding */ if (actual_psize == MMU_PAGE_4K) return pa & HPTE_R_RPN; diff --git a/arch/powerpc/include/asm/cpu_has_feature.h b/arch/powerpc/include/asm/cpu_has_feature.h index b312b152461b05..6e834caa37206a 100644 --- a/arch/powerpc/include/asm/cpu_has_feature.h +++ b/arch/powerpc/include/asm/cpu_has_feature.h @@ -23,7 +23,9 @@ static __always_inline bool cpu_has_feature(unsigned long feature) { int i; +#ifndef __clang__ /* clang can't cope with this */ BUILD_BUG_ON(!__builtin_constant_p(feature)); +#endif #ifdef CONFIG_JUMP_LABEL_FEATURE_CHECK_DEBUG if (!static_key_initialized) { diff --git a/arch/powerpc/include/asm/kvm_host.h b/arch/powerpc/include/asm/kvm_host.h index 28350a294b1e04..5e12e19940e27d 100644 --- a/arch/powerpc/include/asm/kvm_host.h +++ b/arch/powerpc/include/asm/kvm_host.h @@ -546,6 +546,7 @@ struct kvm_vcpu_arch { u64 tfiar; u32 cr_tm; + u64 xer_tm; u64 lr_tm; u64 ctr_tm; u64 amr_tm; diff --git a/arch/powerpc/include/asm/mmu.h b/arch/powerpc/include/asm/mmu.h index e311c25751a411..a244e09d2d8828 100644 --- a/arch/powerpc/include/asm/mmu.h +++ b/arch/powerpc/include/asm/mmu.h @@ -160,7 +160,9 @@ static __always_inline bool mmu_has_feature(unsigned long feature) { int i; +#ifndef __clang__ /* clang can't cope with this */ BUILD_BUG_ON(!__builtin_constant_p(feature)); +#endif #ifdef CONFIG_JUMP_LABEL_FEATURE_CHECK_DEBUG if (!static_key_initialized) { diff --git a/arch/powerpc/include/asm/ppc-opcode.h b/arch/powerpc/include/asm/ppc-opcode.h index c56ea8c84abb17..c4ced1d01d579f 100644 --- a/arch/powerpc/include/asm/ppc-opcode.h +++ b/arch/powerpc/include/asm/ppc-opcode.h @@ -157,7 +157,7 @@ #define PPC_INST_MCRXR 0x7c000400 #define PPC_INST_MCRXR_MASK 0xfc0007fe #define PPC_INST_MFSPR_PVR 0x7c1f42a6 -#define PPC_INST_MFSPR_PVR_MASK 0xfc1fffff +#define PPC_INST_MFSPR_PVR_MASK 0xfc1ffffe #define PPC_INST_MFTMR 0x7c0002dc #define PPC_INST_MSGSND 0x7c00019c #define PPC_INST_MSGCLR 0x7c0001dc @@ -174,13 +174,13 @@ #define PPC_INST_RFDI 0x4c00004e #define PPC_INST_RFMCI 0x4c00004c #define PPC_INST_MFSPR_DSCR 0x7c1102a6 -#define PPC_INST_MFSPR_DSCR_MASK 0xfc1fffff +#define PPC_INST_MFSPR_DSCR_MASK 0xfc1ffffe #define PPC_INST_MTSPR_DSCR 0x7c1103a6 -#define PPC_INST_MTSPR_DSCR_MASK 0xfc1fffff +#define PPC_INST_MTSPR_DSCR_MASK 0xfc1ffffe #define PPC_INST_MFSPR_DSCR_USER 0x7c0302a6 -#define PPC_INST_MFSPR_DSCR_USER_MASK 0xfc1fffff +#define PPC_INST_MFSPR_DSCR_USER_MASK 0xfc1ffffe #define PPC_INST_MTSPR_DSCR_USER 0x7c0303a6 -#define PPC_INST_MTSPR_DSCR_USER_MASK 0xfc1fffff +#define PPC_INST_MTSPR_DSCR_USER_MASK 0xfc1ffffe #define PPC_INST_MFVSRD 0x7c000066 #define PPC_INST_MTVSRD 0x7c000166 #define PPC_INST_SLBFEE 0x7c0007a7 diff --git a/arch/powerpc/include/asm/reg.h b/arch/powerpc/include/asm/reg.h index 9e1499f98deff5..13f5fad21066eb 100644 --- a/arch/powerpc/include/asm/reg.h +++ b/arch/powerpc/include/asm/reg.h @@ -641,9 +641,10 @@ #define SRR1_ISI_N_OR_G 0x10000000 /* ISI: Access is no-exec or G */ #define SRR1_ISI_PROT 0x08000000 /* ISI: Other protection fault */ #define SRR1_WAKEMASK 0x00380000 /* reason for wakeup */ -#define SRR1_WAKEMASK_P8 0x003c0000 /* reason for wakeup on POWER8 */ +#define SRR1_WAKEMASK_P8 0x003c0000 /* reason for wakeup on POWER8 and 9 */ #define SRR1_WAKESYSERR 0x00300000 /* System error */ #define SRR1_WAKEEE 0x00200000 /* External interrupt */ +#define SRR1_WAKEHVI 0x00240000 /* Hypervisor Virtualization Interrupt (P9) */ #define SRR1_WAKEMT 0x00280000 /* mtctrl */ #define SRR1_WAKEHMI 0x00280000 /* Hypervisor maintenance */ #define SRR1_WAKEDEC 0x00180000 /* Decrementer interrupt */ diff --git a/arch/powerpc/include/asm/xics.h b/arch/powerpc/include/asm/xics.h index f0b238516e9b44..e0b9e576905aa2 100644 --- a/arch/powerpc/include/asm/xics.h +++ b/arch/powerpc/include/asm/xics.h @@ -44,6 +44,7 @@ static inline int icp_hv_init(void) { return -ENODEV; } #ifdef CONFIG_PPC_POWERNV extern int icp_opal_init(void); +extern void icp_opal_flush_interrupt(void); #else static inline int icp_opal_init(void) { return -ENODEV; } #endif diff --git a/arch/powerpc/include/uapi/asm/kvm.h b/arch/powerpc/include/uapi/asm/kvm.h index c93cf35ce37955..0fb1326c3ea24b 100644 --- a/arch/powerpc/include/uapi/asm/kvm.h +++ b/arch/powerpc/include/uapi/asm/kvm.h @@ -596,6 +596,7 @@ struct kvm_get_htab_header { #define KVM_REG_PPC_TM_VSCR (KVM_REG_PPC_TM | KVM_REG_SIZE_U32 | 0x67) #define KVM_REG_PPC_TM_DSCR (KVM_REG_PPC_TM | KVM_REG_SIZE_U64 | 0x68) #define KVM_REG_PPC_TM_TAR (KVM_REG_PPC_TM | KVM_REG_SIZE_U64 | 0x69) +#define KVM_REG_PPC_TM_XER (KVM_REG_PPC_TM | KVM_REG_SIZE_U64 | 0x6a) /* PPC64 eXternal Interrupt Controller Specification */ #define KVM_DEV_XICS_GRP_SOURCES 1 /* 64-bit source attributes */ diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c index caec7bf3b99aeb..c833d88c423d8b 100644 --- a/arch/powerpc/kernel/asm-offsets.c +++ b/arch/powerpc/kernel/asm-offsets.c @@ -569,6 +569,7 @@ int main(void) DEFINE(VCPU_VRS_TM, offsetof(struct kvm_vcpu, arch.vr_tm.vr)); DEFINE(VCPU_VRSAVE_TM, offsetof(struct kvm_vcpu, arch.vrsave_tm)); DEFINE(VCPU_CR_TM, offsetof(struct kvm_vcpu, arch.cr_tm)); + DEFINE(VCPU_XER_TM, offsetof(struct kvm_vcpu, arch.xer_tm)); DEFINE(VCPU_LR_TM, offsetof(struct kvm_vcpu, arch.lr_tm)); DEFINE(VCPU_CTR_TM, offsetof(struct kvm_vcpu, arch.ctr_tm)); DEFINE(VCPU_AMR_TM, offsetof(struct kvm_vcpu, arch.amr_tm)); diff --git a/arch/powerpc/kernel/eeh_driver.c b/arch/powerpc/kernel/eeh_driver.c index 5c31369435f242..a5dd493670a063 100644 --- a/arch/powerpc/kernel/eeh_driver.c +++ b/arch/powerpc/kernel/eeh_driver.c @@ -545,7 +545,7 @@ static void *eeh_pe_detach_dev(void *data, void *userdata) static void *__eeh_clear_pe_frozen_state(void *data, void *flag) { struct eeh_pe *pe = (struct eeh_pe *)data; - bool *clear_sw_state = flag; + bool clear_sw_state = *(bool *)flag; int i, rc = 1; for (i = 0; rc && i < 3; i++) diff --git a/arch/powerpc/kernel/head_64.S b/arch/powerpc/kernel/head_64.S index 04c546e20cc05e..1f7f908f186e2a 100644 --- a/arch/powerpc/kernel/head_64.S +++ b/arch/powerpc/kernel/head_64.S @@ -214,9 +214,9 @@ booting_thread_hwid: */ _GLOBAL(book3e_start_thread) LOAD_REG_IMMEDIATE(r5, MSR_KERNEL) - cmpi 0, r3, 0 + cmpwi r3, 0 beq 10f - cmpi 0, r3, 1 + cmpwi r3, 1 beq 11f /* If the thread id is invalid, just exit. */ b 13f @@ -241,9 +241,9 @@ _GLOBAL(book3e_start_thread) * r3 = the thread physical id */ _GLOBAL(book3e_stop_thread) - cmpi 0, r3, 0 + cmpwi r3, 0 beq 10f - cmpi 0, r3, 1 + cmpwi r3, 1 beq 10f /* If the thread id is invalid, just exit. */ b 13f diff --git a/arch/powerpc/kernel/ibmebus.c b/arch/powerpc/kernel/ibmebus.c index 6ca9a2ffaac72d..35f5244782d9f2 100644 --- a/arch/powerpc/kernel/ibmebus.c +++ b/arch/powerpc/kernel/ibmebus.c @@ -180,6 +180,7 @@ static int ibmebus_create_device(struct device_node *dn) static int ibmebus_create_devices(const struct of_device_id *matches) { struct device_node *root, *child; + struct device *dev; int ret = 0; root = of_find_node_by_path("/"); @@ -188,9 +189,12 @@ static int ibmebus_create_devices(const struct of_device_id *matches) if (!of_match_node(matches, child)) continue; - if (bus_find_device(&ibmebus_bus_type, NULL, child, - ibmebus_match_node)) + dev = bus_find_device(&ibmebus_bus_type, NULL, child, + ibmebus_match_node); + if (dev) { + put_device(dev); continue; + } ret = ibmebus_create_device(child); if (ret) { @@ -262,6 +266,7 @@ static ssize_t ibmebus_store_probe(struct bus_type *bus, const char *buf, size_t count) { struct device_node *dn = NULL; + struct device *dev; char *path; ssize_t rc = 0; @@ -269,8 +274,10 @@ static ssize_t ibmebus_store_probe(struct bus_type *bus, if (!path) return -ENOMEM; - if (bus_find_device(&ibmebus_bus_type, NULL, path, - ibmebus_match_path)) { + dev = bus_find_device(&ibmebus_bus_type, NULL, path, + ibmebus_match_path); + if (dev) { + put_device(dev); printk(KERN_WARNING "%s: %s has already been probed\n", __func__, path); rc = -EEXIST; @@ -307,6 +314,7 @@ static ssize_t ibmebus_store_remove(struct bus_type *bus, if ((dev = bus_find_device(&ibmebus_bus_type, NULL, path, ibmebus_match_path))) { of_device_unregister(to_platform_device(dev)); + put_device(dev); kfree(path); return count; diff --git a/arch/powerpc/kernel/misc_32.S b/arch/powerpc/kernel/misc_32.S index 93cf7a5846a6f5..030d72df5dd535 100644 --- a/arch/powerpc/kernel/misc_32.S +++ b/arch/powerpc/kernel/misc_32.S @@ -296,7 +296,7 @@ _GLOBAL(flush_instruction_cache) lis r3, KERNELBASE@h iccci 0,r3 #endif -#elif CONFIG_FSL_BOOKE +#elif defined(CONFIG_FSL_BOOKE) BEGIN_FTR_SECTION mfspr r3,SPRN_L1CSR0 ori r3,r3,L1CSR0_CFI|L1CSR0_CLFC diff --git a/arch/powerpc/kernel/prom_init.c b/arch/powerpc/kernel/prom_init.c index 88ac964f48586b..1e8c57207346e6 100644 --- a/arch/powerpc/kernel/prom_init.c +++ b/arch/powerpc/kernel/prom_init.c @@ -2747,6 +2747,9 @@ static void __init prom_find_boot_cpu(void) cpu_pkg = call_prom("instance-to-package", 1, 1, prom_cpu); + if (!PHANDLE_VALID(cpu_pkg)) + return; + prom_getprop(cpu_pkg, "reg", &rval, sizeof(rval)); prom.cpu = be32_to_cpu(rval); diff --git a/arch/powerpc/kernel/ptrace.c b/arch/powerpc/kernel/ptrace.c index b1ec62f2cc316a..5c8f12fe9721d0 100644 --- a/arch/powerpc/kernel/ptrace.c +++ b/arch/powerpc/kernel/ptrace.c @@ -463,6 +463,10 @@ static int fpr_set(struct task_struct *target, const struct user_regset *regset, flush_fp_to_thread(target); + for (i = 0; i < 32 ; i++) + buf[i] = target->thread.TS_FPR(i); + buf[32] = target->thread.fp_state.fpscr; + /* copy to local buffer then write that out */ i = user_regset_copyin(&pos, &count, &kbuf, &ubuf, buf, 0, -1); if (i) @@ -672,6 +676,9 @@ static int vsr_set(struct task_struct *target, const struct user_regset *regset, flush_altivec_to_thread(target); flush_vsx_to_thread(target); + for (i = 0; i < 32 ; i++) + buf[i] = target->thread.fp_state.fpr[i][TS_VSRLOWOFFSET]; + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, buf, 0, 32 * sizeof(double)); if (!ret) @@ -1019,6 +1026,10 @@ static int tm_cfpr_set(struct task_struct *target, flush_fp_to_thread(target); flush_altivec_to_thread(target); + for (i = 0; i < 32; i++) + buf[i] = target->thread.TS_CKFPR(i); + buf[32] = target->thread.ckfp_state.fpscr; + /* copy to local buffer then write that out */ i = user_regset_copyin(&pos, &count, &kbuf, &ubuf, buf, 0, -1); if (i) @@ -1283,6 +1294,9 @@ static int tm_cvsx_set(struct task_struct *target, flush_altivec_to_thread(target); flush_vsx_to_thread(target); + for (i = 0; i < 32 ; i++) + buf[i] = target->thread.ckfp_state.fpr[i][TS_VSRLOWOFFSET]; + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, buf, 0, 32 * sizeof(double)); if (!ret) diff --git a/arch/powerpc/kernel/ptrace32.c b/arch/powerpc/kernel/ptrace32.c index 010b7b310237e4..1e887f3a61a675 100644 --- a/arch/powerpc/kernel/ptrace32.c +++ b/arch/powerpc/kernel/ptrace32.c @@ -73,7 +73,7 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request, if (get_user(addrOthers, (u32 __user * __user *)addr) != 0) break; - copied = access_process_vm(child, (u64)addrOthers, &tmp, + copied = ptrace_access_vm(child, (u64)addrOthers, &tmp, sizeof(tmp), FOLL_FORCE); if (copied != sizeof(tmp)) break; @@ -178,7 +178,7 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request, if (get_user(addrOthers, (u32 __user * __user *)addr) != 0) break; ret = 0; - if (access_process_vm(child, (u64)addrOthers, &tmp, + if (ptrace_access_vm(child, (u64)addrOthers, &tmp, sizeof(tmp), FOLL_FORCE | FOLL_WRITE) == sizeof(tmp)) break; diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c index 3686471be32bd1..094deb60c6fe0a 100644 --- a/arch/powerpc/kvm/book3s_hv.c +++ b/arch/powerpc/kvm/book3s_hv.c @@ -1288,6 +1288,9 @@ static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, case KVM_REG_PPC_TM_CR: *val = get_reg_val(id, vcpu->arch.cr_tm); break; + case KVM_REG_PPC_TM_XER: + *val = get_reg_val(id, vcpu->arch.xer_tm); + break; case KVM_REG_PPC_TM_LR: *val = get_reg_val(id, vcpu->arch.lr_tm); break; @@ -1498,6 +1501,9 @@ static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, case KVM_REG_PPC_TM_CR: vcpu->arch.cr_tm = set_reg_val(id, *val); break; + case KVM_REG_PPC_TM_XER: + vcpu->arch.xer_tm = set_reg_val(id, *val); + break; case KVM_REG_PPC_TM_LR: vcpu->arch.lr_tm = set_reg_val(id, *val); break; diff --git a/arch/powerpc/kvm/book3s_hv_rm_mmu.c b/arch/powerpc/kvm/book3s_hv_rm_mmu.c index 99b4e9d5dd238c..5420d060c6f61a 100644 --- a/arch/powerpc/kvm/book3s_hv_rm_mmu.c +++ b/arch/powerpc/kvm/book3s_hv_rm_mmu.c @@ -653,6 +653,8 @@ long kvmppc_h_protect(struct kvm_vcpu *vcpu, unsigned long flags, HPTE_V_ABSENT); do_tlbies(kvm, &rb, 1, global_invalidates(kvm, flags), true); + /* Don't lose R/C bit updates done by hardware */ + r |= be64_to_cpu(hpte[1]) & (HPTE_R_R | HPTE_R_C); hpte[1] = cpu_to_be64(r); } } diff --git a/arch/powerpc/kvm/book3s_hv_rmhandlers.S b/arch/powerpc/kvm/book3s_hv_rmhandlers.S index c3c1d1bcfc67da..6f81adb112f1bf 100644 --- a/arch/powerpc/kvm/book3s_hv_rmhandlers.S +++ b/arch/powerpc/kvm/book3s_hv_rmhandlers.S @@ -2600,11 +2600,13 @@ kvmppc_save_tm: mfctr r7 mfspr r8, SPRN_AMR mfspr r10, SPRN_TAR + mfxer r11 std r5, VCPU_LR_TM(r9) stw r6, VCPU_CR_TM(r9) std r7, VCPU_CTR_TM(r9) std r8, VCPU_AMR_TM(r9) std r10, VCPU_TAR_TM(r9) + std r11, VCPU_XER_TM(r9) /* Restore r12 as trap number. */ lwz r12, VCPU_TRAP(r9) @@ -2697,11 +2699,13 @@ kvmppc_restore_tm: ld r7, VCPU_CTR_TM(r4) ld r8, VCPU_AMR_TM(r4) ld r9, VCPU_TAR_TM(r4) + ld r10, VCPU_XER_TM(r4) mtlr r5 mtcr r6 mtctr r7 mtspr SPRN_AMR, r8 mtspr SPRN_TAR, r9 + mtxer r10 /* * Load up PPR and DSCR values but don't put them in the actual SPRs diff --git a/arch/powerpc/mm/hash_native_64.c b/arch/powerpc/mm/hash_native_64.c index 83ddc0e171b0d6..ad9fd5245be280 100644 --- a/arch/powerpc/mm/hash_native_64.c +++ b/arch/powerpc/mm/hash_native_64.c @@ -221,13 +221,18 @@ static long native_hpte_insert(unsigned long hpte_group, unsigned long vpn, return -1; hpte_v = hpte_encode_v(vpn, psize, apsize, ssize) | vflags | HPTE_V_VALID; - hpte_r = hpte_encode_r(pa, psize, apsize, ssize) | rflags; + hpte_r = hpte_encode_r(pa, psize, apsize) | rflags; if (!(vflags & HPTE_V_BOLTED)) { DBG_LOW(" i=%x hpte_v=%016lx, hpte_r=%016lx\n", i, hpte_v, hpte_r); } + if (cpu_has_feature(CPU_FTR_ARCH_300)) { + hpte_r = hpte_old_to_new_r(hpte_v, hpte_r); + hpte_v = hpte_old_to_new_v(hpte_v); + } + hptep->r = cpu_to_be64(hpte_r); /* Guarantee the second dword is visible before the valid bit */ eieio(); @@ -295,6 +300,8 @@ static long native_hpte_updatepp(unsigned long slot, unsigned long newpp, vpn, want_v & HPTE_V_AVPN, slot, newpp); hpte_v = be64_to_cpu(hptep->v); + if (cpu_has_feature(CPU_FTR_ARCH_300)) + hpte_v = hpte_new_to_old_v(hpte_v, be64_to_cpu(hptep->r)); /* * We need to invalidate the TLB always because hpte_remove doesn't do * a tlb invalidate. If a hash bucket gets full, we "evict" a more/less @@ -309,6 +316,8 @@ static long native_hpte_updatepp(unsigned long slot, unsigned long newpp, native_lock_hpte(hptep); /* recheck with locks held */ hpte_v = be64_to_cpu(hptep->v); + if (cpu_has_feature(CPU_FTR_ARCH_300)) + hpte_v = hpte_new_to_old_v(hpte_v, be64_to_cpu(hptep->r)); if (unlikely(!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID))) { ret = -1; @@ -350,6 +359,8 @@ static long native_hpte_find(unsigned long vpn, int psize, int ssize) for (i = 0; i < HPTES_PER_GROUP; i++) { hptep = htab_address + slot; hpte_v = be64_to_cpu(hptep->v); + if (cpu_has_feature(CPU_FTR_ARCH_300)) + hpte_v = hpte_new_to_old_v(hpte_v, be64_to_cpu(hptep->r)); if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) /* HPTE matches */ @@ -409,6 +420,8 @@ static void native_hpte_invalidate(unsigned long slot, unsigned long vpn, want_v = hpte_encode_avpn(vpn, bpsize, ssize); native_lock_hpte(hptep); hpte_v = be64_to_cpu(hptep->v); + if (cpu_has_feature(CPU_FTR_ARCH_300)) + hpte_v = hpte_new_to_old_v(hpte_v, be64_to_cpu(hptep->r)); /* * We need to invalidate the TLB always because hpte_remove doesn't do @@ -467,6 +480,8 @@ static void native_hugepage_invalidate(unsigned long vsid, want_v = hpte_encode_avpn(vpn, psize, ssize); native_lock_hpte(hptep); hpte_v = be64_to_cpu(hptep->v); + if (cpu_has_feature(CPU_FTR_ARCH_300)) + hpte_v = hpte_new_to_old_v(hpte_v, be64_to_cpu(hptep->r)); /* Even if we miss, we need to invalidate the TLB */ if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID)) @@ -504,6 +519,10 @@ static void hpte_decode(struct hash_pte *hpte, unsigned long slot, /* Look at the 8 bit LP value */ unsigned int lp = (hpte_r >> LP_SHIFT) & ((1 << LP_BITS) - 1); + if (cpu_has_feature(CPU_FTR_ARCH_300)) { + hpte_v = hpte_new_to_old_v(hpte_v, hpte_r); + hpte_r = hpte_new_to_old_r(hpte_r); + } if (!(hpte_v & HPTE_V_LARGE)) { size = MMU_PAGE_4K; a_size = MMU_PAGE_4K; @@ -512,11 +531,7 @@ static void hpte_decode(struct hash_pte *hpte, unsigned long slot, a_size = hpte_page_sizes[lp] >> 4; } /* This works for all page sizes, and for 256M and 1T segments */ - if (cpu_has_feature(CPU_FTR_ARCH_300)) - *ssize = hpte_r >> HPTE_R_3_0_SSIZE_SHIFT; - else - *ssize = hpte_v >> HPTE_V_SSIZE_SHIFT; - + *ssize = hpte_v >> HPTE_V_SSIZE_SHIFT; shift = mmu_psize_defs[size].shift; avpn = (HPTE_V_AVPN_VAL(hpte_v) & ~mmu_psize_defs[size].avpnm); @@ -639,6 +654,9 @@ static void native_flush_hash_range(unsigned long number, int local) want_v = hpte_encode_avpn(vpn, psize, ssize); native_lock_hpte(hptep); hpte_v = be64_to_cpu(hptep->v); + if (cpu_has_feature(CPU_FTR_ARCH_300)) + hpte_v = hpte_new_to_old_v(hpte_v, + be64_to_cpu(hptep->r)); if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID)) native_unlock_hpte(hptep); diff --git a/arch/powerpc/mm/pgtable-radix.c b/arch/powerpc/mm/pgtable-radix.c index 688b54517655f1..9a25dce8787575 100644 --- a/arch/powerpc/mm/pgtable-radix.c +++ b/arch/powerpc/mm/pgtable-radix.c @@ -65,7 +65,7 @@ int radix__map_kernel_page(unsigned long ea, unsigned long pa, if (!pmdp) return -ENOMEM; if (map_page_size == PMD_SIZE) { - ptep = (pte_t *)pudp; + ptep = pmdp_ptep(pmdp); goto set_the_pte; } ptep = pte_alloc_kernel(pmdp, ea); @@ -90,7 +90,7 @@ int radix__map_kernel_page(unsigned long ea, unsigned long pa, } pmdp = pmd_offset(pudp, ea); if (map_page_size == PMD_SIZE) { - ptep = (pte_t *)pudp; + ptep = pmdp_ptep(pmdp); goto set_the_pte; } if (!pmd_present(*pmdp)) { @@ -159,7 +159,7 @@ static void __init radix_init_pgtable(void) * Allocate Partition table and process table for the * host. */ - BUILD_BUG_ON_MSG((PRTB_SIZE_SHIFT > 23), "Process table size too large."); + BUILD_BUG_ON_MSG((PRTB_SIZE_SHIFT > 36), "Process table size too large."); process_tb = early_alloc_pgtable(1UL << PRTB_SIZE_SHIFT); /* * Fill in the process table. @@ -181,7 +181,7 @@ static void __init radix_init_partition_table(void) rts_field = radix__get_tree_size(); - BUILD_BUG_ON_MSG((PATB_SIZE_SHIFT > 24), "Partition table size too large."); + BUILD_BUG_ON_MSG((PATB_SIZE_SHIFT > 36), "Partition table size too large."); partition_tb = early_alloc_pgtable(1UL << PATB_SIZE_SHIFT); partition_tb->patb0 = cpu_to_be64(rts_field | __pa(init_mm.pgd) | RADIX_PGD_INDEX_SIZE | PATB_HR); diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c index 3493cf4e045258..71697ff70879f2 100644 --- a/arch/powerpc/mm/tlb-radix.c +++ b/arch/powerpc/mm/tlb-radix.c @@ -50,9 +50,7 @@ static inline void _tlbiel_pid(unsigned long pid, unsigned long ric) for (set = 0; set < POWER9_TLB_SETS_RADIX ; set++) { __tlbiel_pid(pid, set, ric); } - if (cpu_has_feature(CPU_FTR_POWER9_DD1)) - asm volatile(PPC_INVALIDATE_ERAT : : :"memory"); - return; + asm volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory"); } static inline void _tlbie_pid(unsigned long pid, unsigned long ric) @@ -85,8 +83,6 @@ static inline void _tlbiel_va(unsigned long va, unsigned long pid, asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1) : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); asm volatile("ptesync": : :"memory"); - if (cpu_has_feature(CPU_FTR_POWER9_DD1)) - asm volatile(PPC_INVALIDATE_ERAT : : :"memory"); } static inline void _tlbie_va(unsigned long va, unsigned long pid, diff --git a/arch/powerpc/perf/power9-events-list.h b/arch/powerpc/perf/power9-events-list.h index 6447dc1c3d896c..929b56d47ad9bf 100644 --- a/arch/powerpc/perf/power9-events-list.h +++ b/arch/powerpc/perf/power9-events-list.h @@ -16,7 +16,7 @@ EVENT(PM_CYC, 0x0001e) EVENT(PM_ICT_NOSLOT_CYC, 0x100f8) EVENT(PM_CMPLU_STALL, 0x1e054) EVENT(PM_INST_CMPL, 0x00002) -EVENT(PM_BRU_CMPL, 0x40060) +EVENT(PM_BRU_CMPL, 0x10012) EVENT(PM_BR_MPRED_CMPL, 0x400f6) /* All L1 D cache load references counted at finish, gated by reject */ diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c index d4b33dd2d9e740..dcdfee0cd4f2f1 100644 --- a/arch/powerpc/platforms/powernv/pci-ioda.c +++ b/arch/powerpc/platforms/powernv/pci-ioda.c @@ -145,7 +145,7 @@ static struct pnv_ioda_pe *pnv_ioda_init_pe(struct pnv_phb *phb, int pe_no) */ rc = opal_pci_eeh_freeze_clear(phb->opal_id, pe_no, OPAL_EEH_ACTION_CLEAR_FREEZE_ALL); - if (rc != OPAL_SUCCESS) + if (rc != OPAL_SUCCESS && rc != OPAL_UNSUPPORTED) pr_warn("%s: Error %lld unfreezing PHB#%d-PE#%d\n", __func__, rc, phb->hose->global_number, pe_no); diff --git a/arch/powerpc/platforms/powernv/smp.c b/arch/powerpc/platforms/powernv/smp.c index c789258ae1e1ce..eec0e8d0454d11 100644 --- a/arch/powerpc/platforms/powernv/smp.c +++ b/arch/powerpc/platforms/powernv/smp.c @@ -155,8 +155,10 @@ static void pnv_smp_cpu_kill_self(void) wmask = SRR1_WAKEMASK_P8; idle_states = pnv_get_supported_cpuidle_states(); + /* We don't want to take decrementer interrupts while we are offline, - * so clear LPCR:PECE1. We keep PECE2 enabled. + * so clear LPCR:PECE1. We keep PECE2 (and LPCR_PECE_HVEE on P9) + * enabled as to let IPIs in. */ mtspr(SPRN_LPCR, mfspr(SPRN_LPCR) & ~(u64)LPCR_PECE1); @@ -206,8 +208,12 @@ static void pnv_smp_cpu_kill_self(void) * contains 0. */ if (((srr1 & wmask) == SRR1_WAKEEE) || + ((srr1 & wmask) == SRR1_WAKEHVI) || (local_paca->irq_happened & PACA_IRQ_EE)) { - icp_native_flush_interrupt(); + if (cpu_has_feature(CPU_FTR_ARCH_300)) + icp_opal_flush_interrupt(); + else + icp_native_flush_interrupt(); } else if ((srr1 & wmask) == SRR1_WAKEHDBELL) { unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER); asm volatile(PPC_MSGCLR(%0) : : "r" (msg)); @@ -221,6 +227,8 @@ static void pnv_smp_cpu_kill_self(void) if (srr1 && !generic_check_cpu_restart(cpu)) DBG("CPU%d Unexpected exit while offline !\n", cpu); } + + /* Re-enable decrementer interrupts */ mtspr(SPRN_LPCR, mfspr(SPRN_LPCR) | LPCR_PECE1); DBG("CPU%d coming online...\n", cpu); } diff --git a/arch/powerpc/platforms/ps3/htab.c b/arch/powerpc/platforms/ps3/htab.c index cb3c50328de875..cc2b281a3766c6 100644 --- a/arch/powerpc/platforms/ps3/htab.c +++ b/arch/powerpc/platforms/ps3/htab.c @@ -63,7 +63,7 @@ static long ps3_hpte_insert(unsigned long hpte_group, unsigned long vpn, vflags &= ~HPTE_V_SECONDARY; hpte_v = hpte_encode_v(vpn, psize, apsize, ssize) | vflags | HPTE_V_VALID; - hpte_r = hpte_encode_r(ps3_mm_phys_to_lpar(pa), psize, apsize, ssize) | rflags; + hpte_r = hpte_encode_r(ps3_mm_phys_to_lpar(pa), psize, apsize) | rflags; spin_lock_irqsave(&ps3_htab_lock, flags); diff --git a/arch/powerpc/platforms/pseries/lpar.c b/arch/powerpc/platforms/pseries/lpar.c index aa35245d8d6d33..f2c98f6c1c9c93 100644 --- a/arch/powerpc/platforms/pseries/lpar.c +++ b/arch/powerpc/platforms/pseries/lpar.c @@ -145,7 +145,7 @@ static long pSeries_lpar_hpte_insert(unsigned long hpte_group, hpte_group, vpn, pa, rflags, vflags, psize); hpte_v = hpte_encode_v(vpn, psize, apsize, ssize) | vflags | HPTE_V_VALID; - hpte_r = hpte_encode_r(pa, psize, apsize, ssize) | rflags; + hpte_r = hpte_encode_r(pa, psize, apsize) | rflags; if (!(vflags & HPTE_V_BOLTED)) pr_devel(" hpte_v=%016lx, hpte_r=%016lx\n", hpte_v, hpte_r); diff --git a/arch/powerpc/sysdev/xics/icp-opal.c b/arch/powerpc/sysdev/xics/icp-opal.c index d38e86fd5720f1..c96c0cb95d8748 100644 --- a/arch/powerpc/sysdev/xics/icp-opal.c +++ b/arch/powerpc/sysdev/xics/icp-opal.c @@ -20,6 +20,7 @@ #include #include #include +#include static void icp_opal_teardown_cpu(void) { @@ -39,7 +40,26 @@ static void icp_opal_flush_ipi(void) * Should we be flagging idle loop instead? * Or creating some task to be scheduled? */ - opal_int_eoi((0x00 << 24) | XICS_IPI); + if (opal_int_eoi((0x00 << 24) | XICS_IPI) > 0) + force_external_irq_replay(); +} + +static unsigned int icp_opal_get_xirr(void) +{ + unsigned int kvm_xirr; + __be32 hw_xirr; + int64_t rc; + + /* Handle an interrupt latched by KVM first */ + kvm_xirr = kvmppc_get_xics_latch(); + if (kvm_xirr) + return kvm_xirr; + + /* Then ask OPAL */ + rc = opal_int_get_xirr(&hw_xirr, false); + if (rc < 0) + return 0; + return be32_to_cpu(hw_xirr); } static unsigned int icp_opal_get_irq(void) @@ -47,12 +67,8 @@ static unsigned int icp_opal_get_irq(void) unsigned int xirr; unsigned int vec; unsigned int irq; - int64_t rc; - rc = opal_int_get_xirr(&xirr, false); - if (rc < 0) - return 0; - xirr = be32_to_cpu(xirr); + xirr = icp_opal_get_xirr(); vec = xirr & 0x00ffffff; if (vec == XICS_IRQ_SPURIOUS) return 0; @@ -67,7 +83,8 @@ static unsigned int icp_opal_get_irq(void) xics_mask_unknown_vec(vec); /* We might learn about it later, so EOI it */ - opal_int_eoi(xirr); + if (opal_int_eoi(xirr) > 0) + force_external_irq_replay(); return 0; } @@ -115,6 +132,35 @@ static irqreturn_t icp_opal_ipi_action(int irq, void *dev_id) return smp_ipi_demux(); } +/* + * Called when an interrupt is received on an off-line CPU to + * clear the interrupt, so that the CPU can go back to nap mode. + */ +void icp_opal_flush_interrupt(void) +{ + unsigned int xirr; + unsigned int vec; + + do { + xirr = icp_opal_get_xirr(); + vec = xirr & 0x00ffffff; + if (vec == XICS_IRQ_SPURIOUS) + break; + if (vec == XICS_IPI) { + /* Clear pending IPI */ + int cpu = smp_processor_id(); + kvmppc_set_host_ipi(cpu, 0); + opal_int_set_mfrr(get_hard_smp_processor_id(cpu), 0xff); + } else { + pr_err("XICS: hw interrupt 0x%x to offline cpu, " + "disabling\n", vec); + xics_mask_unknown_vec(vec); + } + + /* EOI the interrupt */ + } while (opal_int_eoi(xirr) > 0); +} + #endif /* CONFIG_SMP */ static const struct icp_ops icp_opal_ops = { diff --git a/arch/s390/crypto/prng.c b/arch/s390/crypto/prng.c index 9cc050f9536c10..1113389d0a39fe 100644 --- a/arch/s390/crypto/prng.c +++ b/arch/s390/crypto/prng.c @@ -507,8 +507,10 @@ static ssize_t prng_tdes_read(struct file *file, char __user *ubuf, prng_data->prngws.byte_counter += n; prng_data->prngws.reseed_counter += n; - if (copy_to_user(ubuf, prng_data->buf, chunk)) - return -EFAULT; + if (copy_to_user(ubuf, prng_data->buf, chunk)) { + ret = -EFAULT; + break; + } nbytes -= chunk; ret += chunk; diff --git a/arch/s390/kernel/ptrace.c b/arch/s390/kernel/ptrace.c index 9336e824e2db5e..fc2974b929c376 100644 --- a/arch/s390/kernel/ptrace.c +++ b/arch/s390/kernel/ptrace.c @@ -963,6 +963,11 @@ static int s390_fpregs_set(struct task_struct *target, if (target == current) save_fpu_regs(); + if (MACHINE_HAS_VX) + convert_vx_to_fp(fprs, target->thread.fpu.vxrs); + else + memcpy(&fprs, target->thread.fpu.fprs, sizeof(fprs)); + /* If setting FPC, must validate it first. */ if (count > 0 && pos < offsetof(s390_fp_regs, fprs)) { u32 ufpc[2] = { target->thread.fpu.fpc, 0 }; @@ -1067,6 +1072,9 @@ static int s390_vxrs_low_set(struct task_struct *target, if (target == current) save_fpu_regs(); + for (i = 0; i < __NUM_VXRS_LOW; i++) + vxrs[i] = *((__u64 *)(target->thread.fpu.vxrs + i) + 1); + rc = user_regset_copyin(&pos, &count, &kbuf, &ubuf, vxrs, 0, -1); if (rc == 0) for (i = 0; i < __NUM_VXRS_LOW; i++) diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index 7f7ba5f23f130f..d027f2eb3559f6 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -445,7 +445,7 @@ static void __init setup_resources(void) * part of the System RAM resource. */ if (crashk_res.end) { - memblock_add(crashk_res.start, resource_size(&crashk_res)); + memblock_add_node(crashk_res.start, resource_size(&crashk_res), 0); memblock_reserve(crashk_res.start, resource_size(&crashk_res)); insert_resource(&iomem_resource, &crashk_res); } diff --git a/arch/s390/kernel/topology.c b/arch/s390/kernel/topology.c index e959c02e0cac1f..8705ee66c0874f 100644 --- a/arch/s390/kernel/topology.c +++ b/arch/s390/kernel/topology.c @@ -448,6 +448,7 @@ static int __init s390_topology_init(void) struct sysinfo_15_1_x *info; int i; + set_sched_topology(s390_topology); if (!MACHINE_HAS_TOPOLOGY) return 0; tl_info = (struct sysinfo_15_1_x *)__get_free_page(GFP_KERNEL); @@ -460,7 +461,6 @@ static int __init s390_topology_init(void) alloc_masks(info, &socket_info, 1); alloc_masks(info, &book_info, 2); alloc_masks(info, &drawer_info, 3); - set_sched_topology(s390_topology); return 0; } early_initcall(s390_topology_init); diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 9c7a1ecfe6bd1f..47a1de77b18dc0 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -916,7 +916,7 @@ static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr) memcpy(&mach->fac_mask, kvm->arch.model.fac_mask, S390_ARCH_FAC_LIST_SIZE_BYTE); memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list, - S390_ARCH_FAC_LIST_SIZE_BYTE); + sizeof(S390_lowcore.stfle_fac_list)); if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach))) ret = -EFAULT; kfree(mach); @@ -1437,7 +1437,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) /* Populate the facility mask initially. */ memcpy(kvm->arch.model.fac_mask, S390_lowcore.stfle_fac_list, - S390_ARCH_FAC_LIST_SIZE_BYTE); + sizeof(S390_lowcore.stfle_fac_list)); for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) { if (i < kvm_s390_fac_list_mask_size()) kvm->arch.model.fac_mask[i] &= kvm_s390_fac_list_mask[i]; diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c index 7a1897c51c5495..d56ef26d46816b 100644 --- a/arch/s390/mm/pgtable.c +++ b/arch/s390/mm/pgtable.c @@ -202,7 +202,7 @@ static inline pgste_t ptep_xchg_start(struct mm_struct *mm, return pgste; } -static inline void ptep_xchg_commit(struct mm_struct *mm, +static inline pte_t ptep_xchg_commit(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pgste_t pgste, pte_t old, pte_t new) { @@ -220,6 +220,7 @@ static inline void ptep_xchg_commit(struct mm_struct *mm, } else { *ptep = new; } + return old; } pte_t ptep_xchg_direct(struct mm_struct *mm, unsigned long addr, @@ -231,7 +232,7 @@ pte_t ptep_xchg_direct(struct mm_struct *mm, unsigned long addr, preempt_disable(); pgste = ptep_xchg_start(mm, addr, ptep); old = ptep_flush_direct(mm, addr, ptep); - ptep_xchg_commit(mm, addr, ptep, pgste, old, new); + old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new); preempt_enable(); return old; } @@ -246,7 +247,7 @@ pte_t ptep_xchg_lazy(struct mm_struct *mm, unsigned long addr, preempt_disable(); pgste = ptep_xchg_start(mm, addr, ptep); old = ptep_flush_lazy(mm, addr, ptep); - ptep_xchg_commit(mm, addr, ptep, pgste, old, new); + old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new); preempt_enable(); return old; } diff --git a/arch/s390/pci/pci_dma.c b/arch/s390/pci/pci_dma.c index 6b2f72f523b91b..049e3860ac5483 100644 --- a/arch/s390/pci/pci_dma.c +++ b/arch/s390/pci/pci_dma.c @@ -419,6 +419,7 @@ static int __s390_dma_map_sg(struct device *dev, struct scatterlist *sg, size_t size, dma_addr_t *handle, enum dma_data_direction dir) { + unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; struct zpci_dev *zdev = to_zpci(to_pci_dev(dev)); dma_addr_t dma_addr_base, dma_addr; int flags = ZPCI_PTE_VALID; @@ -426,8 +427,7 @@ static int __s390_dma_map_sg(struct device *dev, struct scatterlist *sg, unsigned long pa = 0; int ret; - size = PAGE_ALIGN(size); - dma_addr_base = dma_alloc_address(dev, size >> PAGE_SHIFT); + dma_addr_base = dma_alloc_address(dev, nr_pages); if (dma_addr_base == DMA_ERROR_CODE) return -ENOMEM; @@ -436,26 +436,27 @@ static int __s390_dma_map_sg(struct device *dev, struct scatterlist *sg, flags |= ZPCI_TABLE_PROTECTED; for (s = sg; dma_addr < dma_addr_base + size; s = sg_next(s)) { - pa = page_to_phys(sg_page(s)) + s->offset; - ret = __dma_update_trans(zdev, pa, dma_addr, s->length, flags); + pa = page_to_phys(sg_page(s)); + ret = __dma_update_trans(zdev, pa, dma_addr, + s->offset + s->length, flags); if (ret) goto unmap; - dma_addr += s->length; + dma_addr += s->offset + s->length; } ret = __dma_purge_tlb(zdev, dma_addr_base, size, flags); if (ret) goto unmap; *handle = dma_addr_base; - atomic64_add(size >> PAGE_SHIFT, &zdev->mapped_pages); + atomic64_add(nr_pages, &zdev->mapped_pages); return ret; unmap: dma_update_trans(zdev, 0, dma_addr_base, dma_addr - dma_addr_base, ZPCI_PTE_INVALID); - dma_free_address(dev, dma_addr_base, size >> PAGE_SHIFT); + dma_free_address(dev, dma_addr_base, nr_pages); zpci_err("map error:\n"); zpci_err_dma(ret, pa); return ret; diff --git a/arch/tile/kernel/ptrace.c b/arch/tile/kernel/ptrace.c index d89b7011667cb4..e279572824b15e 100644 --- a/arch/tile/kernel/ptrace.c +++ b/arch/tile/kernel/ptrace.c @@ -111,7 +111,7 @@ static int tile_gpr_set(struct task_struct *target, const void *kbuf, const void __user *ubuf) { int ret; - struct pt_regs regs; + struct pt_regs regs = *task_pt_regs(target); ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, ®s, 0, sizeof(regs)); diff --git a/arch/x86/Makefile b/arch/x86/Makefile index 2d449337a36051..75725dc4df7a7a 100644 --- a/arch/x86/Makefile +++ b/arch/x86/Makefile @@ -97,6 +97,8 @@ else KBUILD_CFLAGS += $(call cc-option,-mno-80387) KBUILD_CFLAGS += $(call cc-option,-mno-fp-ret-in-387) + KBUILD_CFLAGS += -fno-pic + # Use -mpreferred-stack-boundary=3 if supported. KBUILD_CFLAGS += $(call cc-option,-mpreferred-stack-boundary=3) diff --git a/arch/x86/configs/i386_ranchu_defconfig b/arch/x86/configs/i386_ranchu_defconfig new file mode 100644 index 00000000000000..a1c83c4e78ae84 --- /dev/null +++ b/arch/x86/configs/i386_ranchu_defconfig @@ -0,0 +1,424 @@ +# CONFIG_64BIT is not set +# CONFIG_LOCALVERSION_AUTO is not set +CONFIG_POSIX_MQUEUE=y +CONFIG_AUDIT=y +CONFIG_NO_HZ=y +CONFIG_HIGH_RES_TIMERS=y +CONFIG_BSD_PROCESS_ACCT=y +CONFIG_TASKSTATS=y +CONFIG_TASK_DELAY_ACCT=y +CONFIG_TASK_XACCT=y +CONFIG_TASK_IO_ACCOUNTING=y +CONFIG_CGROUPS=y +CONFIG_CGROUP_DEBUG=y +CONFIG_CGROUP_FREEZER=y +CONFIG_CGROUP_CPUACCT=y +CONFIG_CGROUP_SCHED=y +CONFIG_RT_GROUP_SCHED=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_CC_OPTIMIZE_FOR_SIZE=y +CONFIG_SYSCTL_SYSCALL=y +CONFIG_KALLSYMS_ALL=y +CONFIG_EMBEDDED=y +# CONFIG_COMPAT_BRK is not set +CONFIG_ARCH_MMAP_RND_BITS=16 +CONFIG_PARTITION_ADVANCED=y +CONFIG_OSF_PARTITION=y +CONFIG_AMIGA_PARTITION=y +CONFIG_MAC_PARTITION=y +CONFIG_BSD_DISKLABEL=y +CONFIG_MINIX_SUBPARTITION=y +CONFIG_SOLARIS_X86_PARTITION=y +CONFIG_UNIXWARE_DISKLABEL=y +CONFIG_SGI_PARTITION=y +CONFIG_SUN_PARTITION=y +CONFIG_KARMA_PARTITION=y +CONFIG_SMP=y +CONFIG_X86_BIGSMP=y +CONFIG_MCORE2=y +CONFIG_X86_GENERIC=y +CONFIG_HPET_TIMER=y +CONFIG_NR_CPUS=512 +CONFIG_PREEMPT=y +# CONFIG_X86_MCE is not set +CONFIG_X86_REBOOTFIXUPS=y +CONFIG_X86_MSR=y +CONFIG_X86_CPUID=y +CONFIG_KSM=y +CONFIG_CMA=y +# CONFIG_MTRR_SANITIZER is not set +CONFIG_EFI=y +CONFIG_EFI_STUB=y +CONFIG_HZ_100=y +CONFIG_PHYSICAL_START=0x100000 +CONFIG_PM_AUTOSLEEP=y +CONFIG_PM_WAKELOCKS=y +CONFIG_PM_WAKELOCKS_LIMIT=0 +# CONFIG_PM_WAKELOCKS_GC is not set +CONFIG_PM_DEBUG=y +CONFIG_CPU_FREQ=y +# CONFIG_CPU_FREQ_STAT is not set +CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND=y +CONFIG_CPU_FREQ_GOV_USERSPACE=y +CONFIG_PCIEPORTBUS=y +# CONFIG_PCIEASPM is not set +CONFIG_PCCARD=y +CONFIG_YENTA=y +CONFIG_HOTPLUG_PCI=y +# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set +CONFIG_BINFMT_MISC=y +CONFIG_NET=y +CONFIG_PACKET=y +CONFIG_UNIX=y +CONFIG_XFRM_USER=y +CONFIG_NET_KEY=y +CONFIG_INET=y +CONFIG_IP_MULTICAST=y +CONFIG_IP_ADVANCED_ROUTER=y +CONFIG_IP_MULTIPLE_TABLES=y +CONFIG_IP_ROUTE_MULTIPATH=y +CONFIG_IP_ROUTE_VERBOSE=y +CONFIG_IP_PNP=y +CONFIG_IP_PNP_DHCP=y +CONFIG_IP_PNP_BOOTP=y +CONFIG_IP_PNP_RARP=y +CONFIG_IP_MROUTE=y +CONFIG_IP_PIMSM_V1=y +CONFIG_IP_PIMSM_V2=y +CONFIG_SYN_COOKIES=y +CONFIG_INET_ESP=y +# CONFIG_INET_XFRM_MODE_BEET is not set +# CONFIG_INET_LRO is not set +CONFIG_INET_DIAG_DESTROY=y +CONFIG_IPV6_ROUTER_PREF=y +CONFIG_IPV6_ROUTE_INFO=y +CONFIG_IPV6_OPTIMISTIC_DAD=y +CONFIG_INET6_AH=y +CONFIG_INET6_ESP=y +CONFIG_INET6_IPCOMP=y +CONFIG_IPV6_MIP6=y +CONFIG_IPV6_MULTIPLE_TABLES=y +CONFIG_NETLABEL=y +CONFIG_NETFILTER=y +CONFIG_NF_CONNTRACK=y +CONFIG_NF_CONNTRACK_SECMARK=y +CONFIG_NF_CONNTRACK_EVENTS=y +CONFIG_NF_CT_PROTO_DCCP=y +CONFIG_NF_CT_PROTO_SCTP=y +CONFIG_NF_CT_PROTO_UDPLITE=y +CONFIG_NF_CONNTRACK_AMANDA=y +CONFIG_NF_CONNTRACK_FTP=y +CONFIG_NF_CONNTRACK_H323=y +CONFIG_NF_CONNTRACK_IRC=y +CONFIG_NF_CONNTRACK_NETBIOS_NS=y +CONFIG_NF_CONNTRACK_PPTP=y +CONFIG_NF_CONNTRACK_SANE=y +CONFIG_NF_CONNTRACK_TFTP=y +CONFIG_NF_CT_NETLINK=y +CONFIG_NETFILTER_XT_TARGET_CLASSIFY=y +CONFIG_NETFILTER_XT_TARGET_CONNMARK=y +CONFIG_NETFILTER_XT_TARGET_CONNSECMARK=y +CONFIG_NETFILTER_XT_TARGET_IDLETIMER=y +CONFIG_NETFILTER_XT_TARGET_MARK=y +CONFIG_NETFILTER_XT_TARGET_NFLOG=y +CONFIG_NETFILTER_XT_TARGET_NFQUEUE=y +CONFIG_NETFILTER_XT_TARGET_TPROXY=y +CONFIG_NETFILTER_XT_TARGET_TRACE=y +CONFIG_NETFILTER_XT_TARGET_SECMARK=y +CONFIG_NETFILTER_XT_TARGET_TCPMSS=y +CONFIG_NETFILTER_XT_MATCH_COMMENT=y +CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=y +CONFIG_NETFILTER_XT_MATCH_CONNMARK=y +CONFIG_NETFILTER_XT_MATCH_CONNTRACK=y +CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=y +CONFIG_NETFILTER_XT_MATCH_HELPER=y +CONFIG_NETFILTER_XT_MATCH_IPRANGE=y +CONFIG_NETFILTER_XT_MATCH_LENGTH=y +CONFIG_NETFILTER_XT_MATCH_LIMIT=y +CONFIG_NETFILTER_XT_MATCH_MAC=y +CONFIG_NETFILTER_XT_MATCH_MARK=y +CONFIG_NETFILTER_XT_MATCH_POLICY=y +CONFIG_NETFILTER_XT_MATCH_PKTTYPE=y +CONFIG_NETFILTER_XT_MATCH_QTAGUID=y +CONFIG_NETFILTER_XT_MATCH_QUOTA=y +CONFIG_NETFILTER_XT_MATCH_QUOTA2=y +CONFIG_NETFILTER_XT_MATCH_SOCKET=y +CONFIG_NETFILTER_XT_MATCH_STATE=y +CONFIG_NETFILTER_XT_MATCH_STATISTIC=y +CONFIG_NETFILTER_XT_MATCH_STRING=y +CONFIG_NETFILTER_XT_MATCH_TIME=y +CONFIG_NETFILTER_XT_MATCH_U32=y +CONFIG_NF_CONNTRACK_IPV4=y +CONFIG_IP_NF_IPTABLES=y +CONFIG_IP_NF_MATCH_AH=y +CONFIG_IP_NF_MATCH_ECN=y +CONFIG_IP_NF_MATCH_TTL=y +CONFIG_IP_NF_FILTER=y +CONFIG_IP_NF_TARGET_REJECT=y +CONFIG_IP_NF_MANGLE=y +CONFIG_IP_NF_RAW=y +CONFIG_IP_NF_SECURITY=y +CONFIG_IP_NF_ARPTABLES=y +CONFIG_IP_NF_ARPFILTER=y +CONFIG_IP_NF_ARP_MANGLE=y +CONFIG_NF_CONNTRACK_IPV6=y +CONFIG_IP6_NF_IPTABLES=y +CONFIG_IP6_NF_FILTER=y +CONFIG_IP6_NF_TARGET_REJECT=y +CONFIG_IP6_NF_MANGLE=y +CONFIG_IP6_NF_RAW=y +CONFIG_NET_SCHED=y +CONFIG_NET_SCH_HTB=y +CONFIG_NET_CLS_U32=y +CONFIG_NET_EMATCH=y +CONFIG_NET_EMATCH_U32=y +CONFIG_NET_CLS_ACT=y +CONFIG_CFG80211=y +CONFIG_MAC80211=y +CONFIG_MAC80211_LEDS=y +CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" +CONFIG_DMA_CMA=y +CONFIG_CMA_SIZE_MBYTES=16 +CONFIG_CONNECTOR=y +CONFIG_BLK_DEV_LOOP=y +CONFIG_BLK_DEV_RAM=y +CONFIG_BLK_DEV_RAM_SIZE=8192 +CONFIG_VIRTIO_BLK=y +CONFIG_BLK_DEV_SD=y +CONFIG_BLK_DEV_SR=y +CONFIG_BLK_DEV_SR_VENDOR=y +CONFIG_CHR_DEV_SG=y +CONFIG_SCSI_CONSTANTS=y +CONFIG_SCSI_SPI_ATTRS=y +CONFIG_SCSI_ISCSI_ATTRS=y +# CONFIG_SCSI_LOWLEVEL is not set +CONFIG_ATA=y +CONFIG_SATA_AHCI=y +CONFIG_ATA_PIIX=y +CONFIG_PATA_AMD=y +CONFIG_PATA_OLDPIIX=y +CONFIG_PATA_SCH=y +CONFIG_PATA_MPIIX=y +CONFIG_ATA_GENERIC=y +CONFIG_MD=y +CONFIG_BLK_DEV_MD=y +CONFIG_BLK_DEV_DM=y +CONFIG_DM_DEBUG=y +CONFIG_DM_CRYPT=y +CONFIG_DM_MIRROR=y +CONFIG_DM_ZERO=y +CONFIG_DM_UEVENT=y +CONFIG_DM_VERITY=y +CONFIG_DM_VERITY_FEC=y +CONFIG_NETDEVICES=y +CONFIG_NETCONSOLE=y +CONFIG_TUN=y +CONFIG_VIRTIO_NET=y +CONFIG_BNX2=y +CONFIG_TIGON3=y +CONFIG_NET_TULIP=y +CONFIG_E100=y +CONFIG_E1000=y +CONFIG_E1000E=y +CONFIG_SKY2=y +CONFIG_NE2K_PCI=y +CONFIG_FORCEDETH=y +CONFIG_8139TOO=y +# CONFIG_8139TOO_PIO is not set +CONFIG_R8169=y +CONFIG_FDDI=y +CONFIG_PPP=y +CONFIG_PPP_BSDCOMP=y +CONFIG_PPP_DEFLATE=y +CONFIG_PPP_MPPE=y +CONFIG_PPPOLAC=y +CONFIG_PPPOPNS=y +CONFIG_USB_USBNET=y +CONFIG_INPUT_POLLDEV=y +# CONFIG_INPUT_MOUSEDEV_PSAUX is not set +CONFIG_INPUT_EVDEV=y +CONFIG_INPUT_KEYRESET=y +# CONFIG_KEYBOARD_ATKBD is not set +CONFIG_KEYBOARD_GOLDFISH_EVENTS=y +# CONFIG_INPUT_MOUSE is not set +CONFIG_INPUT_JOYSTICK=y +CONFIG_JOYSTICK_XPAD=y +CONFIG_JOYSTICK_XPAD_FF=y +CONFIG_JOYSTICK_XPAD_LEDS=y +CONFIG_INPUT_TABLET=y +CONFIG_TABLET_USB_ACECAD=y +CONFIG_TABLET_USB_AIPTEK=y +CONFIG_TABLET_USB_GTCO=y +CONFIG_TABLET_USB_HANWANG=y +CONFIG_TABLET_USB_KBTAB=y +CONFIG_INPUT_TOUCHSCREEN=y +CONFIG_INPUT_MISC=y +CONFIG_INPUT_KEYCHORD=y +CONFIG_INPUT_UINPUT=y +CONFIG_INPUT_GPIO=y +# CONFIG_SERIO is not set +# CONFIG_VT is not set +# CONFIG_LEGACY_PTYS is not set +CONFIG_SERIAL_NONSTANDARD=y +# CONFIG_DEVMEM is not set +# CONFIG_DEVKMEM is not set +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_8250_CONSOLE=y +CONFIG_VIRTIO_CONSOLE=y +CONFIG_NVRAM=y +CONFIG_I2C_I801=y +CONFIG_BATTERY_GOLDFISH=y +CONFIG_WATCHDOG=y +CONFIG_MEDIA_SUPPORT=y +CONFIG_AGP=y +CONFIG_AGP_AMD64=y +CONFIG_AGP_INTEL=y +CONFIG_DRM=y +CONFIG_FB_MODE_HELPERS=y +CONFIG_FB_TILEBLITTING=y +CONFIG_FB_EFI=y +CONFIG_FB_GOLDFISH=y +CONFIG_BACKLIGHT_LCD_SUPPORT=y +# CONFIG_LCD_CLASS_DEVICE is not set +CONFIG_SOUND=y +CONFIG_SND=y +CONFIG_HIDRAW=y +CONFIG_UHID=y +CONFIG_HID_A4TECH=y +CONFIG_HID_ACRUX=y +CONFIG_HID_ACRUX_FF=y +CONFIG_HID_APPLE=y +CONFIG_HID_BELKIN=y +CONFIG_HID_CHERRY=y +CONFIG_HID_CHICONY=y +CONFIG_HID_PRODIKEYS=y +CONFIG_HID_CYPRESS=y +CONFIG_HID_DRAGONRISE=y +CONFIG_DRAGONRISE_FF=y +CONFIG_HID_EMS_FF=y +CONFIG_HID_ELECOM=y +CONFIG_HID_EZKEY=y +CONFIG_HID_HOLTEK=y +CONFIG_HID_KEYTOUCH=y +CONFIG_HID_KYE=y +CONFIG_HID_UCLOGIC=y +CONFIG_HID_WALTOP=y +CONFIG_HID_GYRATION=y +CONFIG_HID_TWINHAN=y +CONFIG_HID_KENSINGTON=y +CONFIG_HID_LCPOWER=y +CONFIG_HID_LOGITECH=y +CONFIG_HID_LOGITECH_DJ=y +CONFIG_LOGITECH_FF=y +CONFIG_LOGIRUMBLEPAD2_FF=y +CONFIG_LOGIG940_FF=y +CONFIG_HID_MAGICMOUSE=y +CONFIG_HID_MICROSOFT=y +CONFIG_HID_MONTEREY=y +CONFIG_HID_MULTITOUCH=y +CONFIG_HID_NTRIG=y +CONFIG_HID_ORTEK=y +CONFIG_HID_PANTHERLORD=y +CONFIG_PANTHERLORD_FF=y +CONFIG_HID_PETALYNX=y +CONFIG_HID_PICOLCD=y +CONFIG_HID_PRIMAX=y +CONFIG_HID_ROCCAT=y +CONFIG_HID_SAITEK=y +CONFIG_HID_SAMSUNG=y +CONFIG_HID_SONY=y +CONFIG_HID_SPEEDLINK=y +CONFIG_HID_SUNPLUS=y +CONFIG_HID_GREENASIA=y +CONFIG_GREENASIA_FF=y +CONFIG_HID_SMARTJOYPLUS=y +CONFIG_SMARTJOYPLUS_FF=y +CONFIG_HID_TIVO=y +CONFIG_HID_TOPSEED=y +CONFIG_HID_THRUSTMASTER=y +CONFIG_HID_WACOM=y +CONFIG_HID_WIIMOTE=y +CONFIG_HID_ZEROPLUS=y +CONFIG_HID_ZYDACRON=y +CONFIG_HID_PID=y +CONFIG_USB_HIDDEV=y +CONFIG_USB_ANNOUNCE_NEW_DEVICES=y +CONFIG_USB_MON=y +CONFIG_USB_EHCI_HCD=y +# CONFIG_USB_EHCI_TT_NEWSCHED is not set +CONFIG_USB_OHCI_HCD=y +CONFIG_USB_UHCI_HCD=y +CONFIG_USB_PRINTER=y +CONFIG_USB_STORAGE=y +CONFIG_USB_OTG_WAKELOCK=y +CONFIG_EDAC=y +CONFIG_RTC_CLASS=y +# CONFIG_RTC_HCTOSYS is not set +CONFIG_DMADEVICES=y +CONFIG_VIRTIO_PCI=y +CONFIG_STAGING=y +CONFIG_ASHMEM=y +CONFIG_ANDROID_LOW_MEMORY_KILLER=y +CONFIG_SYNC=y +CONFIG_SW_SYNC=y +CONFIG_SYNC_FILE=y +CONFIG_ION=y +CONFIG_GOLDFISH_AUDIO=y +CONFIG_SND_HDA_INTEL=y +CONFIG_GOLDFISH=y +CONFIG_GOLDFISH_PIPE=y +CONFIG_GOLDFISH_SYNC=y +CONFIG_ANDROID=y +CONFIG_ANDROID_BINDER_IPC=y +CONFIG_ISCSI_IBFT_FIND=y +CONFIG_EXT4_FS=y +CONFIG_EXT4_FS_SECURITY=y +CONFIG_QUOTA=y +CONFIG_QUOTA_NETLINK_INTERFACE=y +# CONFIG_PRINT_QUOTA_WARNING is not set +CONFIG_FUSE_FS=y +CONFIG_ISO9660_FS=y +CONFIG_JOLIET=y +CONFIG_ZISOFS=y +CONFIG_MSDOS_FS=y +CONFIG_VFAT_FS=y +CONFIG_PROC_KCORE=y +CONFIG_TMPFS=y +CONFIG_TMPFS_POSIX_ACL=y +CONFIG_HUGETLBFS=y +CONFIG_PSTORE=y +CONFIG_PSTORE_CONSOLE=y +CONFIG_PSTORE_RAM=y +# CONFIG_NETWORK_FILESYSTEMS is not set +CONFIG_NLS_DEFAULT="utf8" +CONFIG_NLS_CODEPAGE_437=y +CONFIG_NLS_ASCII=y +CONFIG_NLS_ISO8859_1=y +CONFIG_NLS_UTF8=y +CONFIG_PRINTK_TIME=y +CONFIG_DEBUG_INFO=y +# CONFIG_ENABLE_WARN_DEPRECATED is not set +# CONFIG_ENABLE_MUST_CHECK is not set +CONFIG_FRAME_WARN=2048 +# CONFIG_UNUSED_SYMBOLS is not set +CONFIG_MAGIC_SYSRQ=y +CONFIG_DEBUG_MEMORY_INIT=y +CONFIG_PANIC_TIMEOUT=5 +CONFIG_SCHEDSTATS=y +CONFIG_TIMER_STATS=y +CONFIG_SCHED_TRACER=y +CONFIG_BLK_DEV_IO_TRACE=y +CONFIG_PROVIDE_OHCI1394_DMA_INIT=y +CONFIG_KEYS=y +CONFIG_SECURITY=y +CONFIG_SECURITY_NETWORK=y +CONFIG_SECURITY_SELINUX=y +CONFIG_CRYPTO_AES_586=y +CONFIG_CRYPTO_TWOFISH=y +CONFIG_ASYMMETRIC_KEY_TYPE=y +CONFIG_ASYMMETRIC_PUBLIC_KEY_SUBTYPE=y +CONFIG_X509_CERTIFICATE_PARSER=y +CONFIG_PKCS7_MESSAGE_PARSER=y +CONFIG_PKCS7_TEST_KEY=y +# CONFIG_VIRTUALIZATION is not set +CONFIG_CRC_T10DIF=y diff --git a/arch/x86/configs/x86_64_ranchu_defconfig b/arch/x86/configs/x86_64_ranchu_defconfig new file mode 100644 index 00000000000000..d50434f501fbbc --- /dev/null +++ b/arch/x86/configs/x86_64_ranchu_defconfig @@ -0,0 +1,419 @@ +# CONFIG_LOCALVERSION_AUTO is not set +CONFIG_POSIX_MQUEUE=y +CONFIG_AUDIT=y +CONFIG_NO_HZ=y +CONFIG_HIGH_RES_TIMERS=y +CONFIG_BSD_PROCESS_ACCT=y +CONFIG_TASKSTATS=y +CONFIG_TASK_DELAY_ACCT=y +CONFIG_TASK_XACCT=y +CONFIG_TASK_IO_ACCOUNTING=y +CONFIG_CGROUPS=y +CONFIG_CGROUP_DEBUG=y +CONFIG_CGROUP_FREEZER=y +CONFIG_CGROUP_CPUACCT=y +CONFIG_CGROUP_SCHED=y +CONFIG_RT_GROUP_SCHED=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_CC_OPTIMIZE_FOR_SIZE=y +CONFIG_SYSCTL_SYSCALL=y +CONFIG_KALLSYMS_ALL=y +CONFIG_EMBEDDED=y +# CONFIG_COMPAT_BRK is not set +CONFIG_ARCH_MMAP_RND_BITS=32 +CONFIG_ARCH_MMAP_RND_COMPAT_BITS=16 +CONFIG_PARTITION_ADVANCED=y +CONFIG_OSF_PARTITION=y +CONFIG_AMIGA_PARTITION=y +CONFIG_MAC_PARTITION=y +CONFIG_BSD_DISKLABEL=y +CONFIG_MINIX_SUBPARTITION=y +CONFIG_SOLARIS_X86_PARTITION=y +CONFIG_UNIXWARE_DISKLABEL=y +CONFIG_SGI_PARTITION=y +CONFIG_SUN_PARTITION=y +CONFIG_KARMA_PARTITION=y +CONFIG_SMP=y +CONFIG_MCORE2=y +CONFIG_MAXSMP=y +CONFIG_PREEMPT=y +# CONFIG_X86_MCE is not set +CONFIG_X86_MSR=y +CONFIG_X86_CPUID=y +CONFIG_KSM=y +CONFIG_CMA=y +# CONFIG_MTRR_SANITIZER is not set +CONFIG_EFI=y +CONFIG_EFI_STUB=y +CONFIG_HZ_100=y +CONFIG_PHYSICAL_START=0x100000 +CONFIG_PM_AUTOSLEEP=y +CONFIG_PM_WAKELOCKS=y +CONFIG_PM_WAKELOCKS_LIMIT=0 +# CONFIG_PM_WAKELOCKS_GC is not set +CONFIG_PM_DEBUG=y +CONFIG_CPU_FREQ=y +# CONFIG_CPU_FREQ_STAT is not set +CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND=y +CONFIG_CPU_FREQ_GOV_USERSPACE=y +CONFIG_PCI_MMCONFIG=y +CONFIG_PCIEPORTBUS=y +# CONFIG_PCIEASPM is not set +CONFIG_PCCARD=y +CONFIG_YENTA=y +CONFIG_HOTPLUG_PCI=y +# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set +CONFIG_BINFMT_MISC=y +CONFIG_IA32_EMULATION=y +CONFIG_NET=y +CONFIG_PACKET=y +CONFIG_UNIX=y +CONFIG_XFRM_USER=y +CONFIG_NET_KEY=y +CONFIG_INET=y +CONFIG_IP_MULTICAST=y +CONFIG_IP_ADVANCED_ROUTER=y +CONFIG_IP_MULTIPLE_TABLES=y +CONFIG_IP_ROUTE_MULTIPATH=y +CONFIG_IP_ROUTE_VERBOSE=y +CONFIG_IP_PNP=y +CONFIG_IP_PNP_DHCP=y +CONFIG_IP_PNP_BOOTP=y +CONFIG_IP_PNP_RARP=y +CONFIG_IP_MROUTE=y +CONFIG_IP_PIMSM_V1=y +CONFIG_IP_PIMSM_V2=y +CONFIG_SYN_COOKIES=y +CONFIG_INET_ESP=y +# CONFIG_INET_XFRM_MODE_BEET is not set +# CONFIG_INET_LRO is not set +CONFIG_INET_DIAG_DESTROY=y +CONFIG_IPV6_ROUTER_PREF=y +CONFIG_IPV6_ROUTE_INFO=y +CONFIG_IPV6_OPTIMISTIC_DAD=y +CONFIG_INET6_AH=y +CONFIG_INET6_ESP=y +CONFIG_INET6_IPCOMP=y +CONFIG_IPV6_MIP6=y +CONFIG_IPV6_MULTIPLE_TABLES=y +CONFIG_NETLABEL=y +CONFIG_NETFILTER=y +CONFIG_NF_CONNTRACK=y +CONFIG_NF_CONNTRACK_SECMARK=y +CONFIG_NF_CONNTRACK_EVENTS=y +CONFIG_NF_CT_PROTO_DCCP=y +CONFIG_NF_CT_PROTO_SCTP=y +CONFIG_NF_CT_PROTO_UDPLITE=y +CONFIG_NF_CONNTRACK_AMANDA=y +CONFIG_NF_CONNTRACK_FTP=y +CONFIG_NF_CONNTRACK_H323=y +CONFIG_NF_CONNTRACK_IRC=y +CONFIG_NF_CONNTRACK_NETBIOS_NS=y +CONFIG_NF_CONNTRACK_PPTP=y +CONFIG_NF_CONNTRACK_SANE=y +CONFIG_NF_CONNTRACK_TFTP=y +CONFIG_NF_CT_NETLINK=y +CONFIG_NETFILTER_XT_TARGET_CLASSIFY=y +CONFIG_NETFILTER_XT_TARGET_CONNMARK=y +CONFIG_NETFILTER_XT_TARGET_CONNSECMARK=y +CONFIG_NETFILTER_XT_TARGET_IDLETIMER=y +CONFIG_NETFILTER_XT_TARGET_MARK=y +CONFIG_NETFILTER_XT_TARGET_NFLOG=y +CONFIG_NETFILTER_XT_TARGET_NFQUEUE=y +CONFIG_NETFILTER_XT_TARGET_TPROXY=y +CONFIG_NETFILTER_XT_TARGET_TRACE=y +CONFIG_NETFILTER_XT_TARGET_SECMARK=y +CONFIG_NETFILTER_XT_TARGET_TCPMSS=y +CONFIG_NETFILTER_XT_MATCH_COMMENT=y +CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=y +CONFIG_NETFILTER_XT_MATCH_CONNMARK=y +CONFIG_NETFILTER_XT_MATCH_CONNTRACK=y +CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=y +CONFIG_NETFILTER_XT_MATCH_HELPER=y +CONFIG_NETFILTER_XT_MATCH_IPRANGE=y +CONFIG_NETFILTER_XT_MATCH_LENGTH=y +CONFIG_NETFILTER_XT_MATCH_LIMIT=y +CONFIG_NETFILTER_XT_MATCH_MAC=y +CONFIG_NETFILTER_XT_MATCH_MARK=y +CONFIG_NETFILTER_XT_MATCH_POLICY=y +CONFIG_NETFILTER_XT_MATCH_PKTTYPE=y +CONFIG_NETFILTER_XT_MATCH_QTAGUID=y +CONFIG_NETFILTER_XT_MATCH_QUOTA=y +CONFIG_NETFILTER_XT_MATCH_QUOTA2=y +CONFIG_NETFILTER_XT_MATCH_SOCKET=y +CONFIG_NETFILTER_XT_MATCH_STATE=y +CONFIG_NETFILTER_XT_MATCH_STATISTIC=y +CONFIG_NETFILTER_XT_MATCH_STRING=y +CONFIG_NETFILTER_XT_MATCH_TIME=y +CONFIG_NETFILTER_XT_MATCH_U32=y +CONFIG_NF_CONNTRACK_IPV4=y +CONFIG_IP_NF_IPTABLES=y +CONFIG_IP_NF_MATCH_AH=y +CONFIG_IP_NF_MATCH_ECN=y +CONFIG_IP_NF_MATCH_TTL=y +CONFIG_IP_NF_FILTER=y +CONFIG_IP_NF_TARGET_REJECT=y +CONFIG_IP_NF_MANGLE=y +CONFIG_IP_NF_RAW=y +CONFIG_IP_NF_SECURITY=y +CONFIG_IP_NF_ARPTABLES=y +CONFIG_IP_NF_ARPFILTER=y +CONFIG_IP_NF_ARP_MANGLE=y +CONFIG_NF_CONNTRACK_IPV6=y +CONFIG_IP6_NF_IPTABLES=y +CONFIG_IP6_NF_FILTER=y +CONFIG_IP6_NF_TARGET_REJECT=y +CONFIG_IP6_NF_MANGLE=y +CONFIG_IP6_NF_RAW=y +CONFIG_NET_SCHED=y +CONFIG_NET_SCH_HTB=y +CONFIG_NET_CLS_U32=y +CONFIG_NET_EMATCH=y +CONFIG_NET_EMATCH_U32=y +CONFIG_NET_CLS_ACT=y +CONFIG_CFG80211=y +CONFIG_MAC80211=y +CONFIG_MAC80211_LEDS=y +CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" +CONFIG_DMA_CMA=y +CONFIG_CONNECTOR=y +CONFIG_BLK_DEV_LOOP=y +CONFIG_BLK_DEV_RAM=y +CONFIG_BLK_DEV_RAM_SIZE=8192 +CONFIG_VIRTIO_BLK=y +CONFIG_BLK_DEV_SD=y +CONFIG_BLK_DEV_SR=y +CONFIG_BLK_DEV_SR_VENDOR=y +CONFIG_CHR_DEV_SG=y +CONFIG_SCSI_CONSTANTS=y +CONFIG_SCSI_SPI_ATTRS=y +CONFIG_SCSI_ISCSI_ATTRS=y +# CONFIG_SCSI_LOWLEVEL is not set +CONFIG_ATA=y +CONFIG_SATA_AHCI=y +CONFIG_ATA_PIIX=y +CONFIG_PATA_AMD=y +CONFIG_PATA_OLDPIIX=y +CONFIG_PATA_SCH=y +CONFIG_PATA_MPIIX=y +CONFIG_ATA_GENERIC=y +CONFIG_MD=y +CONFIG_BLK_DEV_MD=y +CONFIG_BLK_DEV_DM=y +CONFIG_DM_DEBUG=y +CONFIG_DM_CRYPT=y +CONFIG_DM_MIRROR=y +CONFIG_DM_ZERO=y +CONFIG_DM_UEVENT=y +CONFIG_DM_VERITY=y +CONFIG_DM_VERITY_FEC=y +CONFIG_NETDEVICES=y +CONFIG_NETCONSOLE=y +CONFIG_TUN=y +CONFIG_VIRTIO_NET=y +CONFIG_BNX2=y +CONFIG_TIGON3=y +CONFIG_NET_TULIP=y +CONFIG_E100=y +CONFIG_E1000=y +CONFIG_E1000E=y +CONFIG_SKY2=y +CONFIG_NE2K_PCI=y +CONFIG_FORCEDETH=y +CONFIG_8139TOO=y +# CONFIG_8139TOO_PIO is not set +CONFIG_R8169=y +CONFIG_FDDI=y +CONFIG_PPP=y +CONFIG_PPP_BSDCOMP=y +CONFIG_PPP_DEFLATE=y +CONFIG_PPP_MPPE=y +CONFIG_PPPOLAC=y +CONFIG_PPPOPNS=y +CONFIG_USB_USBNET=y +CONFIG_INPUT_POLLDEV=y +# CONFIG_INPUT_MOUSEDEV_PSAUX is not set +CONFIG_INPUT_EVDEV=y +CONFIG_INPUT_KEYRESET=y +# CONFIG_KEYBOARD_ATKBD is not set +CONFIG_KEYBOARD_GOLDFISH_EVENTS=y +# CONFIG_INPUT_MOUSE is not set +CONFIG_INPUT_JOYSTICK=y +CONFIG_JOYSTICK_XPAD=y +CONFIG_JOYSTICK_XPAD_FF=y +CONFIG_JOYSTICK_XPAD_LEDS=y +CONFIG_INPUT_TABLET=y +CONFIG_TABLET_USB_ACECAD=y +CONFIG_TABLET_USB_AIPTEK=y +CONFIG_TABLET_USB_GTCO=y +CONFIG_TABLET_USB_HANWANG=y +CONFIG_TABLET_USB_KBTAB=y +CONFIG_INPUT_TOUCHSCREEN=y +CONFIG_INPUT_MISC=y +CONFIG_INPUT_KEYCHORD=y +CONFIG_INPUT_UINPUT=y +CONFIG_INPUT_GPIO=y +# CONFIG_SERIO is not set +# CONFIG_VT is not set +# CONFIG_LEGACY_PTYS is not set +CONFIG_SERIAL_NONSTANDARD=y +# CONFIG_DEVMEM is not set +# CONFIG_DEVKMEM is not set +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_8250_CONSOLE=y +CONFIG_VIRTIO_CONSOLE=y +CONFIG_NVRAM=y +CONFIG_I2C_I801=y +CONFIG_BATTERY_GOLDFISH=y +CONFIG_WATCHDOG=y +CONFIG_MEDIA_SUPPORT=y +CONFIG_AGP=y +CONFIG_AGP_AMD64=y +CONFIG_AGP_INTEL=y +CONFIG_DRM=y +CONFIG_FB_MODE_HELPERS=y +CONFIG_FB_TILEBLITTING=y +CONFIG_FB_EFI=y +CONFIG_FB_GOLDFISH=y +CONFIG_BACKLIGHT_LCD_SUPPORT=y +# CONFIG_LCD_CLASS_DEVICE is not set +CONFIG_SOUND=y +CONFIG_SND=y +CONFIG_HIDRAW=y +CONFIG_UHID=y +CONFIG_HID_A4TECH=y +CONFIG_HID_ACRUX=y +CONFIG_HID_ACRUX_FF=y +CONFIG_HID_APPLE=y +CONFIG_HID_BELKIN=y +CONFIG_HID_CHERRY=y +CONFIG_HID_CHICONY=y +CONFIG_HID_PRODIKEYS=y +CONFIG_HID_CYPRESS=y +CONFIG_HID_DRAGONRISE=y +CONFIG_DRAGONRISE_FF=y +CONFIG_HID_EMS_FF=y +CONFIG_HID_ELECOM=y +CONFIG_HID_EZKEY=y +CONFIG_HID_HOLTEK=y +CONFIG_HID_KEYTOUCH=y +CONFIG_HID_KYE=y +CONFIG_HID_UCLOGIC=y +CONFIG_HID_WALTOP=y +CONFIG_HID_GYRATION=y +CONFIG_HID_TWINHAN=y +CONFIG_HID_KENSINGTON=y +CONFIG_HID_LCPOWER=y +CONFIG_HID_LOGITECH=y +CONFIG_HID_LOGITECH_DJ=y +CONFIG_LOGITECH_FF=y +CONFIG_LOGIRUMBLEPAD2_FF=y +CONFIG_LOGIG940_FF=y +CONFIG_HID_MAGICMOUSE=y +CONFIG_HID_MICROSOFT=y +CONFIG_HID_MONTEREY=y +CONFIG_HID_MULTITOUCH=y +CONFIG_HID_NTRIG=y +CONFIG_HID_ORTEK=y +CONFIG_HID_PANTHERLORD=y +CONFIG_PANTHERLORD_FF=y +CONFIG_HID_PETALYNX=y +CONFIG_HID_PICOLCD=y +CONFIG_HID_PRIMAX=y +CONFIG_HID_ROCCAT=y +CONFIG_HID_SAITEK=y +CONFIG_HID_SAMSUNG=y +CONFIG_HID_SONY=y +CONFIG_HID_SPEEDLINK=y +CONFIG_HID_SUNPLUS=y +CONFIG_HID_GREENASIA=y +CONFIG_GREENASIA_FF=y +CONFIG_HID_SMARTJOYPLUS=y +CONFIG_SMARTJOYPLUS_FF=y +CONFIG_HID_TIVO=y +CONFIG_HID_TOPSEED=y +CONFIG_HID_THRUSTMASTER=y +CONFIG_HID_WACOM=y +CONFIG_HID_WIIMOTE=y +CONFIG_HID_ZEROPLUS=y +CONFIG_HID_ZYDACRON=y +CONFIG_HID_PID=y +CONFIG_USB_HIDDEV=y +CONFIG_USB_ANNOUNCE_NEW_DEVICES=y +CONFIG_USB_MON=y +CONFIG_USB_EHCI_HCD=y +# CONFIG_USB_EHCI_TT_NEWSCHED is not set +CONFIG_USB_OHCI_HCD=y +CONFIG_USB_UHCI_HCD=y +CONFIG_USB_PRINTER=y +CONFIG_USB_STORAGE=y +CONFIG_USB_OTG_WAKELOCK=y +CONFIG_EDAC=y +CONFIG_RTC_CLASS=y +# CONFIG_RTC_HCTOSYS is not set +CONFIG_DMADEVICES=y +CONFIG_VIRTIO_PCI=y +CONFIG_STAGING=y +CONFIG_ASHMEM=y +CONFIG_ANDROID_LOW_MEMORY_KILLER=y +CONFIG_SYNC=y +CONFIG_SW_SYNC=y +CONFIG_SYNC_FILE=y +CONFIG_ION=y +CONFIG_GOLDFISH_AUDIO=y +CONFIG_SND_HDA_INTEL=y +CONFIG_GOLDFISH=y +CONFIG_GOLDFISH_PIPE=y +CONFIG_GOLDFISH_SYNC=y +CONFIG_ANDROID=y +CONFIG_ANDROID_BINDER_IPC=y +CONFIG_ISCSI_IBFT_FIND=y +CONFIG_EXT4_FS=y +CONFIG_EXT4_FS_SECURITY=y +CONFIG_QUOTA=y +CONFIG_QUOTA_NETLINK_INTERFACE=y +# CONFIG_PRINT_QUOTA_WARNING is not set +CONFIG_FUSE_FS=y +CONFIG_ISO9660_FS=y +CONFIG_JOLIET=y +CONFIG_ZISOFS=y +CONFIG_MSDOS_FS=y +CONFIG_VFAT_FS=y +CONFIG_PROC_KCORE=y +CONFIG_TMPFS=y +CONFIG_TMPFS_POSIX_ACL=y +CONFIG_HUGETLBFS=y +CONFIG_PSTORE=y +CONFIG_PSTORE_CONSOLE=y +CONFIG_PSTORE_RAM=y +# CONFIG_NETWORK_FILESYSTEMS is not set +CONFIG_NLS_DEFAULT="utf8" +CONFIG_NLS_CODEPAGE_437=y +CONFIG_NLS_ASCII=y +CONFIG_NLS_ISO8859_1=y +CONFIG_NLS_UTF8=y +CONFIG_PRINTK_TIME=y +CONFIG_DEBUG_INFO=y +# CONFIG_ENABLE_WARN_DEPRECATED is not set +# CONFIG_ENABLE_MUST_CHECK is not set +# CONFIG_UNUSED_SYMBOLS is not set +CONFIG_MAGIC_SYSRQ=y +CONFIG_DEBUG_MEMORY_INIT=y +CONFIG_PANIC_TIMEOUT=5 +CONFIG_SCHEDSTATS=y +CONFIG_TIMER_STATS=y +CONFIG_SCHED_TRACER=y +CONFIG_BLK_DEV_IO_TRACE=y +CONFIG_PROVIDE_OHCI1394_DMA_INIT=y +CONFIG_KEYS=y +CONFIG_SECURITY=y +CONFIG_SECURITY_NETWORK=y +CONFIG_SECURITY_SELINUX=y +CONFIG_CRYPTO_TWOFISH=y +CONFIG_ASYMMETRIC_KEY_TYPE=y +CONFIG_ASYMMETRIC_PUBLIC_KEY_SUBTYPE=y +CONFIG_X509_CERTIFICATE_PARSER=y +CONFIG_PKCS7_MESSAGE_PARSER=y +CONFIG_PKCS7_TEST_KEY=y +# CONFIG_VIRTUALIZATION is not set +CONFIG_CRC_T10DIF=y diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S index 21b352a11b493f..edba8606b99a0d 100644 --- a/arch/x86/entry/entry_32.S +++ b/arch/x86/entry/entry_32.S @@ -889,8 +889,8 @@ ftrace_graph_call: jmp ftrace_stub #endif -.globl ftrace_stub -ftrace_stub: +/* This is weak to keep gas from relaxing the jumps */ +WEAK(ftrace_stub) ret END(ftrace_caller) diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 6e395c99690028..7fe88bb57e366c 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -365,7 +365,11 @@ int x86_add_exclusive(unsigned int what) { int i; - if (x86_pmu.lbr_pt_coexist) + /* + * When lbr_pt_coexist we allow PT to coexist with either LBR or BTS. + * LBR and BTS are still mutually exclusive. + */ + if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt) return 0; if (!atomic_inc_not_zero(&x86_pmu.lbr_exclusive[what])) { @@ -388,7 +392,7 @@ int x86_add_exclusive(unsigned int what) void x86_del_exclusive(unsigned int what) { - if (x86_pmu.lbr_pt_coexist) + if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt) return; atomic_dec(&x86_pmu.lbr_exclusive[what]); diff --git a/arch/x86/events/intel/cstate.c b/arch/x86/events/intel/cstate.c index da51e5a3e2ff79..fec8a461bdef6d 100644 --- a/arch/x86/events/intel/cstate.c +++ b/arch/x86/events/intel/cstate.c @@ -594,6 +594,9 @@ static int __init cstate_probe(const struct cstate_model *cm) static inline void cstate_cleanup(void) { + cpuhp_remove_state_nocalls(CPUHP_AP_PERF_X86_CSTATE_ONLINE); + cpuhp_remove_state_nocalls(CPUHP_AP_PERF_X86_CSTATE_STARTING); + if (has_cstate_core) perf_pmu_unregister(&cstate_core_pmu); @@ -606,16 +609,16 @@ static int __init cstate_init(void) int err; cpuhp_setup_state(CPUHP_AP_PERF_X86_CSTATE_STARTING, - "AP_PERF_X86_CSTATE_STARTING", cstate_cpu_init, - NULL); + "perf/x86/cstate:starting", cstate_cpu_init, NULL); cpuhp_setup_state(CPUHP_AP_PERF_X86_CSTATE_ONLINE, - "AP_PERF_X86_CSTATE_ONLINE", NULL, cstate_cpu_exit); + "perf/x86/cstate:online", NULL, cstate_cpu_exit); if (has_cstate_core) { err = perf_pmu_register(&cstate_core_pmu, cstate_core_pmu.name, -1); if (err) { has_cstate_core = false; pr_info("Failed to register cstate core pmu\n"); + cstate_cleanup(); return err; } } @@ -629,8 +632,7 @@ static int __init cstate_init(void) return err; } } - - return err; + return 0; } static int __init cstate_pmu_init(void) @@ -655,8 +657,6 @@ module_init(cstate_pmu_init); static void __exit cstate_pmu_exit(void) { - cpuhp_remove_state_nocalls(CPUHP_AP_PERF_X86_CSTATE_ONLINE); - cpuhp_remove_state_nocalls(CPUHP_AP_PERF_X86_CSTATE_STARTING); cstate_cleanup(); } module_exit(cstate_pmu_exit); diff --git a/arch/x86/events/intel/uncore.c b/arch/x86/events/intel/uncore.c index dbaaf7dc8373cb..19d646a783fd6c 100644 --- a/arch/x86/events/intel/uncore.c +++ b/arch/x86/events/intel/uncore.c @@ -763,30 +763,6 @@ static void uncore_pmu_unregister(struct intel_uncore_pmu *pmu) pmu->registered = false; } -static void __uncore_exit_boxes(struct intel_uncore_type *type, int cpu) -{ - struct intel_uncore_pmu *pmu = type->pmus; - struct intel_uncore_box *box; - int i, pkg; - - if (pmu) { - pkg = topology_physical_package_id(cpu); - for (i = 0; i < type->num_boxes; i++, pmu++) { - box = pmu->boxes[pkg]; - if (box) - uncore_box_exit(box); - } - } -} - -static void uncore_exit_boxes(void *dummy) -{ - struct intel_uncore_type **types; - - for (types = uncore_msr_uncores; *types; types++) - __uncore_exit_boxes(*types++, smp_processor_id()); -} - static void uncore_free_boxes(struct intel_uncore_pmu *pmu) { int pkg; @@ -1077,22 +1053,12 @@ static int uncore_cpu_dying(unsigned int cpu) return 0; } -static int first_init; - static int uncore_cpu_starting(unsigned int cpu) { struct intel_uncore_type *type, **types = uncore_msr_uncores; struct intel_uncore_pmu *pmu; struct intel_uncore_box *box; - int i, pkg, ncpus = 1; - - if (first_init) { - /* - * On init we get the number of online cpus in the package - * and set refcount for all of them. - */ - ncpus = cpumask_weight(topology_core_cpumask(cpu)); - } + int i, pkg; pkg = topology_logical_package_id(cpu); for (; *types; types++) { @@ -1103,7 +1069,7 @@ static int uncore_cpu_starting(unsigned int cpu) if (!box) continue; /* The first cpu on a package activates the box */ - if (atomic_add_return(ncpus, &box->refcnt) == ncpus) + if (atomic_inc_return(&box->refcnt) == 1) uncore_box_init(box); } } @@ -1407,19 +1373,17 @@ static int __init intel_uncore_init(void) "PERF_X86_UNCORE_PREP", uncore_cpu_prepare, NULL); } - first_init = 1; + cpuhp_setup_state(CPUHP_AP_PERF_X86_UNCORE_STARTING, "AP_PERF_X86_UNCORE_STARTING", uncore_cpu_starting, uncore_cpu_dying); - first_init = 0; + cpuhp_setup_state(CPUHP_AP_PERF_X86_UNCORE_ONLINE, "AP_PERF_X86_UNCORE_ONLINE", uncore_event_cpu_online, uncore_event_cpu_offline); return 0; err: - /* Undo box->init_box() */ - on_each_cpu_mask(&uncore_cpu_mask, uncore_exit_boxes, NULL, 1); uncore_types_exit(uncore_msr_uncores); uncore_pci_exit(); return ret; diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index a77ee026643d23..bcbb1d2ae10b21 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -604,7 +604,7 @@ struct x86_pmu { u64 lbr_sel_mask; /* LBR_SELECT valid bits */ const int *lbr_sel_map; /* lbr_select mappings */ bool lbr_double_abort; /* duplicated lbr aborts */ - bool lbr_pt_coexist; /* LBR may coexist with PT */ + bool lbr_pt_coexist; /* (LBR|BTS) may coexist with PT */ /* * Intel PT/LBR/BTS are exclusive diff --git a/arch/x86/include/asm/asm-prototypes.h b/arch/x86/include/asm/asm-prototypes.h new file mode 100644 index 00000000000000..44b8762fa0c784 --- /dev/null +++ b/arch/x86/include/asm/asm-prototypes.h @@ -0,0 +1,16 @@ +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + +#ifndef CONFIG_X86_CMPXCHG64 +extern void cmpxchg8b_emu(void); +#endif diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index a39629206864e5..ed10b5bf9b93d7 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -311,4 +311,6 @@ #define X86_BUG_NULL_SEG X86_BUG(10) /* Nulling a selector preserves the base */ #define X86_BUG_SWAPGS_FENCE X86_BUG(11) /* SWAPGS without input dep on GS */ #define X86_BUG_MONITOR X86_BUG(12) /* IPI required to wake up remote CPU */ +#define X86_BUG_AMD_E400 X86_BUG(13) /* CPU is among the affected by Erratum 400 */ + #endif /* _ASM_X86_CPUFEATURES_H */ diff --git a/arch/x86/include/asm/idle.h b/arch/x86/include/asm/idle.h index c5d1785373ed38..02bab09707f28c 100644 --- a/arch/x86/include/asm/idle.h +++ b/arch/x86/include/asm/idle.h @@ -1,13 +1,6 @@ #ifndef _ASM_X86_IDLE_H #define _ASM_X86_IDLE_H -#define IDLE_START 1 -#define IDLE_END 2 - -struct notifier_block; -void idle_notifier_register(struct notifier_block *n); -void idle_notifier_unregister(struct notifier_block *n); - #ifdef CONFIG_X86_64 void enter_idle(void); void exit_idle(void); diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index 984a7bf17f6a72..83db0eae99797d 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -104,6 +104,7 @@ struct cpuinfo_x86 { __u8 x86_phys_bits; /* CPUID returned core id bits: */ __u8 x86_coreid_bits; + __u8 cu_id; /* Max extended CPUID function supported: */ __u32 extended_cpuid_level; /* Maximum supported CPUID level, -1=no CPUID: */ diff --git a/arch/x86/include/uapi/asm/prctl.h b/arch/x86/include/uapi/asm/prctl.h index ae135de547f561..835aa51c7f6ebb 100644 --- a/arch/x86/include/uapi/asm/prctl.h +++ b/arch/x86/include/uapi/asm/prctl.h @@ -6,10 +6,8 @@ #define ARCH_GET_FS 0x1003 #define ARCH_GET_GS 0x1004 -#ifdef CONFIG_CHECKPOINT_RESTORE -# define ARCH_MAP_VDSO_X32 0x2001 -# define ARCH_MAP_VDSO_32 0x2002 -# define ARCH_MAP_VDSO_64 0x2003 -#endif +#define ARCH_MAP_VDSO_X32 0x2001 +#define ARCH_MAP_VDSO_32 0x2002 +#define ARCH_MAP_VDSO_64 0x2003 #endif /* _ASM_X86_PRCTL_H */ diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index 88c657b057e223..f2234918e4944e 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -2158,21 +2158,6 @@ int __generic_processor_info(int apicid, int version, bool enabled) } } - /* - * This can happen on physical hotplug. The sanity check at boot time - * is done from native_smp_prepare_cpus() after num_possible_cpus() is - * established. - */ - if (topology_update_package_map(apicid, cpu) < 0) { - int thiscpu = max + disabled_cpus; - - pr_warning("APIC: Package limit reached. Processor %d/0x%x ignored.\n", - thiscpu, apicid); - - disabled_cpus++; - return -ENOSPC; - } - /* * Validate version */ diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c index 48e6d84f173e23..d1e25564b3c192 100644 --- a/arch/x86/kernel/apic/io_apic.c +++ b/arch/x86/kernel/apic/io_apic.c @@ -2116,6 +2116,7 @@ static inline void __init check_timer(void) if (idx != -1 && irq_trigger(idx)) unmask_ioapic_irq(irq_get_chip_data(0)); } + irq_domain_deactivate_irq(irq_data); irq_domain_activate_irq(irq_data); if (timer_irq_works()) { if (disable_timer_pin_1 > 0) @@ -2137,6 +2138,7 @@ static inline void __init check_timer(void) * legacy devices should be connected to IO APIC #0 */ replace_pin_at_irq_node(data, node, apic1, pin1, apic2, pin2); + irq_domain_deactivate_irq(irq_data); irq_domain_activate_irq(irq_data); legacy_pic->unmask(0); if (timer_irq_works()) { diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index 1e81a37c034e78..2b4cf04239b6c0 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -20,6 +20,10 @@ #include "cpu.h" +static const int amd_erratum_383[]; +static const int amd_erratum_400[]; +static bool cpu_has_amd_erratum(struct cpuinfo_x86 *cpu, const int *erratum); + /* * nodes_per_socket: Stores the number of nodes per socket. * Refer to Fam15h Models 00-0fh BKDG - CPUID Fn8000_001E_ECX @@ -308,17 +312,43 @@ static void amd_get_topology(struct cpuinfo_x86 *c) u32 eax, ebx, ecx, edx; cpuid(0x8000001e, &eax, &ebx, &ecx, &edx); - node_id = ecx & 7; - /* get compute unit information */ - smp_num_siblings = ((ebx >> 8) & 3) + 1; - c->x86_max_cores /= smp_num_siblings; - c->cpu_core_id = ebx & 0xff; + node_id = ecx & 0xff; + smp_num_siblings = ((ebx >> 8) & 0xff) + 1; + + if (c->x86 == 0x15) + c->cu_id = ebx & 0xff; + + if (c->x86 >= 0x17) { + c->cpu_core_id = ebx & 0xff; + + if (smp_num_siblings > 1) + c->x86_max_cores /= smp_num_siblings; + } + + /* + * We may have multiple LLCs if L3 caches exist, so check if we + * have an L3 cache by looking at the L3 cache CPUID leaf. + */ + if (cpuid_edx(0x80000006)) { + if (c->x86 == 0x17) { + /* + * LLC is at the core complex level. + * Core complex id is ApicId[3]. + */ + per_cpu(cpu_llc_id, cpu) = c->apicid >> 3; + } else { + /* LLC is at the node level. */ + per_cpu(cpu_llc_id, cpu) = node_id; + } + } } else if (cpu_has(c, X86_FEATURE_NODEID_MSR)) { u64 value; rdmsrl(MSR_FAM10H_NODE_ID, value); node_id = value & 7; + + per_cpu(cpu_llc_id, cpu) = node_id; } else return; @@ -329,9 +359,6 @@ static void amd_get_topology(struct cpuinfo_x86 *c) set_cpu_cap(c, X86_FEATURE_AMD_DCM); cus_per_node = c->x86_max_cores / nodes_per_socket; - /* store NodeID, use llc_shared_map to store sibling info */ - per_cpu(cpu_llc_id, cpu) = node_id; - /* core id has to be in the [0 .. cores_per_node - 1] range */ c->cpu_core_id %= cus_per_node; } @@ -356,15 +383,6 @@ static void amd_detect_cmp(struct cpuinfo_x86 *c) /* use socket ID also for last level cache */ per_cpu(cpu_llc_id, cpu) = c->phys_proc_id; amd_get_topology(c); - - /* - * Fix percpu cpu_llc_id here as LLC topology is different - * for Fam17h systems. - */ - if (c->x86 != 0x17 || !cpuid_edx(0x80000006)) - return; - - per_cpu(cpu_llc_id, cpu) = c->apicid >> 3; #endif } @@ -585,11 +603,16 @@ static void early_init_amd(struct cpuinfo_x86 *c) /* F16h erratum 793, CVE-2013-6885 */ if (c->x86 == 0x16 && c->x86_model <= 0xf) msr_set_bit(MSR_AMD64_LS_CFG, 15); -} -static const int amd_erratum_383[]; -static const int amd_erratum_400[]; -static bool cpu_has_amd_erratum(struct cpuinfo_x86 *cpu, const int *erratum); + /* + * Check whether the machine is affected by erratum 400. This is + * used to select the proper idle routine and to enable the check + * whether the machine is affected in arch_post_acpi_init(), which + * sets the X86_BUG_AMD_APIC_C1E bug depending on the MSR check. + */ + if (cpu_has_amd_erratum(c, amd_erratum_400)) + set_cpu_bug(c, X86_BUG_AMD_E400); +} static void init_amd_k8(struct cpuinfo_x86 *c) { @@ -770,9 +793,6 @@ static void init_amd(struct cpuinfo_x86 *c) if (c->x86 > 0x11) set_cpu_cap(c, X86_FEATURE_ARAT); - if (cpu_has_amd_erratum(c, amd_erratum_400)) - set_cpu_bug(c, X86_BUG_AMD_APIC_C1E); - rdmsr_safe(MSR_AMD64_PATCH_LEVEL, &c->microcode, &dummy); /* 3DNow or LM implies PREFETCHW */ diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index cc9e980c68ec47..4eece91ada374d 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -667,13 +667,14 @@ void get_cpu_cap(struct cpuinfo_x86 *c) c->x86_capability[CPUID_1_EDX] = edx; } + /* Thermal and Power Management Leaf: level 0x00000006 (eax) */ + if (c->cpuid_level >= 0x00000006) + c->x86_capability[CPUID_6_EAX] = cpuid_eax(0x00000006); + /* Additional Intel-defined flags: level 0x00000007 */ if (c->cpuid_level >= 0x00000007) { cpuid_count(0x00000007, 0, &eax, &ebx, &ecx, &edx); - c->x86_capability[CPUID_7_0_EBX] = ebx; - - c->x86_capability[CPUID_6_EAX] = cpuid_eax(0x00000006); c->x86_capability[CPUID_7_ECX] = ecx; } @@ -979,29 +980,21 @@ static void x86_init_cache_qos(struct cpuinfo_x86 *c) } /* - * The physical to logical package id mapping is initialized from the - * acpi/mptables information. Make sure that CPUID actually agrees with - * that. + * Validate that ACPI/mptables have the same information about the + * effective APIC id and update the package map. */ -static void sanitize_package_id(struct cpuinfo_x86 *c) +static void validate_apic_and_package_id(struct cpuinfo_x86 *c) { #ifdef CONFIG_SMP - unsigned int pkg, apicid, cpu = smp_processor_id(); + unsigned int apicid, cpu = smp_processor_id(); apicid = apic->cpu_present_to_apicid(cpu); - pkg = apicid >> boot_cpu_data.x86_coreid_bits; - if (apicid != c->initial_apicid) { - pr_err(FW_BUG "CPU%u: APIC id mismatch. Firmware: %x CPUID: %x\n", + if (apicid != c->apicid) { + pr_err(FW_BUG "CPU%u: APIC id mismatch. Firmware: %x APIC: %x\n", cpu, apicid, c->initial_apicid); - c->initial_apicid = apicid; - } - if (pkg != c->phys_proc_id) { - pr_err(FW_BUG "CPU%u: Using firmware package id %u instead of %u\n", - cpu, pkg, c->phys_proc_id); - c->phys_proc_id = pkg; } - c->logical_proc_id = topology_phys_to_logical_pkg(pkg); + BUG_ON(topology_update_package_map(c->phys_proc_id, cpu)); #else c->logical_proc_id = 0; #endif @@ -1022,6 +1015,7 @@ static void identify_cpu(struct cpuinfo_x86 *c) c->x86_model_id[0] = '\0'; /* Unset */ c->x86_max_cores = 1; c->x86_coreid_bits = 0; + c->cu_id = 0xff; #ifdef CONFIG_X86_64 c->x86_clflush_size = 64; c->x86_phys_bits = 36; @@ -1132,7 +1126,6 @@ static void identify_cpu(struct cpuinfo_x86 *c) #ifdef CONFIG_NUMA numa_add_cpu(smp_processor_id()); #endif - sanitize_package_id(c); } /* @@ -1188,6 +1181,7 @@ void identify_secondary_cpu(struct cpuinfo_x86 *c) enable_sep_cpu(); #endif mtrr_ap_init(); + validate_apic_and_package_id(c); } struct msr_range { @@ -1282,7 +1276,7 @@ static __init int setup_disablecpuid(char *arg) { int bit; - if (get_option(&arg, &bit) && bit < NCAPINTS*32) + if (get_option(&arg, &bit) && bit >= 0 && bit < NCAPINTS * 32) setup_clear_cpu_cap(bit); else return 0; diff --git a/arch/x86/kernel/hpet.c b/arch/x86/kernel/hpet.c index 274fab99169d82..932348fbb6ea05 100644 --- a/arch/x86/kernel/hpet.c +++ b/arch/x86/kernel/hpet.c @@ -352,6 +352,7 @@ static int hpet_resume(struct clock_event_device *evt, int timer) } else { struct hpet_dev *hdev = EVT_TO_HPET_DEV(evt); + irq_domain_deactivate_irq(irq_get_irq_data(hdev->irq)); irq_domain_activate_irq(irq_get_irq_data(hdev->irq)); disable_irq(hdev->irq); irq_set_affinity(hdev->irq, cpumask_of(hdev->cpu)); diff --git a/arch/x86/kernel/pci-swiotlb.c b/arch/x86/kernel/pci-swiotlb.c index b47edb8f52566e..8da13d4e77cc1e 100644 --- a/arch/x86/kernel/pci-swiotlb.c +++ b/arch/x86/kernel/pci-swiotlb.c @@ -70,7 +70,7 @@ int __init pci_swiotlb_detect_override(void) { int use_swiotlb = swiotlb | swiotlb_force; - if (swiotlb_force) + if (swiotlb_force == SWIOTLB_FORCE) swiotlb = 1; return use_swiotlb; diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 0888a879120fcb..fc7cf64587f2c8 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -67,19 +67,6 @@ EXPORT_PER_CPU_SYMBOL(cpu_tss); #ifdef CONFIG_X86_64 static DEFINE_PER_CPU(unsigned char, is_idle); -static ATOMIC_NOTIFIER_HEAD(idle_notifier); - -void idle_notifier_register(struct notifier_block *n) -{ - atomic_notifier_chain_register(&idle_notifier, n); -} -EXPORT_SYMBOL_GPL(idle_notifier_register); - -void idle_notifier_unregister(struct notifier_block *n) -{ - atomic_notifier_chain_unregister(&idle_notifier, n); -} -EXPORT_SYMBOL_GPL(idle_notifier_unregister); #endif /* @@ -255,14 +242,14 @@ static inline void play_dead(void) void enter_idle(void) { this_cpu_write(is_idle, 1); - atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL); + idle_notifier_call_chain(IDLE_START); } static void __exit_idle(void) { if (x86_test_and_clear_bit_percpu(0, is_idle) == 0) return; - atomic_notifier_call_chain(&idle_notifier, IDLE_END, NULL); + idle_notifier_call_chain(IDLE_END); } /* Called from interrupts to signify idle end */ @@ -448,8 +435,7 @@ void select_idle_routine(const struct cpuinfo_x86 *c) if (x86_idle || boot_option_idle_override == IDLE_POLL) return; - if (cpu_has_bug(c, X86_BUG_AMD_APIC_C1E)) { - /* E400: APIC timer interrupt does not wake up CPU from C1e */ + if (boot_cpu_has_bug(X86_BUG_AMD_E400)) { pr_info("using AMD E400 aware idle routine\n"); x86_idle = amd_e400_idle; } else if (prefer_mwait_c1_over_halt(c)) { diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index 42f5eb7b4f6c85..36171bcd91f81c 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -104,7 +104,6 @@ static unsigned int max_physical_pkg_id __read_mostly; unsigned int __max_logical_packages __read_mostly; EXPORT_SYMBOL(__max_logical_packages); static unsigned int logical_packages __read_mostly; -static bool logical_packages_frozen __read_mostly; /* Maximum number of SMT threads on any online core */ int __max_smt_threads __read_mostly; @@ -263,9 +262,14 @@ static void notrace start_secondary(void *unused) cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); } -int topology_update_package_map(unsigned int apicid, unsigned int cpu) +/** + * topology_update_package_map - Update the physical to logical package map + * @pkg: The physical package id as retrieved via CPUID + * @cpu: The cpu for which this is updated + */ +int topology_update_package_map(unsigned int pkg, unsigned int cpu) { - unsigned int new, pkg = apicid >> boot_cpu_data.x86_coreid_bits; + unsigned int new; /* Called from early boot ? */ if (!physical_package_map) @@ -278,16 +282,17 @@ int topology_update_package_map(unsigned int apicid, unsigned int cpu) if (test_and_set_bit(pkg, physical_package_map)) goto found; - if (logical_packages_frozen) { - physical_to_logical_pkg[pkg] = -1; - pr_warn("APIC(%x) Package %u exceeds logical package max\n", - apicid, pkg); + if (logical_packages >= __max_logical_packages) { + pr_warn("Package %u of CPU %u exceeds BIOS package data %u.\n", + logical_packages, cpu, __max_logical_packages); return -ENOSPC; } new = logical_packages++; - pr_info("APIC(%x) Converting physical %u to logical package %u\n", - apicid, pkg, new); + if (new != pkg) { + pr_info("CPU %u Converting physical %u to logical package %u\n", + cpu, pkg, new); + } physical_to_logical_pkg[pkg] = new; found: @@ -308,9 +313,9 @@ int topology_phys_to_logical_pkg(unsigned int phys_pkg) } EXPORT_SYMBOL(topology_phys_to_logical_pkg); -static void __init smp_init_package_map(void) +static void __init smp_init_package_map(struct cpuinfo_x86 *c, unsigned int cpu) { - unsigned int ncpus, cpu; + unsigned int ncpus; size_t size; /* @@ -355,27 +360,9 @@ static void __init smp_init_package_map(void) size = BITS_TO_LONGS(max_physical_pkg_id) * sizeof(unsigned long); physical_package_map = kzalloc(size, GFP_KERNEL); - for_each_present_cpu(cpu) { - unsigned int apicid = apic->cpu_present_to_apicid(cpu); - - if (apicid == BAD_APICID || !apic->apic_id_valid(apicid)) - continue; - if (!topology_update_package_map(apicid, cpu)) - continue; - pr_warn("CPU %u APICId %x disabled\n", cpu, apicid); - per_cpu(x86_bios_cpu_apicid, cpu) = BAD_APICID; - set_cpu_possible(cpu, false); - set_cpu_present(cpu, false); - } - - if (logical_packages > __max_logical_packages) { - pr_warn("Detected more packages (%u), then computed by BIOS data (%u).\n", - logical_packages, __max_logical_packages); - logical_packages_frozen = true; - __max_logical_packages = logical_packages; - } - pr_info("Max logical packages: %u\n", __max_logical_packages); + + topology_update_package_map(c->phys_proc_id, cpu); } void __init smp_store_boot_cpu_info(void) @@ -385,7 +372,7 @@ void __init smp_store_boot_cpu_info(void) *c = boot_cpu_data; c->cpu_index = id; - smp_init_package_map(); + smp_init_package_map(c, id); } /* @@ -436,9 +423,15 @@ static bool match_smt(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) int cpu1 = c->cpu_index, cpu2 = o->cpu_index; if (c->phys_proc_id == o->phys_proc_id && - per_cpu(cpu_llc_id, cpu1) == per_cpu(cpu_llc_id, cpu2) && - c->cpu_core_id == o->cpu_core_id) - return topology_sane(c, o, "smt"); + per_cpu(cpu_llc_id, cpu1) == per_cpu(cpu_llc_id, cpu2)) { + if (c->cpu_core_id == o->cpu_core_id) + return topology_sane(c, o, "smt"); + + if ((c->cu_id != 0xff) && + (o->cu_id != 0xff) && + (c->cu_id == o->cu_id)) + return topology_sane(c, o, "smt"); + } } else if (c->phys_proc_id == o->phys_proc_id && c->cpu_core_id == o->cpu_core_id) { diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index a3ce9d260d6875..9f676adcdfc20b 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -171,6 +171,7 @@ #define NearBranch ((u64)1 << 52) /* Near branches */ #define No16 ((u64)1 << 53) /* No 16 bit operand */ #define IncSP ((u64)1 << 54) /* SP is incremented before ModRM calc */ +#define Aligned16 ((u64)1 << 55) /* Aligned to 16 byte boundary (e.g. FXSAVE) */ #define DstXacc (DstAccLo | SrcAccHi | SrcWrite) @@ -446,6 +447,26 @@ FOP_END; FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET FOP_END; +/* + * XXX: inoutclob user must know where the argument is being expanded. + * Relying on CC_HAVE_ASM_GOTO would allow us to remove _fault. + */ +#define asm_safe(insn, inoutclob...) \ +({ \ + int _fault = 0; \ + \ + asm volatile("1:" insn "\n" \ + "2:\n" \ + ".pushsection .fixup, \"ax\"\n" \ + "3: movl $1, %[_fault]\n" \ + " jmp 2b\n" \ + ".popsection\n" \ + _ASM_EXTABLE(1b, 3b) \ + : [_fault] "+qm"(_fault) inoutclob ); \ + \ + _fault ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; \ +}) + static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt, enum x86_intercept intercept, enum x86_intercept_stage stage) @@ -632,21 +653,24 @@ static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector, * depending on whether they're AVX encoded or not. * * Also included is CMPXCHG16B which is not a vector instruction, yet it is - * subject to the same check. + * subject to the same check. FXSAVE and FXRSTOR are checked here too as their + * 512 bytes of data must be aligned to a 16 byte boundary. */ -static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size) +static unsigned insn_alignment(struct x86_emulate_ctxt *ctxt, unsigned size) { if (likely(size < 16)) - return false; + return 1; if (ctxt->d & Aligned) - return true; + return size; else if (ctxt->d & Unaligned) - return false; + return 1; else if (ctxt->d & Avx) - return false; + return 1; + else if (ctxt->d & Aligned16) + return 16; else - return true; + return size; } static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt, @@ -704,7 +728,7 @@ static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt, } break; } - if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0)) + if (la & (insn_alignment(ctxt, size) - 1)) return emulate_gp(ctxt, 0); return X86EMUL_CONTINUE; bad: @@ -791,6 +815,20 @@ static int segmented_read_std(struct x86_emulate_ctxt *ctxt, return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception); } +static int segmented_write_std(struct x86_emulate_ctxt *ctxt, + struct segmented_address addr, + void *data, + unsigned int size) +{ + int rc; + ulong linear; + + rc = linearize(ctxt, addr, size, true, &linear); + if (rc != X86EMUL_CONTINUE) + return rc; + return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception); +} + /* * Prefetch the remaining bytes of the instruction without crossing page * boundary if they are not in fetch_cache yet. @@ -1544,7 +1582,6 @@ static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt, &ctxt->exception); } -/* Does not support long mode */ static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt, u16 selector, int seg, u8 cpl, enum x86_transfer_type transfer, @@ -1581,20 +1618,34 @@ static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt, rpl = selector & 3; - /* NULL selector is not valid for TR, CS and SS (except for long mode) */ - if ((seg == VCPU_SREG_CS - || (seg == VCPU_SREG_SS - && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl)) - || seg == VCPU_SREG_TR) - && null_selector) - goto exception; - /* TR should be in GDT only */ if (seg == VCPU_SREG_TR && (selector & (1 << 2))) goto exception; - if (null_selector) /* for NULL selector skip all following checks */ + /* NULL selector is not valid for TR, CS and (except for long mode) SS */ + if (null_selector) { + if (seg == VCPU_SREG_CS || seg == VCPU_SREG_TR) + goto exception; + + if (seg == VCPU_SREG_SS) { + if (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl) + goto exception; + + /* + * ctxt->ops->set_segment expects the CPL to be in + * SS.DPL, so fake an expand-up 32-bit data segment. + */ + seg_desc.type = 3; + seg_desc.p = 1; + seg_desc.s = 1; + seg_desc.dpl = cpl; + seg_desc.d = 1; + seg_desc.g = 1; + } + + /* Skip all following checks */ goto load; + } ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr); if (ret != X86EMUL_CONTINUE) @@ -1710,6 +1761,21 @@ static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt, u16 selector, int seg) { u8 cpl = ctxt->ops->cpl(ctxt); + + /* + * None of MOV, POP and LSS can load a NULL selector in CPL=3, but + * they can load it at CPL<3 (Intel's manual says only LSS can, + * but it's wrong). + * + * However, the Intel manual says that putting IST=1/DPL=3 in + * an interrupt gate will result in SS=3 (the AMD manual instead + * says it doesn't), so allow SS=3 in __load_segment_descriptor + * and only forbid it here. + */ + if (seg == VCPU_SREG_SS && selector == 3 && + ctxt->mode == X86EMUL_MODE_PROT64) + return emulate_exception(ctxt, GP_VECTOR, 0, true); + return __load_segment_descriptor(ctxt, selector, seg, cpl, X86_TRANSFER_NONE, NULL); } @@ -3658,8 +3724,8 @@ static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt, } /* Disable writeback. */ ctxt->dst.type = OP_NONE; - return segmented_write(ctxt, ctxt->dst.addr.mem, - &desc_ptr, 2 + ctxt->op_bytes); + return segmented_write_std(ctxt, ctxt->dst.addr.mem, + &desc_ptr, 2 + ctxt->op_bytes); } static int em_sgdt(struct x86_emulate_ctxt *ctxt) @@ -3842,6 +3908,131 @@ static int em_movsxd(struct x86_emulate_ctxt *ctxt) return X86EMUL_CONTINUE; } +static int check_fxsr(struct x86_emulate_ctxt *ctxt) +{ + u32 eax = 1, ebx, ecx = 0, edx; + + ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx); + if (!(edx & FFL(FXSR))) + return emulate_ud(ctxt); + + if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM)) + return emulate_nm(ctxt); + + /* + * Don't emulate a case that should never be hit, instead of working + * around a lack of fxsave64/fxrstor64 on old compilers. + */ + if (ctxt->mode >= X86EMUL_MODE_PROT64) + return X86EMUL_UNHANDLEABLE; + + return X86EMUL_CONTINUE; +} + +/* + * FXSAVE and FXRSTOR have 4 different formats depending on execution mode, + * 1) 16 bit mode + * 2) 32 bit mode + * - like (1), but FIP and FDP (foo) are only 16 bit. At least Intel CPUs + * preserve whole 32 bit values, though, so (1) and (2) are the same wrt. + * save and restore + * 3) 64-bit mode with REX.W prefix + * - like (2), but XMM 8-15 are being saved and restored + * 4) 64-bit mode without REX.W prefix + * - like (3), but FIP and FDP are 64 bit + * + * Emulation uses (3) for (1) and (2) and preserves XMM 8-15 to reach the + * desired result. (4) is not emulated. + * + * Note: Guest and host CPUID.(EAX=07H,ECX=0H):EBX[bit 13] (deprecate FPU CS + * and FPU DS) should match. + */ +static int em_fxsave(struct x86_emulate_ctxt *ctxt) +{ + struct fxregs_state fx_state; + size_t size; + int rc; + + rc = check_fxsr(ctxt); + if (rc != X86EMUL_CONTINUE) + return rc; + + ctxt->ops->get_fpu(ctxt); + + rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_state)); + + ctxt->ops->put_fpu(ctxt); + + if (rc != X86EMUL_CONTINUE) + return rc; + + if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR) + size = offsetof(struct fxregs_state, xmm_space[8 * 16/4]); + else + size = offsetof(struct fxregs_state, xmm_space[0]); + + return segmented_write_std(ctxt, ctxt->memop.addr.mem, &fx_state, size); +} + +static int fxrstor_fixup(struct x86_emulate_ctxt *ctxt, + struct fxregs_state *new) +{ + int rc = X86EMUL_CONTINUE; + struct fxregs_state old; + + rc = asm_safe("fxsave %[fx]", , [fx] "+m"(old)); + if (rc != X86EMUL_CONTINUE) + return rc; + + /* + * 64 bit host will restore XMM 8-15, which is not correct on non-64 + * bit guests. Load the current values in order to preserve 64 bit + * XMMs after fxrstor. + */ +#ifdef CONFIG_X86_64 + /* XXX: accessing XMM 8-15 very awkwardly */ + memcpy(&new->xmm_space[8 * 16/4], &old.xmm_space[8 * 16/4], 8 * 16); +#endif + + /* + * Hardware doesn't save and restore XMM 0-7 without CR4.OSFXSR, but + * does save and restore MXCSR. + */ + if (!(ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR)) + memcpy(new->xmm_space, old.xmm_space, 8 * 16); + + return rc; +} + +static int em_fxrstor(struct x86_emulate_ctxt *ctxt) +{ + struct fxregs_state fx_state; + int rc; + + rc = check_fxsr(ctxt); + if (rc != X86EMUL_CONTINUE) + return rc; + + rc = segmented_read_std(ctxt, ctxt->memop.addr.mem, &fx_state, 512); + if (rc != X86EMUL_CONTINUE) + return rc; + + if (fx_state.mxcsr >> 16) + return emulate_gp(ctxt, 0); + + ctxt->ops->get_fpu(ctxt); + + if (ctxt->mode < X86EMUL_MODE_PROT64) + rc = fxrstor_fixup(ctxt, &fx_state); + + if (rc == X86EMUL_CONTINUE) + rc = asm_safe("fxrstor %[fx]", : [fx] "m"(fx_state)); + + ctxt->ops->put_fpu(ctxt); + + return rc; +} + static bool valid_cr(int nr) { switch (nr) { @@ -4194,7 +4385,9 @@ static const struct gprefix pfx_0f_ae_7 = { }; static const struct group_dual group15 = { { - N, N, N, N, N, N, N, GP(0, &pfx_0f_ae_7), + I(ModRM | Aligned16, em_fxsave), + I(ModRM | Aligned16, em_fxrstor), + N, N, N, N, N, GP(0, &pfx_0f_ae_7), }, { N, N, N, N, N, N, N, N, } }; @@ -5066,21 +5259,13 @@ static bool string_insn_completed(struct x86_emulate_ctxt *ctxt) static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt) { - bool fault = false; + int rc; ctxt->ops->get_fpu(ctxt); - asm volatile("1: fwait \n\t" - "2: \n\t" - ".pushsection .fixup,\"ax\" \n\t" - "3: \n\t" - "movb $1, %[fault] \n\t" - "jmp 2b \n\t" - ".popsection \n\t" - _ASM_EXTABLE(1b, 3b) - : [fault]"+qm"(fault)); + rc = asm_safe("fwait"); ctxt->ops->put_fpu(ctxt); - if (unlikely(fault)) + if (unlikely(rc != X86EMUL_CONTINUE)) return emulate_exception(ctxt, MF_VECTOR, 0, false); return X86EMUL_CONTINUE; diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 6f69340f9fa314..3f05c044720b7a 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -2360,3 +2360,9 @@ void kvm_lapic_init(void) jump_label_rate_limit(&apic_hw_disabled, HZ); jump_label_rate_limit(&apic_sw_disabled, HZ); } + +void kvm_lapic_exit(void) +{ + static_key_deferred_flush(&apic_hw_disabled); + static_key_deferred_flush(&apic_sw_disabled); +} diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h index f60d01c29d5107..4dfe4d6cb3381c 100644 --- a/arch/x86/kvm/lapic.h +++ b/arch/x86/kvm/lapic.h @@ -108,6 +108,7 @@ static inline bool kvm_hv_vapic_assist_page_enabled(struct kvm_vcpu *vcpu) int kvm_lapic_enable_pv_eoi(struct kvm_vcpu *vcpu, u64 data); void kvm_lapic_init(void); +void kvm_lapic_exit(void); #define VEC_POS(v) ((v) & (32 - 1)) #define REG_POS(v) (((v) >> 5) << 4) diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c index 5382b82462fcba..64774f419c723d 100644 --- a/arch/x86/kvm/vmx.c +++ b/arch/x86/kvm/vmx.c @@ -1343,10 +1343,10 @@ static inline bool nested_cpu_has_posted_intr(struct vmcs12 *vmcs12) return vmcs12->pin_based_vm_exec_control & PIN_BASED_POSTED_INTR; } -static inline bool is_exception(u32 intr_info) +static inline bool is_nmi(u32 intr_info) { return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK)) - == (INTR_TYPE_HARD_EXCEPTION | INTR_INFO_VALID_MASK); + == (INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK); } static void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason, @@ -5476,7 +5476,7 @@ static int handle_exception(struct kvm_vcpu *vcpu) if (is_machine_check(intr_info)) return handle_machine_check(vcpu); - if ((intr_info & INTR_INFO_INTR_TYPE_MASK) == INTR_TYPE_NMI_INTR) + if (is_nmi(intr_info)) return 1; /* already handled by vmx_vcpu_run() */ if (is_no_device(intr_info)) { @@ -8018,7 +8018,7 @@ static bool nested_vmx_exit_handled(struct kvm_vcpu *vcpu) switch (exit_reason) { case EXIT_REASON_EXCEPTION_NMI: - if (!is_exception(intr_info)) + if (is_nmi(intr_info)) return false; else if (is_page_fault(intr_info)) return enable_ept; @@ -8611,8 +8611,7 @@ static void vmx_complete_atomic_exit(struct vcpu_vmx *vmx) kvm_machine_check(); /* We need to handle NMIs before interrupts are enabled */ - if ((exit_intr_info & INTR_INFO_INTR_TYPE_MASK) == INTR_TYPE_NMI_INTR && - (exit_intr_info & INTR_INFO_VALID_MASK)) { + if (is_nmi(exit_intr_info)) { kvm_before_handle_nmi(&vmx->vcpu); asm("int $2"); kvm_after_handle_nmi(&vmx->vcpu); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 04c5d96b1d678a..731044efb195f4 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -3036,6 +3036,8 @@ static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu, memset(&events->reserved, 0, sizeof(events->reserved)); } +static void kvm_set_hflags(struct kvm_vcpu *vcpu, unsigned emul_flags); + static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu, struct kvm_vcpu_events *events) { @@ -3072,10 +3074,13 @@ static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu, vcpu->arch.apic->sipi_vector = events->sipi_vector; if (events->flags & KVM_VCPUEVENT_VALID_SMM) { + u32 hflags = vcpu->arch.hflags; if (events->smi.smm) - vcpu->arch.hflags |= HF_SMM_MASK; + hflags |= HF_SMM_MASK; else - vcpu->arch.hflags &= ~HF_SMM_MASK; + hflags &= ~HF_SMM_MASK; + kvm_set_hflags(vcpu, hflags); + vcpu->arch.smi_pending = events->smi.pending; if (events->smi.smm_inside_nmi) vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK; @@ -3143,6 +3148,7 @@ static void fill_xsave(u8 *dest, struct kvm_vcpu *vcpu) memcpy(dest, xsave, XSAVE_HDR_OFFSET); /* Set XSTATE_BV */ + xstate_bv &= vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FPSSE; *(u64 *)(dest + XSAVE_HDR_OFFSET) = xstate_bv; /* @@ -3303,6 +3309,8 @@ static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, switch (cap->cap) { case KVM_CAP_HYPERV_SYNIC: + if (!irqchip_in_kernel(vcpu->kvm)) + return -EINVAL; return kvm_hv_activate_synic(vcpu); default: return -EINVAL; @@ -5958,6 +5966,7 @@ int kvm_arch_init(void *opaque) void kvm_arch_exit(void) { + kvm_lapic_exit(); perf_unregister_guest_info_callbacks(&kvm_guest_cbs); if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) diff --git a/arch/x86/mm/dump_pagetables.c b/arch/x86/mm/dump_pagetables.c index ea9c49adaa1fce..8aa6bea1cd6cc4 100644 --- a/arch/x86/mm/dump_pagetables.c +++ b/arch/x86/mm/dump_pagetables.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -406,6 +407,7 @@ static void ptdump_walk_pgd_level_core(struct seq_file *m, pgd_t *pgd, } else note_page(m, &st, __pgprot(0), 1); + cond_resched(); start++; } diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index fe04a04dab8ec0..15f74361592366 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1172,6 +1172,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) set_memory_ro((unsigned long)header, header->pages); prog->bpf_func = (void *)image; prog->jited = 1; + } else { + prog = orig_prog; } out_addrs: diff --git a/arch/x86/pci/acpi.c b/arch/x86/pci/acpi.c index 3cd69832d7f4c6..3961103e91760a 100644 --- a/arch/x86/pci/acpi.c +++ b/arch/x86/pci/acpi.c @@ -114,6 +114,16 @@ static const struct dmi_system_id pci_crs_quirks[] __initconst = { DMI_MATCH(DMI_BIOS_VERSION, "6JET85WW (1.43 )"), }, }, + /* https://bugzilla.kernel.org/show_bug.cgi?id=42606 */ + { + .callback = set_nouse_crs, + .ident = "Supermicro X8DTH", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Supermicro"), + DMI_MATCH(DMI_PRODUCT_NAME, "X8DTH-i/6/iF/6F"), + DMI_MATCH(DMI_BIOS_VERSION, "2.0a"), + }, + }, /* https://bugzilla.kernel.org/show_bug.cgi?id=15362 */ { diff --git a/arch/x86/platform/efi/efi.c b/arch/x86/platform/efi/efi.c index 936a488d6cf6df..274dfc48184977 100644 --- a/arch/x86/platform/efi/efi.c +++ b/arch/x86/platform/efi/efi.c @@ -210,6 +210,70 @@ int __init efi_memblock_x86_reserve_range(void) return 0; } +#define OVERFLOW_ADDR_SHIFT (64 - EFI_PAGE_SHIFT) +#define OVERFLOW_ADDR_MASK (U64_MAX << OVERFLOW_ADDR_SHIFT) +#define U64_HIGH_BIT (~(U64_MAX >> 1)) + +static bool __init efi_memmap_entry_valid(const efi_memory_desc_t *md, int i) +{ + u64 end = (md->num_pages << EFI_PAGE_SHIFT) + md->phys_addr - 1; + u64 end_hi = 0; + char buf[64]; + + if (md->num_pages == 0) { + end = 0; + } else if (md->num_pages > EFI_PAGES_MAX || + EFI_PAGES_MAX - md->num_pages < + (md->phys_addr >> EFI_PAGE_SHIFT)) { + end_hi = (md->num_pages & OVERFLOW_ADDR_MASK) + >> OVERFLOW_ADDR_SHIFT; + + if ((md->phys_addr & U64_HIGH_BIT) && !(end & U64_HIGH_BIT)) + end_hi += 1; + } else { + return true; + } + + pr_warn_once(FW_BUG "Invalid EFI memory map entries:\n"); + + if (end_hi) { + pr_warn("mem%02u: %s range=[0x%016llx-0x%llx%016llx] (invalid)\n", + i, efi_md_typeattr_format(buf, sizeof(buf), md), + md->phys_addr, end_hi, end); + } else { + pr_warn("mem%02u: %s range=[0x%016llx-0x%016llx] (invalid)\n", + i, efi_md_typeattr_format(buf, sizeof(buf), md), + md->phys_addr, end); + } + return false; +} + +static void __init efi_clean_memmap(void) +{ + efi_memory_desc_t *out = efi.memmap.map; + const efi_memory_desc_t *in = out; + const efi_memory_desc_t *end = efi.memmap.map_end; + int i, n_removal; + + for (i = n_removal = 0; in < end; i++) { + if (efi_memmap_entry_valid(in, i)) { + if (out != in) + memcpy(out, in, efi.memmap.desc_size); + out = (void *)out + efi.memmap.desc_size; + } else { + n_removal++; + } + in = (void *)in + efi.memmap.desc_size; + } + + if (n_removal > 0) { + u64 size = efi.memmap.nr_map - n_removal; + + pr_warn("Removing %d invalid memory map entries.\n", n_removal); + efi_memmap_install(efi.memmap.phys_map, size); + } +} + void __init efi_print_memmap(void) { efi_memory_desc_t *md; @@ -472,6 +536,8 @@ void __init efi_init(void) } } + efi_clean_memmap(); + if (efi_enabled(EFI_DBG)) efi_print_memmap(); } diff --git a/arch/x86/platform/efi/efi_64.c b/arch/x86/platform/efi/efi_64.c index 319148bd4b0509..2f25a363068cf9 100644 --- a/arch/x86/platform/efi/efi_64.c +++ b/arch/x86/platform/efi/efi_64.c @@ -268,6 +268,22 @@ int __init efi_setup_page_tables(unsigned long pa_memmap, unsigned num_pages) efi_scratch.use_pgd = true; + /* + * Certain firmware versions are way too sentimential and still believe + * they are exclusive and unquestionable owners of the first physical page, + * even though they explicitly mark it as EFI_CONVENTIONAL_MEMORY + * (but then write-access it later during SetVirtualAddressMap()). + * + * Create a 1:1 mapping for this page, to avoid triple faults during early + * boot with such firmware. We are free to hand this page to the BIOS, + * as trim_bios_range() will reserve the first page and isolate it away + * from memory allocators anyway. + */ + if (kernel_map_pages_in_pgd(pgd, 0x0, 0x0, 1, _PAGE_RW)) { + pr_err("Failed to create 1:1 mapping for the first page!\n"); + return 1; + } + /* * When making calls to the firmware everything needs to be 1:1 * mapped and addressable with 32-bit pointers. Map the kernel diff --git a/arch/x86/platform/efi/quirks.c b/arch/x86/platform/efi/quirks.c index 10aca63a50d7bb..30031d5293c483 100644 --- a/arch/x86/platform/efi/quirks.c +++ b/arch/x86/platform/efi/quirks.c @@ -214,7 +214,7 @@ void __init efi_arch_mem_reserve(phys_addr_t addr, u64 size) new_size = efi.memmap.desc_size * num_entries; - new_phys = memblock_alloc(new_size, 0); + new_phys = efi_memmap_alloc(num_entries); if (!new_phys) { pr_err("Could not allocate boot services memmap\n"); return; @@ -355,7 +355,7 @@ void __init efi_free_boot_services(void) } new_size = efi.memmap.desc_size * num_entries; - new_phys = memblock_alloc(new_size, 0); + new_phys = efi_memmap_alloc(num_entries); if (!new_phys) { pr_err("Failed to allocate new EFI memmap\n"); return; diff --git a/arch/x86/platform/mellanox/mlx-platform.c b/arch/x86/platform/mellanox/mlx-platform.c index 7dcfcca9739975..c0355d789fce22 100644 --- a/arch/x86/platform/mellanox/mlx-platform.c +++ b/arch/x86/platform/mellanox/mlx-platform.c @@ -233,7 +233,7 @@ static int __init mlxplat_init(void) return 0; fail_platform_mux_register: - for (i--; i > 0 ; i--) + while (--i >= 0) platform_device_unregister(priv->pdev_mux[i]); platform_device_unregister(priv->pdev_i2c); fail_alloc: diff --git a/arch/x86/xen/pci-swiotlb-xen.c b/arch/x86/xen/pci-swiotlb-xen.c index 0e98e5d241d048..5f8b4b0302b61f 100644 --- a/arch/x86/xen/pci-swiotlb-xen.c +++ b/arch/x86/xen/pci-swiotlb-xen.c @@ -49,7 +49,7 @@ int __init pci_xen_swiotlb_detect(void) * activate this IOMMU. If running as PV privileged, activate it * irregardless. */ - if ((xen_initial_domain() || swiotlb || swiotlb_force)) + if (xen_initial_domain() || swiotlb || swiotlb_force == SWIOTLB_FORCE) xen_swiotlb = 1; /* If we are running under Xen, we MUST disable the native SWIOTLB. diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c index 9fa27ceeecfdae..311acad7dad2ab 100644 --- a/arch/x86/xen/smp.c +++ b/arch/x86/xen/smp.c @@ -87,12 +87,6 @@ static void cpu_bringup(void) cpu_data(cpu).x86_max_cores = 1; set_cpu_sibling_map(cpu); - /* - * identify_cpu() may have set logical_pkg_id to -1 due - * to incorrect phys_proc_id. Let's re-comupte it. - */ - topology_update_package_map(apic->cpu_present_to_apicid(cpu), cpu); - xen_setup_cpu_clockevents(); notify_cpu_starting(cpu); diff --git a/arch/xtensa/kernel/setup.c b/arch/xtensa/kernel/setup.c index 88a044af7504dd..32cdc2c52e98c2 100644 --- a/arch/xtensa/kernel/setup.c +++ b/arch/xtensa/kernel/setup.c @@ -540,7 +540,7 @@ subsys_initcall(topology_init); void cpu_reset(void) { -#if XCHAL_HAVE_PTP_MMU +#if XCHAL_HAVE_PTP_MMU && IS_ENABLED(CONFIG_MMU) local_irq_disable(); /* * We have full MMU: all autoload ways, ways 7, 8 and 9 of DTLB must diff --git a/block/blk-core.c b/block/blk-core.c index 14d7c0740dc07a..df9e160a5cdf83 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -40,6 +40,8 @@ #include "blk.h" #include "blk-mq.h" +#include + EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_remap); EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap); EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_complete); @@ -3547,3 +3549,85 @@ int __init blk_dev_init(void) return 0; } + +/* + * Blk IO latency support. We want this to be as cheap as possible, so doing + * this lockless (and avoiding atomics), a few off by a few errors in this + * code is not harmful, and we don't want to do anything that is + * perf-impactful. + * TODO : If necessary, we can make the histograms per-cpu and aggregate + * them when printing them out. + */ +void +blk_zero_latency_hist(struct io_latency_state *s) +{ + memset(s->latency_y_axis_read, 0, + sizeof(s->latency_y_axis_read)); + memset(s->latency_y_axis_write, 0, + sizeof(s->latency_y_axis_write)); + s->latency_reads_elems = 0; + s->latency_writes_elems = 0; +} +EXPORT_SYMBOL(blk_zero_latency_hist); + +ssize_t +blk_latency_hist_show(struct io_latency_state *s, char *buf) +{ + int i; + int bytes_written = 0; + u_int64_t num_elem, elem; + int pct; + + num_elem = s->latency_reads_elems; + if (num_elem > 0) { + bytes_written += scnprintf(buf + bytes_written, + PAGE_SIZE - bytes_written, + "IO svc_time Read Latency Histogram (n = %llu):\n", + num_elem); + for (i = 0; + i < ARRAY_SIZE(latency_x_axis_us); + i++) { + elem = s->latency_y_axis_read[i]; + pct = div64_u64(elem * 100, num_elem); + bytes_written += scnprintf(buf + bytes_written, + PAGE_SIZE - bytes_written, + "\t< %5lluus%15llu%15d%%\n", + latency_x_axis_us[i], + elem, pct); + } + /* Last element in y-axis table is overflow */ + elem = s->latency_y_axis_read[i]; + pct = div64_u64(elem * 100, num_elem); + bytes_written += scnprintf(buf + bytes_written, + PAGE_SIZE - bytes_written, + "\t> %5dms%15llu%15d%%\n", 10, + elem, pct); + } + num_elem = s->latency_writes_elems; + if (num_elem > 0) { + bytes_written += scnprintf(buf + bytes_written, + PAGE_SIZE - bytes_written, + "IO svc_time Write Latency Histogram (n = %llu):\n", + num_elem); + for (i = 0; + i < ARRAY_SIZE(latency_x_axis_us); + i++) { + elem = s->latency_y_axis_write[i]; + pct = div64_u64(elem * 100, num_elem); + bytes_written += scnprintf(buf + bytes_written, + PAGE_SIZE - bytes_written, + "\t< %5lluus%15llu%15d%%\n", + latency_x_axis_us[i], + elem, pct); + } + /* Last element in y-axis table is overflow */ + elem = s->latency_y_axis_write[i]; + pct = div64_u64(elem * 100, num_elem); + bytes_written += scnprintf(buf + bytes_written, + PAGE_SIZE - bytes_written, + "\t> %5dms%15llu%15d%%\n", 10, + elem, pct); + } + return bytes_written; +} +EXPORT_SYMBOL(blk_latency_hist_show); diff --git a/block/blk-mq.c b/block/blk-mq.c index f3d27a6dee09df..81caceb96c3c1c 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -895,7 +895,7 @@ static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx) return WORK_CPU_UNBOUND; if (--hctx->next_cpu_batch <= 0) { - int cpu = hctx->next_cpu, next_cpu; + int next_cpu; next_cpu = cpumask_next(hctx->next_cpu, hctx->cpumask); if (next_cpu >= nr_cpu_ids) @@ -903,8 +903,6 @@ static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx) hctx->next_cpu = next_cpu; hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH; - - return cpu; } return hctx->next_cpu; @@ -1332,9 +1330,9 @@ static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio) blk_mq_put_ctx(data.ctx); if (!old_rq) goto done; - if (!blk_mq_direct_issue_request(old_rq, &cookie)) - goto done; - blk_mq_insert_request(old_rq, false, true, true); + if (test_bit(BLK_MQ_S_STOPPED, &data.hctx->state) || + blk_mq_direct_issue_request(old_rq, &cookie) != 0) + blk_mq_insert_request(old_rq, false, true, true); goto done; } diff --git a/block/bsg.c b/block/bsg.c index d214e929ce1855..b9a53615bdef49 100644 --- a/block/bsg.c +++ b/block/bsg.c @@ -655,6 +655,9 @@ bsg_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) dprintk("%s: write %Zd bytes\n", bd->name, count); + if (unlikely(segment_eq(get_fs(), KERNEL_DS))) + return -EINVAL; + bsg_set_block(bd, file); bytes_written = 0; diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index 5e24d880306c2a..3ab6807773eeb8 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c @@ -1596,7 +1596,7 @@ static struct blkcg_policy_data *cfq_cpd_alloc(gfp_t gfp) { struct cfq_group_data *cgd; - cgd = kzalloc(sizeof(*cgd), GFP_KERNEL); + cgd = kzalloc(sizeof(*cgd), gfp); if (!cgd) return NULL; return &cgd->cpd; diff --git a/build.config b/build.config new file mode 100644 index 00000000000000..b0ae146fcf9657 --- /dev/null +++ b/build.config @@ -0,0 +1,14 @@ +ARCH=arm64 +BRANCH=mirror-aosp-android-hikey-linaro-4.9 +CROSS_COMPILE=aarch64-linux-android- +DEFCONFIG=hikey_defconfig +EXTRA_CMDS='' +KERNEL_DIR=hikey-linaro +LINUX_GCC_CROSS_COMPILE_PREBUILTS_BIN=prebuilts/gcc/linux-x86/aarch64/aarch64-linux-android-4.9/bin +FILES=" +arch/arm64/boot/Image +arch/arm64/boot/Image-dtb +arch/arm64/boot/dts/hisilicon/hi6220-hikey.dtb +vmlinux +System.map +" diff --git a/build.config.goldfish.arm b/build.config.goldfish.arm new file mode 100644 index 00000000000000..866da9361b7110 --- /dev/null +++ b/build.config.goldfish.arm @@ -0,0 +1,12 @@ +ARCH=arm +BRANCH=android-4.4 +CROSS_COMPILE=arm-linux-androidkernel- +DEFCONFIG=ranchu_defconfig +EXTRA_CMDS='' +KERNEL_DIR=common +LINUX_GCC_CROSS_COMPILE_PREBUILTS_BIN=prebuilts/gcc/linux-x86/arm/arm-linux-androideabi-4.9/bin +FILES=" +arch/arm/boot/zImage +vmlinux +System.map +" diff --git a/build.config.goldfish.arm64 b/build.config.goldfish.arm64 new file mode 100644 index 00000000000000..9c963cf4a3d8d7 --- /dev/null +++ b/build.config.goldfish.arm64 @@ -0,0 +1,12 @@ +ARCH=arm64 +BRANCH=android-4.4 +CROSS_COMPILE=aarch64-linux-android- +DEFCONFIG=ranchu64_defconfig +EXTRA_CMDS='' +KERNEL_DIR=common +LINUX_GCC_CROSS_COMPILE_PREBUILTS_BIN=prebuilts/gcc/linux-x86/aarch64/aarch64-linux-android-4.9/bin +FILES=" +arch/arm64/boot/Image +vmlinux +System.map +" diff --git a/build.config.goldfish.mips b/build.config.goldfish.mips new file mode 100644 index 00000000000000..8af53d2c294048 --- /dev/null +++ b/build.config.goldfish.mips @@ -0,0 +1,11 @@ +ARCH=mips +BRANCH=android-4.4 +CROSS_COMPILE=mips64el-linux-android- +DEFCONFIG=ranchu_defconfig +EXTRA_CMDS='' +KERNEL_DIR=common +LINUX_GCC_CROSS_COMPILE_PREBUILTS_BIN=prebuilts/gcc/linux-x86/mips/mips64el-linux-android-4.9/bin +FILES=" +vmlinux +System.map +" diff --git a/build.config.goldfish.mips64 b/build.config.goldfish.mips64 new file mode 100644 index 00000000000000..2a33d36dc4c8e8 --- /dev/null +++ b/build.config.goldfish.mips64 @@ -0,0 +1,11 @@ +ARCH=mips +BRANCH=android-4.4 +CROSS_COMPILE=mips64el-linux-android- +DEFCONFIG=ranchu64_defconfig +EXTRA_CMDS='' +KERNEL_DIR=common +LINUX_GCC_CROSS_COMPILE_PREBUILTS_BIN=prebuilts/gcc/linux-x86/mips/mips64el-linux-android-4.9/bin +FILES=" +vmlinux +System.map +" diff --git a/build.config.goldfish.x86 b/build.config.goldfish.x86 new file mode 100644 index 00000000000000..f86253f58d4d17 --- /dev/null +++ b/build.config.goldfish.x86 @@ -0,0 +1,12 @@ +ARCH=x86 +BRANCH=android-4.4 +CROSS_COMPILE=x86_64-linux-android- +DEFCONFIG=i386_ranchu_defconfig +EXTRA_CMDS='' +KERNEL_DIR=common +LINUX_GCC_CROSS_COMPILE_PREBUILTS_BIN=prebuilts/gcc/linux-x86/x86/x86_64-linux-android-4.9/bin +FILES=" +arch/x86/boot/bzImage +vmlinux +System.map +" diff --git a/build.config.goldfish.x86_64 b/build.config.goldfish.x86_64 new file mode 100644 index 00000000000000..e1738861ec5c25 --- /dev/null +++ b/build.config.goldfish.x86_64 @@ -0,0 +1,12 @@ +ARCH=x86_64 +BRANCH=android-4.4 +CROSS_COMPILE=x86_64-linux-android- +DEFCONFIG=x86_64_ranchu_defconfig +EXTRA_CMDS='' +KERNEL_DIR=common +LINUX_GCC_CROSS_COMPILE_PREBUILTS_BIN=prebuilts/gcc/linux-x86/x86/x86_64-linux-android-4.9/bin +FILES=" +arch/x86/boot/bzImage +vmlinux +System.map +" diff --git a/build.config.net_test b/build.config.net_test new file mode 100644 index 00000000000000..e1808d938ee563 --- /dev/null +++ b/build.config.net_test @@ -0,0 +1,9 @@ +ARCH=um +SUBARCH=x86_64 +CROSS_COMPILE= +EXTRA_CMDS='' +KERNEL_DIR=hikey-linaro +LINUX_GCC_CROSS_COMPILE_PREBUILTS_BIN= +FILES=" +linux +" diff --git a/build_test.sh b/build_test.sh new file mode 100755 index 00000000000000..654e65c39e18d2 --- /dev/null +++ b/build_test.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# Usage: +# build/build_test.sh + +export MAKE_ARGS=$@ +export ROOT_DIR=$(dirname $(readlink -f $0)) +export NET_TEST=${ROOT_DIR}/../kernel/tests/net/test +export BUILD_CONFIG=build/build.config.net_test + +test=all_tests.sh +set -e +source ${ROOT_DIR}/envsetup.sh + +echo "========================================================" +echo " Building kernel and running tests " + +cd ${KERNEL_DIR} +$NET_TEST/run_net_test.sh --builder $test +echo $? +echo "======Finished running tests======" diff --git a/crypto/algapi.c b/crypto/algapi.c index df939b54b09f73..1fad2a6b3bbbf0 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -356,6 +356,7 @@ int crypto_register_alg(struct crypto_alg *alg) struct crypto_larval *larval; int err; + alg->cra_flags &= ~CRYPTO_ALG_DEAD; err = crypto_check_alg(alg); if (err) return err; diff --git a/crypto/algif_aead.c b/crypto/algif_aead.c index e9c0993b131dc3..e8817e2f0597ea 100644 --- a/crypto/algif_aead.c +++ b/crypto/algif_aead.c @@ -671,9 +671,9 @@ static int aead_recvmsg_sync(struct socket *sock, struct msghdr *msg, int flags) unlock: list_for_each_entry_safe(rsgl, tmp, &ctx->list, list) { af_alg_free_sg(&rsgl->sgl); + list_del(&rsgl->list); if (rsgl != &ctx->first_rsgl) sock_kfree_s(sk, rsgl, sizeof(*rsgl)); - list_del(&rsgl->list); } INIT_LIST_HEAD(&ctx->list); aead_wmem_wakeup(sk); diff --git a/drivers/Kconfig b/drivers/Kconfig index e1e2066cecdb60..de581c13ec9ad1 100644 --- a/drivers/Kconfig +++ b/drivers/Kconfig @@ -202,4 +202,6 @@ source "drivers/hwtracing/intel_th/Kconfig" source "drivers/fpga/Kconfig" +source "drivers/tee/Kconfig" + endmenu diff --git a/drivers/Makefile b/drivers/Makefile index 194d20bee7dce4..4d5336794fcdf2 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -173,3 +173,4 @@ obj-$(CONFIG_STM) += hwtracing/stm/ obj-$(CONFIG_ANDROID) += android/ obj-$(CONFIG_NVMEM) += nvmem/ obj-$(CONFIG_FPGA) += fpga/ +obj-$(CONFIG_TEE) += tee/ diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c index 0d099a24f776ac..e53bef6cf53c62 100644 --- a/drivers/acpi/apei/ghes.c +++ b/drivers/acpi/apei/ghes.c @@ -852,6 +852,8 @@ static int ghes_notify_nmi(unsigned int cmd, struct pt_regs *regs) if (ghes_read_estatus(ghes, 1)) { ghes_clear_estatus(ghes); continue; + } else { + ret = NMI_HANDLED; } sev = ghes_severity(ghes->estatus->error_severity); @@ -863,12 +865,11 @@ static int ghes_notify_nmi(unsigned int cmd, struct pt_regs *regs) __process_error(ghes); ghes_clear_estatus(ghes); - - ret = NMI_HANDLED; } #ifdef CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG - irq_work_queue(&ghes_proc_irq_work); + if (ret == NMI_HANDLED) + irq_work_queue(&ghes_proc_irq_work); #endif atomic_dec(&ghes_in_nmi); return ret; diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c index d0d0504b7c895b..e0ea8f56d2bfd9 100644 --- a/drivers/acpi/cppc_acpi.c +++ b/drivers/acpi/cppc_acpi.c @@ -784,8 +784,10 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr) /* Add per logical CPU nodes for reading its feedback counters. */ cpu_dev = get_cpu_device(pr->id); - if (!cpu_dev) + if (!cpu_dev) { + ret = -EINVAL; goto out_free; + } ret = kobject_init_and_add(&cpc_ptr->kobj, &cppc_ktype, &cpu_dev->kobj, "acpi_cppc"); diff --git a/drivers/acpi/nfit/core.c b/drivers/acpi/nfit/core.c index 312c4b4dc363fd..6eb6733a7a5c9d 100644 --- a/drivers/acpi/nfit/core.c +++ b/drivers/acpi/nfit/core.c @@ -2704,6 +2704,7 @@ static int acpi_nfit_flush_probe(struct nvdimm_bus_descriptor *nd_desc) struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc); struct device *dev = acpi_desc->dev; struct acpi_nfit_flush_work flush; + int rc; /* bounce the device lock to flush acpi_nfit_add / acpi_nfit_notify */ device_lock(dev); @@ -2716,7 +2717,10 @@ static int acpi_nfit_flush_probe(struct nvdimm_bus_descriptor *nd_desc) INIT_WORK_ONSTACK(&flush.work, flush_probe); COMPLETION_INITIALIZER_ONSTACK(flush.cmp); queue_work(nfit_wq, &flush.work); - return wait_for_completion_interruptible(&flush.cmp); + + rc = wait_for_completion_interruptible(&flush.cmp); + cancel_work_sync(&flush.work); + return rc; } static int acpi_nfit_clear_to_send(struct nvdimm_bus_descriptor *nd_desc, diff --git a/drivers/acpi/video_detect.c b/drivers/acpi/video_detect.c index a6b36fc53aec54..02ded25c82e4a0 100644 --- a/drivers/acpi/video_detect.c +++ b/drivers/acpi/video_detect.c @@ -296,6 +296,26 @@ static const struct dmi_system_id video_detect_dmi_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "Vostro V131"), }, }, + { + /* https://bugzilla.redhat.com/show_bug.cgi?id=1123661 */ + .callback = video_detect_force_native, + .ident = "Dell XPS 17 L702X", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_MATCH(DMI_PRODUCT_NAME, "Dell System XPS L702X"), + }, + }, + { + /* https://bugzilla.redhat.com/show_bug.cgi?id=1204476 */ + /* https://bugs.launchpad.net/ubuntu/+source/linux-lts-trusty/+bug/1416940 */ + .callback = video_detect_force_native, + .ident = "HP Pavilion dv6", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Hewlett-Packard"), + DMI_MATCH(DMI_PRODUCT_NAME, "HP Pavilion dv6 Notebook PC"), + }, + }, + { }, }; diff --git a/drivers/android/Kconfig b/drivers/android/Kconfig index bdfc6c6f4f5a7c..a82fc022d34ba8 100644 --- a/drivers/android/Kconfig +++ b/drivers/android/Kconfig @@ -19,6 +19,18 @@ config ANDROID_BINDER_IPC Android process, using Binder to identify, invoke and pass arguments between said processes. +config ANDROID_BINDER_DEVICES + string "Android Binder devices" + depends on ANDROID_BINDER_IPC + default "binder" + ---help--- + Default value for the binder.devices parameter. + + The binder.devices parameter is a comma-separated list of strings + that specifies the names of the binder device nodes that will be + created. Each binder device has its own context manager, and is + therefore logically separated from the other devices. + config ANDROID_BINDER_IPC_32BIT bool depends on !64BIT && ANDROID_BINDER_IPC diff --git a/drivers/android/binder.c b/drivers/android/binder.c index 3c71b982bf2a35..1bd840198c28f0 100644 --- a/drivers/android/binder.c +++ b/drivers/android/binder.c @@ -50,14 +50,13 @@ static DEFINE_MUTEX(binder_main_lock); static DEFINE_MUTEX(binder_deferred_lock); static DEFINE_MUTEX(binder_mmap_lock); +static HLIST_HEAD(binder_devices); static HLIST_HEAD(binder_procs); static HLIST_HEAD(binder_deferred_list); static HLIST_HEAD(binder_dead_nodes); static struct dentry *binder_debugfs_dir_entry_root; static struct dentry *binder_debugfs_dir_entry_proc; -static struct binder_node *binder_context_mgr_node; -static kuid_t binder_context_mgr_uid = INVALID_UID; static int binder_last_id; #define BINDER_DEBUG_ENTRY(name) \ @@ -115,6 +114,9 @@ module_param_named(debug_mask, binder_debug_mask, uint, S_IWUSR | S_IRUGO); static bool binder_debug_no_lock; module_param_named(proc_no_lock, binder_debug_no_lock, bool, S_IWUSR | S_IRUGO); +static char *binder_devices_param = CONFIG_ANDROID_BINDER_DEVICES; +module_param_named(devices, binder_devices_param, charp, S_IRUGO); + static DECLARE_WAIT_QUEUE_HEAD(binder_user_error_wait); static int binder_stop_on_user_error; @@ -145,6 +147,17 @@ module_param_call(stop_on_user_error, binder_set_stop_on_user_error, binder_stop_on_user_error = 2; \ } while (0) +#define to_flat_binder_object(hdr) \ + container_of(hdr, struct flat_binder_object, hdr) + +#define to_binder_fd_object(hdr) container_of(hdr, struct binder_fd_object, hdr) + +#define to_binder_buffer_object(hdr) \ + container_of(hdr, struct binder_buffer_object, hdr) + +#define to_binder_fd_array_object(hdr) \ + container_of(hdr, struct binder_fd_array_object, hdr) + enum binder_stat_types { BINDER_STAT_PROC, BINDER_STAT_THREAD, @@ -158,7 +171,7 @@ enum binder_stat_types { struct binder_stats { int br[_IOC_NR(BR_FAILED_REPLY) + 1]; - int bc[_IOC_NR(BC_DEAD_BINDER_DONE) + 1]; + int bc[_IOC_NR(BC_REPLY_SG) + 1]; int obj_created[BINDER_STAT_COUNT]; int obj_deleted[BINDER_STAT_COUNT]; }; @@ -186,6 +199,7 @@ struct binder_transaction_log_entry { int to_node; int data_size; int offsets_size; + const char *context_name; }; struct binder_transaction_log { int next; @@ -210,6 +224,18 @@ static struct binder_transaction_log_entry *binder_transaction_log_add( return e; } +struct binder_context { + struct binder_node *binder_context_mgr_node; + kuid_t binder_context_mgr_uid; + const char *name; +}; + +struct binder_device { + struct hlist_node hlist; + struct miscdevice miscdev; + struct binder_context context; +}; + struct binder_work { struct list_head entry; enum { @@ -282,6 +308,7 @@ struct binder_buffer { struct binder_node *target_node; size_t data_size; size_t offsets_size; + size_t extra_buffers_size; uint8_t data[0]; }; @@ -325,6 +352,7 @@ struct binder_proc { int ready_threads; long default_priority; struct dentry *debugfs_entry; + struct binder_context *context; }; enum { @@ -648,7 +676,9 @@ static int binder_update_page_range(struct binder_proc *proc, int allocate, static struct binder_buffer *binder_alloc_buf(struct binder_proc *proc, size_t data_size, - size_t offsets_size, int is_async) + size_t offsets_size, + size_t extra_buffers_size, + int is_async) { struct rb_node *n = proc->free_buffers.rb_node; struct binder_buffer *buffer; @@ -656,7 +686,7 @@ static struct binder_buffer *binder_alloc_buf(struct binder_proc *proc, struct rb_node *best_fit = NULL; void *has_page_addr; void *end_page_addr; - size_t size; + size_t size, data_offsets_size; if (proc->vma == NULL) { pr_err("%d: binder_alloc_buf, no vma\n", @@ -664,15 +694,20 @@ static struct binder_buffer *binder_alloc_buf(struct binder_proc *proc, return NULL; } - size = ALIGN(data_size, sizeof(void *)) + + data_offsets_size = ALIGN(data_size, sizeof(void *)) + ALIGN(offsets_size, sizeof(void *)); - if (size < data_size || size < offsets_size) { + if (data_offsets_size < data_size || data_offsets_size < offsets_size) { binder_user_error("%d: got transaction with invalid size %zd-%zd\n", proc->pid, data_size, offsets_size); return NULL; } - + size = data_offsets_size + ALIGN(extra_buffers_size, sizeof(void *)); + if (size < data_offsets_size || size < extra_buffers_size) { + binder_user_error("%d: got transaction with invalid extra_buffers_size %zd\n", + proc->pid, extra_buffers_size); + return NULL; + } if (is_async && proc->free_async_space < size + sizeof(struct binder_buffer)) { binder_debug(BINDER_DEBUG_BUFFER_ALLOC, @@ -741,6 +776,7 @@ static struct binder_buffer *binder_alloc_buf(struct binder_proc *proc, proc->pid, size, buffer); buffer->data_size = data_size; buffer->offsets_size = offsets_size; + buffer->extra_buffers_size = extra_buffers_size; buffer->async_transaction = is_async; if (is_async) { proc->free_async_space -= size + sizeof(struct binder_buffer); @@ -815,7 +851,8 @@ static void binder_free_buf(struct binder_proc *proc, buffer_size = binder_buffer_size(proc, buffer); size = ALIGN(buffer->data_size, sizeof(void *)) + - ALIGN(buffer->offsets_size, sizeof(void *)); + ALIGN(buffer->offsets_size, sizeof(void *)) + + ALIGN(buffer->extra_buffers_size, sizeof(void *)); binder_debug(BINDER_DEBUG_BUFFER_ALLOC, "%d: binder_free_buf %p size %zd buffer_size %zd\n", @@ -929,8 +966,10 @@ static int binder_inc_node(struct binder_node *node, int strong, int internal, if (internal) { if (target_list == NULL && node->internal_strong_refs == 0 && - !(node == binder_context_mgr_node && - node->has_strong_ref)) { + !(node->proc && + node == node->proc->context-> + binder_context_mgr_node && + node->has_strong_ref)) { pr_err("invalid inc strong node for %d\n", node->debug_id); return -EINVAL; @@ -1031,6 +1070,7 @@ static struct binder_ref *binder_get_ref_for_node(struct binder_proc *proc, struct rb_node **p = &proc->refs_by_node.rb_node; struct rb_node *parent = NULL; struct binder_ref *ref, *new_ref; + struct binder_context *context = proc->context; while (*p) { parent = *p; @@ -1053,7 +1093,7 @@ static struct binder_ref *binder_get_ref_for_node(struct binder_proc *proc, rb_link_node(&new_ref->rb_node_node, parent, p); rb_insert_color(&new_ref->rb_node_node, &proc->refs_by_node); - new_ref->desc = (node == binder_context_mgr_node) ? 0 : 1; + new_ref->desc = (node == context->binder_context_mgr_node) ? 0 : 1; for (n = rb_first(&proc->refs_by_desc); n != NULL; n = rb_next(n)) { ref = rb_entry(n, struct binder_ref, rb_node_desc); if (ref->desc > new_ref->desc) @@ -1240,11 +1280,158 @@ static void binder_send_failed_reply(struct binder_transaction *t, } } +/** + * binder_validate_object() - checks for a valid metadata object in a buffer. + * @buffer: binder_buffer that we're parsing. + * @offset: offset in the buffer at which to validate an object. + * + * Return: If there's a valid metadata object at @offset in @buffer, the + * size of that object. Otherwise, it returns zero. + */ +static size_t binder_validate_object(struct binder_buffer *buffer, u64 offset) +{ + /* Check if we can read a header first */ + struct binder_object_header *hdr; + size_t object_size = 0; + + if (offset > buffer->data_size - sizeof(*hdr) || + buffer->data_size < sizeof(*hdr) || + !IS_ALIGNED(offset, sizeof(u32))) + return 0; + + /* Ok, now see if we can read a complete object. */ + hdr = (struct binder_object_header *)(buffer->data + offset); + switch (hdr->type) { + case BINDER_TYPE_BINDER: + case BINDER_TYPE_WEAK_BINDER: + case BINDER_TYPE_HANDLE: + case BINDER_TYPE_WEAK_HANDLE: + object_size = sizeof(struct flat_binder_object); + break; + case BINDER_TYPE_FD: + object_size = sizeof(struct binder_fd_object); + break; + case BINDER_TYPE_PTR: + object_size = sizeof(struct binder_buffer_object); + break; + case BINDER_TYPE_FDA: + object_size = sizeof(struct binder_fd_array_object); + break; + default: + return 0; + } + if (offset <= buffer->data_size - object_size && + buffer->data_size >= object_size) + return object_size; + else + return 0; +} + +/** + * binder_validate_ptr() - validates binder_buffer_object in a binder_buffer. + * @b: binder_buffer containing the object + * @index: index in offset array at which the binder_buffer_object is + * located + * @start: points to the start of the offset array + * @num_valid: the number of valid offsets in the offset array + * + * Return: If @index is within the valid range of the offset array + * described by @start and @num_valid, and if there's a valid + * binder_buffer_object at the offset found in index @index + * of the offset array, that object is returned. Otherwise, + * %NULL is returned. + * Note that the offset found in index @index itself is not + * verified; this function assumes that @num_valid elements + * from @start were previously verified to have valid offsets. + */ +static struct binder_buffer_object *binder_validate_ptr(struct binder_buffer *b, + binder_size_t index, + binder_size_t *start, + binder_size_t num_valid) +{ + struct binder_buffer_object *buffer_obj; + binder_size_t *offp; + + if (index >= num_valid) + return NULL; + + offp = start + index; + buffer_obj = (struct binder_buffer_object *)(b->data + *offp); + if (buffer_obj->hdr.type != BINDER_TYPE_PTR) + return NULL; + + return buffer_obj; +} + +/** + * binder_validate_fixup() - validates pointer/fd fixups happen in order. + * @b: transaction buffer + * @objects_start start of objects buffer + * @buffer: binder_buffer_object in which to fix up + * @offset: start offset in @buffer to fix up + * @last_obj: last binder_buffer_object that we fixed up in + * @last_min_offset: minimum fixup offset in @last_obj + * + * Return: %true if a fixup in buffer @buffer at offset @offset is + * allowed. + * + * For safety reasons, we only allow fixups inside a buffer to happen + * at increasing offsets; additionally, we only allow fixup on the last + * buffer object that was verified, or one of its parents. + * + * Example of what is allowed: + * + * A + * B (parent = A, offset = 0) + * C (parent = A, offset = 16) + * D (parent = C, offset = 0) + * E (parent = A, offset = 32) // min_offset is 16 (C.parent_offset) + * + * Examples of what is not allowed: + * + * Decreasing offsets within the same parent: + * A + * C (parent = A, offset = 16) + * B (parent = A, offset = 0) // decreasing offset within A + * + * Referring to a parent that wasn't the last object or any of its parents: + * A + * B (parent = A, offset = 0) + * C (parent = A, offset = 0) + * C (parent = A, offset = 16) + * D (parent = B, offset = 0) // B is not A or any of A's parents + */ +static bool binder_validate_fixup(struct binder_buffer *b, + binder_size_t *objects_start, + struct binder_buffer_object *buffer, + binder_size_t fixup_offset, + struct binder_buffer_object *last_obj, + binder_size_t last_min_offset) +{ + if (!last_obj) { + /* Nothing to fix up in */ + return false; + } + + while (last_obj != buffer) { + /* + * Safe to retrieve the parent of last_obj, since it + * was already previously verified by the driver. + */ + if ((last_obj->flags & BINDER_BUFFER_FLAG_HAS_PARENT) == 0) + return false; + last_min_offset = last_obj->parent_offset + sizeof(uintptr_t); + last_obj = (struct binder_buffer_object *) + (b->data + *(objects_start + last_obj->parent)); + } + return (fixup_offset >= last_min_offset); +} + static void binder_transaction_buffer_release(struct binder_proc *proc, struct binder_buffer *buffer, binder_size_t *failed_at) { - binder_size_t *offp, *off_end; + binder_size_t *offp, *off_start, *off_end; int debug_id = buffer->debug_id; binder_debug(BINDER_DEBUG_TRANSACTION, @@ -1255,28 +1442,30 @@ static void binder_transaction_buffer_release(struct binder_proc *proc, if (buffer->target_node) binder_dec_node(buffer->target_node, 1, 0); - offp = (binder_size_t *)(buffer->data + - ALIGN(buffer->data_size, sizeof(void *))); + off_start = (binder_size_t *)(buffer->data + + ALIGN(buffer->data_size, sizeof(void *))); if (failed_at) off_end = failed_at; else - off_end = (void *)offp + buffer->offsets_size; - for (; offp < off_end; offp++) { - struct flat_binder_object *fp; + off_end = (void *)off_start + buffer->offsets_size; + for (offp = off_start; offp < off_end; offp++) { + struct binder_object_header *hdr; + size_t object_size = binder_validate_object(buffer, *offp); - if (*offp > buffer->data_size - sizeof(*fp) || - buffer->data_size < sizeof(*fp) || - !IS_ALIGNED(*offp, sizeof(u32))) { - pr_err("transaction release %d bad offset %lld, size %zd\n", + if (object_size == 0) { + pr_err("transaction release %d bad object at offset %lld, size %zd\n", debug_id, (u64)*offp, buffer->data_size); continue; } - fp = (struct flat_binder_object *)(buffer->data + *offp); - switch (fp->type) { + hdr = (struct binder_object_header *)(buffer->data + *offp); + switch (hdr->type) { case BINDER_TYPE_BINDER: case BINDER_TYPE_WEAK_BINDER: { - struct binder_node *node = binder_get_node(proc, fp->binder); + struct flat_binder_object *fp; + struct binder_node *node; + fp = to_flat_binder_object(hdr); + node = binder_get_node(proc, fp->binder); if (node == NULL) { pr_err("transaction release %d bad node %016llx\n", debug_id, (u64)fp->binder); @@ -1285,15 +1474,17 @@ static void binder_transaction_buffer_release(struct binder_proc *proc, binder_debug(BINDER_DEBUG_TRANSACTION, " node %d u%016llx\n", node->debug_id, (u64)node->ptr); - binder_dec_node(node, fp->type == BINDER_TYPE_BINDER, 0); + binder_dec_node(node, hdr->type == BINDER_TYPE_BINDER, + 0); } break; case BINDER_TYPE_HANDLE: case BINDER_TYPE_WEAK_HANDLE: { + struct flat_binder_object *fp; struct binder_ref *ref; + fp = to_flat_binder_object(hdr); ref = binder_get_ref(proc, fp->handle, - fp->type == BINDER_TYPE_HANDLE); - + hdr->type == BINDER_TYPE_HANDLE); if (ref == NULL) { pr_err("transaction release %d bad handle %d\n", debug_id, fp->handle); @@ -1302,32 +1493,348 @@ static void binder_transaction_buffer_release(struct binder_proc *proc, binder_debug(BINDER_DEBUG_TRANSACTION, " ref %d desc %d (node %d)\n", ref->debug_id, ref->desc, ref->node->debug_id); - binder_dec_ref(ref, fp->type == BINDER_TYPE_HANDLE); + binder_dec_ref(ref, hdr->type == BINDER_TYPE_HANDLE); } break; - case BINDER_TYPE_FD: + case BINDER_TYPE_FD: { + struct binder_fd_object *fp = to_binder_fd_object(hdr); + binder_debug(BINDER_DEBUG_TRANSACTION, - " fd %d\n", fp->handle); + " fd %d\n", fp->fd); if (failed_at) - task_close_fd(proc, fp->handle); + task_close_fd(proc, fp->fd); + } break; + case BINDER_TYPE_PTR: + /* + * Nothing to do here, this will get cleaned up when the + * transaction buffer gets freed + */ break; - + case BINDER_TYPE_FDA: { + struct binder_fd_array_object *fda; + struct binder_buffer_object *parent; + uintptr_t parent_buffer; + u32 *fd_array; + size_t fd_index; + binder_size_t fd_buf_size; + + fda = to_binder_fd_array_object(hdr); + parent = binder_validate_ptr(buffer, fda->parent, + off_start, + offp - off_start); + if (!parent) { + pr_err("transaction release %d bad parent offset", + debug_id); + continue; + } + /* + * Since the parent was already fixed up, convert it + * back to kernel address space to access it + */ + parent_buffer = parent->buffer - + proc->user_buffer_offset; + + fd_buf_size = sizeof(u32) * fda->num_fds; + if (fda->num_fds >= SIZE_MAX / sizeof(u32)) { + pr_err("transaction release %d invalid number of fds (%lld)\n", + debug_id, (u64)fda->num_fds); + continue; + } + if (fd_buf_size > parent->length || + fda->parent_offset > parent->length - fd_buf_size) { + /* No space for all file descriptors here. */ + pr_err("transaction release %d not enough space for %lld fds in buffer\n", + debug_id, (u64)fda->num_fds); + continue; + } + fd_array = (u32 *)(parent_buffer + fda->parent_offset); + for (fd_index = 0; fd_index < fda->num_fds; fd_index++) + task_close_fd(proc, fd_array[fd_index]); + } break; default: pr_err("transaction release %d bad object type %x\n", - debug_id, fp->type); + debug_id, hdr->type); break; } } } +static int binder_translate_binder(struct flat_binder_object *fp, + struct binder_transaction *t, + struct binder_thread *thread) +{ + struct binder_node *node; + struct binder_ref *ref; + struct binder_proc *proc = thread->proc; + struct binder_proc *target_proc = t->to_proc; + + node = binder_get_node(proc, fp->binder); + if (!node) { + node = binder_new_node(proc, fp->binder, fp->cookie); + if (!node) + return -ENOMEM; + + node->min_priority = fp->flags & FLAT_BINDER_FLAG_PRIORITY_MASK; + node->accept_fds = !!(fp->flags & FLAT_BINDER_FLAG_ACCEPTS_FDS); + } + if (fp->cookie != node->cookie) { + binder_user_error("%d:%d sending u%016llx node %d, cookie mismatch %016llx != %016llx\n", + proc->pid, thread->pid, (u64)fp->binder, + node->debug_id, (u64)fp->cookie, + (u64)node->cookie); + return -EINVAL; + } + if (security_binder_transfer_binder(proc->tsk, target_proc->tsk)) + return -EPERM; + + ref = binder_get_ref_for_node(target_proc, node); + if (!ref) + return -EINVAL; + + if (fp->hdr.type == BINDER_TYPE_BINDER) + fp->hdr.type = BINDER_TYPE_HANDLE; + else + fp->hdr.type = BINDER_TYPE_WEAK_HANDLE; + fp->binder = 0; + fp->handle = ref->desc; + fp->cookie = 0; + binder_inc_ref(ref, fp->hdr.type == BINDER_TYPE_HANDLE, &thread->todo); + + trace_binder_transaction_node_to_ref(t, node, ref); + binder_debug(BINDER_DEBUG_TRANSACTION, + " node %d u%016llx -> ref %d desc %d\n", + node->debug_id, (u64)node->ptr, + ref->debug_id, ref->desc); + + return 0; +} + +static int binder_translate_handle(struct flat_binder_object *fp, + struct binder_transaction *t, + struct binder_thread *thread) +{ + struct binder_ref *ref; + struct binder_proc *proc = thread->proc; + struct binder_proc *target_proc = t->to_proc; + + ref = binder_get_ref(proc, fp->handle, + fp->hdr.type == BINDER_TYPE_HANDLE); + if (!ref) { + binder_user_error("%d:%d got transaction with invalid handle, %d\n", + proc->pid, thread->pid, fp->handle); + return -EINVAL; + } + if (security_binder_transfer_binder(proc->tsk, target_proc->tsk)) + return -EPERM; + + if (ref->node->proc == target_proc) { + if (fp->hdr.type == BINDER_TYPE_HANDLE) + fp->hdr.type = BINDER_TYPE_BINDER; + else + fp->hdr.type = BINDER_TYPE_WEAK_BINDER; + fp->binder = ref->node->ptr; + fp->cookie = ref->node->cookie; + binder_inc_node(ref->node, fp->hdr.type == BINDER_TYPE_BINDER, + 0, NULL); + trace_binder_transaction_ref_to_node(t, ref); + binder_debug(BINDER_DEBUG_TRANSACTION, + " ref %d desc %d -> node %d u%016llx\n", + ref->debug_id, ref->desc, ref->node->debug_id, + (u64)ref->node->ptr); + } else { + struct binder_ref *new_ref; + + new_ref = binder_get_ref_for_node(target_proc, ref->node); + if (!new_ref) + return -EINVAL; + + fp->binder = 0; + fp->handle = new_ref->desc; + fp->cookie = 0; + binder_inc_ref(new_ref, fp->hdr.type == BINDER_TYPE_HANDLE, + NULL); + trace_binder_transaction_ref_to_ref(t, ref, new_ref); + binder_debug(BINDER_DEBUG_TRANSACTION, + " ref %d desc %d -> ref %d desc %d (node %d)\n", + ref->debug_id, ref->desc, new_ref->debug_id, + new_ref->desc, ref->node->debug_id); + } + return 0; +} + +static int binder_translate_fd(int fd, + struct binder_transaction *t, + struct binder_thread *thread, + struct binder_transaction *in_reply_to) +{ + struct binder_proc *proc = thread->proc; + struct binder_proc *target_proc = t->to_proc; + int target_fd; + struct file *file; + int ret; + bool target_allows_fd; + + if (in_reply_to) + target_allows_fd = !!(in_reply_to->flags & TF_ACCEPT_FDS); + else + target_allows_fd = t->buffer->target_node->accept_fds; + if (!target_allows_fd) { + binder_user_error("%d:%d got %s with fd, %d, but target does not allow fds\n", + proc->pid, thread->pid, + in_reply_to ? "reply" : "transaction", + fd); + ret = -EPERM; + goto err_fd_not_accepted; + } + + file = fget(fd); + if (!file) { + binder_user_error("%d:%d got transaction with invalid fd, %d\n", + proc->pid, thread->pid, fd); + ret = -EBADF; + goto err_fget; + } + ret = security_binder_transfer_file(proc->tsk, target_proc->tsk, file); + if (ret < 0) { + ret = -EPERM; + goto err_security; + } + + target_fd = task_get_unused_fd_flags(target_proc, O_CLOEXEC); + if (target_fd < 0) { + ret = -ENOMEM; + goto err_get_unused_fd; + } + task_fd_install(target_proc, target_fd, file); + trace_binder_transaction_fd(t, fd, target_fd); + binder_debug(BINDER_DEBUG_TRANSACTION, " fd %d -> %d\n", + fd, target_fd); + + return target_fd; + +err_get_unused_fd: +err_security: + fput(file); +err_fget: +err_fd_not_accepted: + return ret; +} + +static int binder_translate_fd_array(struct binder_fd_array_object *fda, + struct binder_buffer_object *parent, + struct binder_transaction *t, + struct binder_thread *thread, + struct binder_transaction *in_reply_to) +{ + binder_size_t fdi, fd_buf_size, num_installed_fds; + int target_fd; + uintptr_t parent_buffer; + u32 *fd_array; + struct binder_proc *proc = thread->proc; + struct binder_proc *target_proc = t->to_proc; + + fd_buf_size = sizeof(u32) * fda->num_fds; + if (fda->num_fds >= SIZE_MAX / sizeof(u32)) { + binder_user_error("%d:%d got transaction with invalid number of fds (%lld)\n", + proc->pid, thread->pid, (u64)fda->num_fds); + return -EINVAL; + } + if (fd_buf_size > parent->length || + fda->parent_offset > parent->length - fd_buf_size) { + /* No space for all file descriptors here. */ + binder_user_error("%d:%d not enough space to store %lld fds in buffer\n", + proc->pid, thread->pid, (u64)fda->num_fds); + return -EINVAL; + } + /* + * Since the parent was already fixed up, convert it + * back to the kernel address space to access it + */ + parent_buffer = parent->buffer - target_proc->user_buffer_offset; + fd_array = (u32 *)(parent_buffer + fda->parent_offset); + if (!IS_ALIGNED((unsigned long)fd_array, sizeof(u32))) { + binder_user_error("%d:%d parent offset not aligned correctly.\n", + proc->pid, thread->pid); + return -EINVAL; + } + for (fdi = 0; fdi < fda->num_fds; fdi++) { + target_fd = binder_translate_fd(fd_array[fdi], t, thread, + in_reply_to); + if (target_fd < 0) + goto err_translate_fd_failed; + fd_array[fdi] = target_fd; + } + return 0; + +err_translate_fd_failed: + /* + * Failed to allocate fd or security error, free fds + * installed so far. + */ + num_installed_fds = fdi; + for (fdi = 0; fdi < num_installed_fds; fdi++) + task_close_fd(target_proc, fd_array[fdi]); + return target_fd; +} + +static int binder_fixup_parent(struct binder_transaction *t, + struct binder_thread *thread, + struct binder_buffer_object *bp, + binder_size_t *off_start, + binder_size_t num_valid, + struct binder_buffer_object *last_fixup_obj, + binder_size_t last_fixup_min_off) +{ + struct binder_buffer_object *parent; + u8 *parent_buffer; + struct binder_buffer *b = t->buffer; + struct binder_proc *proc = thread->proc; + struct binder_proc *target_proc = t->to_proc; + + if (!(bp->flags & BINDER_BUFFER_FLAG_HAS_PARENT)) + return 0; + + parent = binder_validate_ptr(b, bp->parent, off_start, num_valid); + if (!parent) { + binder_user_error("%d:%d got transaction with invalid parent offset or type\n", + proc->pid, thread->pid); + return -EINVAL; + } + + if (!binder_validate_fixup(b, off_start, + parent, bp->parent_offset, + last_fixup_obj, + last_fixup_min_off)) { + binder_user_error("%d:%d got transaction with out-of-order buffer fixup\n", + proc->pid, thread->pid); + return -EINVAL; + } + + if (parent->length < sizeof(binder_uintptr_t) || + bp->parent_offset > parent->length - sizeof(binder_uintptr_t)) { + /* No space for a pointer here! */ + binder_user_error("%d:%d got transaction with invalid parent offset\n", + proc->pid, thread->pid); + return -EINVAL; + } + parent_buffer = (u8 *)(parent->buffer - + target_proc->user_buffer_offset); + *(binder_uintptr_t *)(parent_buffer + bp->parent_offset) = bp->buffer; + + return 0; +} + static void binder_transaction(struct binder_proc *proc, struct binder_thread *thread, - struct binder_transaction_data *tr, int reply) + struct binder_transaction_data *tr, int reply, + binder_size_t extra_buffers_size) { + int ret; struct binder_transaction *t; struct binder_work *tcomplete; - binder_size_t *offp, *off_end; + binder_size_t *offp, *off_end, *off_start; binder_size_t off_min; + u8 *sg_bufp, *sg_buf_end; struct binder_proc *target_proc; struct binder_thread *target_thread = NULL; struct binder_node *target_node = NULL; @@ -1336,6 +1843,9 @@ static void binder_transaction(struct binder_proc *proc, struct binder_transaction *in_reply_to = NULL; struct binder_transaction_log_entry *e; uint32_t return_error; + struct binder_buffer_object *last_fixup_obj = NULL; + binder_size_t last_fixup_min_off = 0; + struct binder_context *context = proc->context; e = binder_transaction_log_add(&binder_transaction_log); e->call_type = reply ? 2 : !!(tr->flags & TF_ONE_WAY); @@ -1344,6 +1854,7 @@ static void binder_transaction(struct binder_proc *proc, e->target_handle = tr->target.handle; e->data_size = tr->data_size; e->offsets_size = tr->offsets_size; + e->context_name = proc->context->name; if (reply) { in_reply_to = thread->transaction_stack; @@ -1396,7 +1907,7 @@ static void binder_transaction(struct binder_proc *proc, } target_node = ref->node; } else { - target_node = binder_context_mgr_node; + target_node = context->binder_context_mgr_node; if (target_node == NULL) { return_error = BR_DEAD_REPLY; goto err_no_context_mgr_node; @@ -1463,20 +1974,22 @@ static void binder_transaction(struct binder_proc *proc, if (reply) binder_debug(BINDER_DEBUG_TRANSACTION, - "%d:%d BC_REPLY %d -> %d:%d, data %016llx-%016llx size %lld-%lld\n", + "%d:%d BC_REPLY %d -> %d:%d, data %016llx-%016llx size %lld-%lld-%lld\n", proc->pid, thread->pid, t->debug_id, target_proc->pid, target_thread->pid, (u64)tr->data.ptr.buffer, (u64)tr->data.ptr.offsets, - (u64)tr->data_size, (u64)tr->offsets_size); + (u64)tr->data_size, (u64)tr->offsets_size, + (u64)extra_buffers_size); else binder_debug(BINDER_DEBUG_TRANSACTION, - "%d:%d BC_TRANSACTION %d -> %d - node %d, data %016llx-%016llx size %lld-%lld\n", + "%d:%d BC_TRANSACTION %d -> %d - node %d, data %016llx-%016llx size %lld-%lld-%lld\n", proc->pid, thread->pid, t->debug_id, target_proc->pid, target_node->debug_id, (u64)tr->data.ptr.buffer, (u64)tr->data.ptr.offsets, - (u64)tr->data_size, (u64)tr->offsets_size); + (u64)tr->data_size, (u64)tr->offsets_size, + (u64)extra_buffers_size); if (!reply && !(tr->flags & TF_ONE_WAY)) t->from = thread; @@ -1492,7 +2005,8 @@ static void binder_transaction(struct binder_proc *proc, trace_binder_transaction(reply, t, target_node); t->buffer = binder_alloc_buf(target_proc, tr->data_size, - tr->offsets_size, !reply && (t->flags & TF_ONE_WAY)); + tr->offsets_size, extra_buffers_size, + !reply && (t->flags & TF_ONE_WAY)); if (t->buffer == NULL) { return_error = BR_FAILED_REPLY; goto err_binder_alloc_buf_failed; @@ -1505,8 +2019,9 @@ static void binder_transaction(struct binder_proc *proc, if (target_node) binder_inc_node(target_node, 1, 0, NULL); - offp = (binder_size_t *)(t->buffer->data + - ALIGN(tr->data_size, sizeof(void *))); + off_start = (binder_size_t *)(t->buffer->data + + ALIGN(tr->data_size, sizeof(void *))); + offp = off_start; if (copy_from_user(t->buffer->data, (const void __user *)(uintptr_t) tr->data.ptr.buffer, tr->data_size)) { @@ -1528,177 +2043,138 @@ static void binder_transaction(struct binder_proc *proc, return_error = BR_FAILED_REPLY; goto err_bad_offset; } - off_end = (void *)offp + tr->offsets_size; + if (!IS_ALIGNED(extra_buffers_size, sizeof(u64))) { + binder_user_error("%d:%d got transaction with unaligned buffers size, %lld\n", + proc->pid, thread->pid, + (u64)extra_buffers_size); + return_error = BR_FAILED_REPLY; + goto err_bad_offset; + } + off_end = (void *)off_start + tr->offsets_size; + sg_bufp = (u8 *)(PTR_ALIGN(off_end, sizeof(void *))); + sg_buf_end = sg_bufp + extra_buffers_size; off_min = 0; for (; offp < off_end; offp++) { - struct flat_binder_object *fp; + struct binder_object_header *hdr; + size_t object_size = binder_validate_object(t->buffer, *offp); - if (*offp > t->buffer->data_size - sizeof(*fp) || - *offp < off_min || - t->buffer->data_size < sizeof(*fp) || - !IS_ALIGNED(*offp, sizeof(u32))) { - binder_user_error("%d:%d got transaction with invalid offset, %lld (min %lld, max %lld)\n", + if (object_size == 0 || *offp < off_min) { + binder_user_error("%d:%d got transaction with invalid offset (%lld, min %lld max %lld) or object.\n", proc->pid, thread->pid, (u64)*offp, (u64)off_min, - (u64)(t->buffer->data_size - - sizeof(*fp))); + (u64)t->buffer->data_size); return_error = BR_FAILED_REPLY; goto err_bad_offset; } - fp = (struct flat_binder_object *)(t->buffer->data + *offp); - off_min = *offp + sizeof(struct flat_binder_object); - switch (fp->type) { + + hdr = (struct binder_object_header *)(t->buffer->data + *offp); + off_min = *offp + object_size; + switch (hdr->type) { case BINDER_TYPE_BINDER: case BINDER_TYPE_WEAK_BINDER: { - struct binder_ref *ref; - struct binder_node *node = binder_get_node(proc, fp->binder); + struct flat_binder_object *fp; - if (node == NULL) { - node = binder_new_node(proc, fp->binder, fp->cookie); - if (node == NULL) { - return_error = BR_FAILED_REPLY; - goto err_binder_new_node_failed; - } - node->min_priority = fp->flags & FLAT_BINDER_FLAG_PRIORITY_MASK; - node->accept_fds = !!(fp->flags & FLAT_BINDER_FLAG_ACCEPTS_FDS); - } - if (fp->cookie != node->cookie) { - binder_user_error("%d:%d sending u%016llx node %d, cookie mismatch %016llx != %016llx\n", - proc->pid, thread->pid, - (u64)fp->binder, node->debug_id, - (u64)fp->cookie, (u64)node->cookie); - return_error = BR_FAILED_REPLY; - goto err_binder_get_ref_for_node_failed; - } - if (security_binder_transfer_binder(proc->tsk, - target_proc->tsk)) { + fp = to_flat_binder_object(hdr); + ret = binder_translate_binder(fp, t, thread); + if (ret < 0) { return_error = BR_FAILED_REPLY; - goto err_binder_get_ref_for_node_failed; + goto err_translate_failed; } - ref = binder_get_ref_for_node(target_proc, node); - if (ref == NULL) { - return_error = BR_FAILED_REPLY; - goto err_binder_get_ref_for_node_failed; - } - if (fp->type == BINDER_TYPE_BINDER) - fp->type = BINDER_TYPE_HANDLE; - else - fp->type = BINDER_TYPE_WEAK_HANDLE; - fp->binder = 0; - fp->handle = ref->desc; - fp->cookie = 0; - binder_inc_ref(ref, fp->type == BINDER_TYPE_HANDLE, - &thread->todo); - - trace_binder_transaction_node_to_ref(t, node, ref); - binder_debug(BINDER_DEBUG_TRANSACTION, - " node %d u%016llx -> ref %d desc %d\n", - node->debug_id, (u64)node->ptr, - ref->debug_id, ref->desc); } break; case BINDER_TYPE_HANDLE: case BINDER_TYPE_WEAK_HANDLE: { - struct binder_ref *ref; + struct flat_binder_object *fp; - ref = binder_get_ref(proc, fp->handle, - fp->type == BINDER_TYPE_HANDLE); + fp = to_flat_binder_object(hdr); + ret = binder_translate_handle(fp, t, thread); + if (ret < 0) { + return_error = BR_FAILED_REPLY; + goto err_translate_failed; + } + } break; - if (ref == NULL) { - binder_user_error("%d:%d got transaction with invalid handle, %d\n", - proc->pid, - thread->pid, fp->handle); + case BINDER_TYPE_FD: { + struct binder_fd_object *fp = to_binder_fd_object(hdr); + int target_fd = binder_translate_fd(fp->fd, t, thread, + in_reply_to); + + if (target_fd < 0) { return_error = BR_FAILED_REPLY; - goto err_binder_get_ref_failed; + goto err_translate_failed; } - if (security_binder_transfer_binder(proc->tsk, - target_proc->tsk)) { + fp->pad_binder = 0; + fp->fd = target_fd; + } break; + case BINDER_TYPE_FDA: { + struct binder_fd_array_object *fda = + to_binder_fd_array_object(hdr); + struct binder_buffer_object *parent = + binder_validate_ptr(t->buffer, fda->parent, + off_start, + offp - off_start); + if (!parent) { + binder_user_error("%d:%d got transaction with invalid parent offset or type\n", + proc->pid, thread->pid); return_error = BR_FAILED_REPLY; - goto err_binder_get_ref_failed; + goto err_bad_parent; } - if (ref->node->proc == target_proc) { - if (fp->type == BINDER_TYPE_HANDLE) - fp->type = BINDER_TYPE_BINDER; - else - fp->type = BINDER_TYPE_WEAK_BINDER; - fp->binder = ref->node->ptr; - fp->cookie = ref->node->cookie; - binder_inc_node(ref->node, fp->type == BINDER_TYPE_BINDER, 0, NULL); - trace_binder_transaction_ref_to_node(t, ref); - binder_debug(BINDER_DEBUG_TRANSACTION, - " ref %d desc %d -> node %d u%016llx\n", - ref->debug_id, ref->desc, ref->node->debug_id, - (u64)ref->node->ptr); - } else { - struct binder_ref *new_ref; - - new_ref = binder_get_ref_for_node(target_proc, ref->node); - if (new_ref == NULL) { - return_error = BR_FAILED_REPLY; - goto err_binder_get_ref_for_node_failed; - } - fp->binder = 0; - fp->handle = new_ref->desc; - fp->cookie = 0; - binder_inc_ref(new_ref, fp->type == BINDER_TYPE_HANDLE, NULL); - trace_binder_transaction_ref_to_ref(t, ref, - new_ref); - binder_debug(BINDER_DEBUG_TRANSACTION, - " ref %d desc %d -> ref %d desc %d (node %d)\n", - ref->debug_id, ref->desc, new_ref->debug_id, - new_ref->desc, ref->node->debug_id); + if (!binder_validate_fixup(t->buffer, off_start, + parent, fda->parent_offset, + last_fixup_obj, + last_fixup_min_off)) { + binder_user_error("%d:%d got transaction with out-of-order buffer fixup\n", + proc->pid, thread->pid); + return_error = BR_FAILED_REPLY; + goto err_bad_parent; } - } break; - - case BINDER_TYPE_FD: { - int target_fd; - struct file *file; - - if (reply) { - if (!(in_reply_to->flags & TF_ACCEPT_FDS)) { - binder_user_error("%d:%d got reply with fd, %d, but target does not allow fds\n", - proc->pid, thread->pid, fp->handle); - return_error = BR_FAILED_REPLY; - goto err_fd_not_allowed; - } - } else if (!target_node->accept_fds) { - binder_user_error("%d:%d got transaction with fd, %d, but target does not allow fds\n", - proc->pid, thread->pid, fp->handle); + ret = binder_translate_fd_array(fda, parent, t, thread, + in_reply_to); + if (ret < 0) { return_error = BR_FAILED_REPLY; - goto err_fd_not_allowed; + goto err_translate_failed; } - - file = fget(fp->handle); - if (file == NULL) { - binder_user_error("%d:%d got transaction with invalid fd, %d\n", - proc->pid, thread->pid, fp->handle); + last_fixup_obj = parent; + last_fixup_min_off = + fda->parent_offset + sizeof(u32) * fda->num_fds; + } break; + case BINDER_TYPE_PTR: { + struct binder_buffer_object *bp = + to_binder_buffer_object(hdr); + size_t buf_left = sg_buf_end - sg_bufp; + + if (bp->length > buf_left) { + binder_user_error("%d:%d got transaction with too large buffer\n", + proc->pid, thread->pid); return_error = BR_FAILED_REPLY; - goto err_fget_failed; + goto err_bad_offset; } - if (security_binder_transfer_file(proc->tsk, - target_proc->tsk, - file) < 0) { - fput(file); + if (copy_from_user(sg_bufp, + (const void __user *)(uintptr_t) + bp->buffer, bp->length)) { + binder_user_error("%d:%d got transaction with invalid offsets ptr\n", + proc->pid, thread->pid); return_error = BR_FAILED_REPLY; - goto err_get_unused_fd_failed; + goto err_copy_data_failed; } - target_fd = task_get_unused_fd_flags(target_proc, O_CLOEXEC); - if (target_fd < 0) { - fput(file); + /* Fixup buffer pointer to target proc address space */ + bp->buffer = (uintptr_t)sg_bufp + + target_proc->user_buffer_offset; + sg_bufp += ALIGN(bp->length, sizeof(u64)); + + ret = binder_fixup_parent(t, thread, bp, off_start, + offp - off_start, + last_fixup_obj, + last_fixup_min_off); + if (ret < 0) { return_error = BR_FAILED_REPLY; - goto err_get_unused_fd_failed; + goto err_translate_failed; } - task_fd_install(target_proc, target_fd, file); - trace_binder_transaction_fd(t, fp->handle, target_fd); - binder_debug(BINDER_DEBUG_TRANSACTION, - " fd %d -> %d\n", fp->handle, target_fd); - /* TODO: fput? */ - fp->binder = 0; - fp->handle = target_fd; + last_fixup_obj = bp; + last_fixup_min_off = 0; } break; - default: binder_user_error("%d:%d got transaction with invalid object type, %x\n", - proc->pid, thread->pid, fp->type); + proc->pid, thread->pid, hdr->type); return_error = BR_FAILED_REPLY; goto err_bad_object_type; } @@ -1728,14 +2204,10 @@ static void binder_transaction(struct binder_proc *proc, wake_up_interruptible(target_wait); return; -err_get_unused_fd_failed: -err_fget_failed: -err_fd_not_allowed: -err_binder_get_ref_for_node_failed: -err_binder_get_ref_failed: -err_binder_new_node_failed: +err_translate_failed: err_bad_object_type: err_bad_offset: +err_bad_parent: err_copy_data_failed: trace_binder_transaction_failed_buffer_release(t->buffer); binder_transaction_buffer_release(target_proc, t->buffer, offp); @@ -1779,6 +2251,7 @@ static int binder_thread_write(struct binder_proc *proc, binder_size_t *consumed) { uint32_t cmd; + struct binder_context *context = proc->context; void __user *buffer = (void __user *)(uintptr_t)binder_buffer; void __user *ptr = buffer + *consumed; void __user *end = buffer + size; @@ -1805,10 +2278,10 @@ static int binder_thread_write(struct binder_proc *proc, if (get_user(target, (uint32_t __user *)ptr)) return -EFAULT; ptr += sizeof(uint32_t); - if (target == 0 && binder_context_mgr_node && + if (target == 0 && context->binder_context_mgr_node && (cmd == BC_INCREFS || cmd == BC_ACQUIRE)) { ref = binder_get_ref_for_node(proc, - binder_context_mgr_node); + context->binder_context_mgr_node); if (ref->desc != target) { binder_user_error("%d:%d tried to acquire reference to desc 0, got %d instead\n", proc->pid, thread->pid, @@ -1953,6 +2426,17 @@ static int binder_thread_write(struct binder_proc *proc, break; } + case BC_TRANSACTION_SG: + case BC_REPLY_SG: { + struct binder_transaction_data_sg tr; + + if (copy_from_user(&tr, ptr, sizeof(tr))) + return -EFAULT; + ptr += sizeof(tr); + binder_transaction(proc, thread, &tr.transaction_data, + cmd == BC_REPLY_SG, tr.buffers_size); + break; + } case BC_TRANSACTION: case BC_REPLY: { struct binder_transaction_data tr; @@ -1960,7 +2444,8 @@ static int binder_thread_write(struct binder_proc *proc, if (copy_from_user(&tr, ptr, sizeof(tr))) return -EFAULT; ptr += sizeof(tr); - binder_transaction(proc, thread, &tr, cmd == BC_REPLY); + binder_transaction(proc, thread, &tr, + cmd == BC_REPLY, 0); break; } @@ -2714,9 +3199,11 @@ static int binder_ioctl_set_ctx_mgr(struct file *filp) { int ret = 0; struct binder_proc *proc = filp->private_data; + struct binder_context *context = proc->context; + kuid_t curr_euid = current_euid(); - if (binder_context_mgr_node != NULL) { + if (context->binder_context_mgr_node) { pr_err("BINDER_SET_CONTEXT_MGR already set\n"); ret = -EBUSY; goto out; @@ -2724,27 +3211,27 @@ static int binder_ioctl_set_ctx_mgr(struct file *filp) ret = security_binder_set_context_mgr(proc->tsk); if (ret < 0) goto out; - if (uid_valid(binder_context_mgr_uid)) { - if (!uid_eq(binder_context_mgr_uid, curr_euid)) { + if (uid_valid(context->binder_context_mgr_uid)) { + if (!uid_eq(context->binder_context_mgr_uid, curr_euid)) { pr_err("BINDER_SET_CONTEXT_MGR bad uid %d != %d\n", from_kuid(&init_user_ns, curr_euid), from_kuid(&init_user_ns, - binder_context_mgr_uid)); + context->binder_context_mgr_uid)); ret = -EPERM; goto out; } } else { - binder_context_mgr_uid = curr_euid; + context->binder_context_mgr_uid = curr_euid; } - binder_context_mgr_node = binder_new_node(proc, 0, 0); - if (binder_context_mgr_node == NULL) { + context->binder_context_mgr_node = binder_new_node(proc, 0, 0); + if (!context->binder_context_mgr_node) { ret = -ENOMEM; goto out; } - binder_context_mgr_node->local_weak_refs++; - binder_context_mgr_node->local_strong_refs++; - binder_context_mgr_node->has_strong_ref = 1; - binder_context_mgr_node->has_weak_ref = 1; + context->binder_context_mgr_node->local_weak_refs++; + context->binder_context_mgr_node->local_strong_refs++; + context->binder_context_mgr_node->has_strong_ref = 1; + context->binder_context_mgr_node->has_weak_ref = 1; out: return ret; } @@ -2969,6 +3456,7 @@ static int binder_mmap(struct file *filp, struct vm_area_struct *vma) static int binder_open(struct inode *nodp, struct file *filp) { struct binder_proc *proc; + struct binder_device *binder_dev; binder_debug(BINDER_DEBUG_OPEN_CLOSE, "binder_open: %d:%d\n", current->group_leader->pid, current->pid); @@ -2982,6 +3470,9 @@ static int binder_open(struct inode *nodp, struct file *filp) INIT_LIST_HEAD(&proc->todo); init_waitqueue_head(&proc->wait); proc->default_priority = task_nice(current); + binder_dev = container_of(filp->private_data, struct binder_device, + miscdev); + proc->context = &binder_dev->context; binder_lock(__func__); @@ -2997,8 +3488,17 @@ static int binder_open(struct inode *nodp, struct file *filp) char strbuf[11]; snprintf(strbuf, sizeof(strbuf), "%u", proc->pid); + /* + * proc debug entries are shared between contexts, so + * this will fail if the process tries to open the driver + * again with a different context. The priting code will + * anyway print all contexts that a given PID has, so this + * is not a problem. + */ proc->debugfs_entry = debugfs_create_file(strbuf, S_IRUGO, - binder_debugfs_dir_entry_proc, proc, &binder_proc_fops); + binder_debugfs_dir_entry_proc, + (void *)(unsigned long)proc->pid, + &binder_proc_fops); } return 0; @@ -3091,6 +3591,7 @@ static int binder_node_release(struct binder_node *node, int refs) static void binder_deferred_release(struct binder_proc *proc) { struct binder_transaction *t; + struct binder_context *context = proc->context; struct rb_node *n; int threads, nodes, incoming_refs, outgoing_refs, buffers, active_transactions, page_count; @@ -3100,11 +3601,12 @@ static void binder_deferred_release(struct binder_proc *proc) hlist_del(&proc->proc_node); - if (binder_context_mgr_node && binder_context_mgr_node->proc == proc) { + if (context->binder_context_mgr_node && + context->binder_context_mgr_node->proc == proc) { binder_debug(BINDER_DEBUG_DEAD_BINDER, "%s: %d context_mgr_node gone\n", __func__, proc->pid); - binder_context_mgr_node = NULL; + context->binder_context_mgr_node = NULL; } threads = 0; @@ -3391,6 +3893,7 @@ static void print_binder_proc(struct seq_file *m, size_t header_pos; seq_printf(m, "proc %d\n", proc->pid); + seq_printf(m, "context %s\n", proc->context->name); header_pos = m->count; for (n = rb_first(&proc->threads); n != NULL; n = rb_next(n)) @@ -3460,7 +3963,9 @@ static const char * const binder_command_strings[] = { "BC_EXIT_LOOPER", "BC_REQUEST_DEATH_NOTIFICATION", "BC_CLEAR_DEATH_NOTIFICATION", - "BC_DEAD_BINDER_DONE" + "BC_DEAD_BINDER_DONE", + "BC_TRANSACTION_SG", + "BC_REPLY_SG", }; static const char * const binder_objstat_strings[] = { @@ -3515,6 +4020,7 @@ static void print_binder_proc_stats(struct seq_file *m, int count, strong, weak; seq_printf(m, "proc %d\n", proc->pid); + seq_printf(m, "context %s\n", proc->context->name); count = 0; for (n = rb_first(&proc->threads); n != NULL; n = rb_next(n)) count++; @@ -3622,23 +4128,18 @@ static int binder_transactions_show(struct seq_file *m, void *unused) static int binder_proc_show(struct seq_file *m, void *unused) { struct binder_proc *itr; - struct binder_proc *proc = m->private; + int pid = (unsigned long)m->private; int do_lock = !binder_debug_no_lock; - bool valid_proc = false; if (do_lock) binder_lock(__func__); hlist_for_each_entry(itr, &binder_procs, proc_node) { - if (itr == proc) { - valid_proc = true; - break; + if (itr->pid == pid) { + seq_puts(m, "binder proc state:\n"); + print_binder_proc(m, itr, 1); } } - if (valid_proc) { - seq_puts(m, "binder proc state:\n"); - print_binder_proc(m, proc, 1); - } if (do_lock) binder_unlock(__func__); return 0; @@ -3648,11 +4149,11 @@ static void print_binder_transaction_log_entry(struct seq_file *m, struct binder_transaction_log_entry *e) { seq_printf(m, - "%d: %s from %d:%d to %d:%d node %d handle %d size %d:%d\n", + "%d: %s from %d:%d to %d:%d context %s node %d handle %d size %d:%d\n", e->debug_id, (e->call_type == 2) ? "reply" : ((e->call_type == 1) ? "async" : "call "), e->from_proc, - e->from_thread, e->to_proc, e->to_thread, e->to_node, - e->target_handle, e->data_size, e->offsets_size); + e->from_thread, e->to_proc, e->to_thread, e->context_name, + e->to_node, e->target_handle, e->data_size, e->offsets_size); } static int binder_transaction_log_show(struct seq_file *m, void *unused) @@ -3680,26 +4181,50 @@ static const struct file_operations binder_fops = { .release = binder_release, }; -static struct miscdevice binder_miscdev = { - .minor = MISC_DYNAMIC_MINOR, - .name = "binder", - .fops = &binder_fops -}; - BINDER_DEBUG_ENTRY(state); BINDER_DEBUG_ENTRY(stats); BINDER_DEBUG_ENTRY(transactions); BINDER_DEBUG_ENTRY(transaction_log); +static int __init init_binder_device(const char *name) +{ + int ret; + struct binder_device *binder_device; + + binder_device = kzalloc(sizeof(*binder_device), GFP_KERNEL); + if (!binder_device) + return -ENOMEM; + + binder_device->miscdev.fops = &binder_fops; + binder_device->miscdev.minor = MISC_DYNAMIC_MINOR; + binder_device->miscdev.name = name; + + binder_device->context.binder_context_mgr_uid = INVALID_UID; + binder_device->context.name = name; + + ret = misc_register(&binder_device->miscdev); + if (ret < 0) { + kfree(binder_device); + return ret; + } + + hlist_add_head(&binder_device->hlist, &binder_devices); + + return ret; +} + static int __init binder_init(void) { int ret; + char *device_name, *device_names; + struct binder_device *device; + struct hlist_node *tmp; binder_debugfs_dir_entry_root = debugfs_create_dir("binder", NULL); if (binder_debugfs_dir_entry_root) binder_debugfs_dir_entry_proc = debugfs_create_dir("proc", binder_debugfs_dir_entry_root); - ret = misc_register(&binder_miscdev); + if (binder_debugfs_dir_entry_root) { debugfs_create_file("state", S_IRUGO, @@ -3727,6 +4252,35 @@ static int __init binder_init(void) &binder_transaction_log_failed, &binder_transaction_log_fops); } + + /* + * Copy the module_parameter string, because we don't want to + * tokenize it in-place. + */ + device_names = kzalloc(strlen(binder_devices_param) + 1, GFP_KERNEL); + if (!device_names) { + ret = -ENOMEM; + goto err_alloc_device_names_failed; + } + strcpy(device_names, binder_devices_param); + + while ((device_name = strsep(&device_names, ","))) { + ret = init_binder_device(device_name); + if (ret) + goto err_init_binder_device_failed; + } + + return ret; + +err_init_binder_device_failed: + hlist_for_each_entry_safe(device, tmp, &binder_devices, hlist) { + misc_deregister(&device->miscdev); + hlist_del(&device->hlist); + kfree(device); + } +err_alloc_device_names_failed: + debugfs_remove_recursive(binder_debugfs_dir_entry_root); + return ret; } diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c index 223a770f78f365..33e363dcc63bf8 100644 --- a/drivers/ata/libata-core.c +++ b/drivers/ata/libata-core.c @@ -1695,6 +1695,8 @@ unsigned ata_exec_internal_sg(struct ata_device *dev, if (qc->err_mask & ~AC_ERR_OTHER) qc->err_mask &= ~AC_ERR_OTHER; + } else if (qc->tf.command == ATA_CMD_REQ_SENSE_DATA) { + qc->result_tf.command |= ATA_SENSE; } /* finish up */ @@ -4316,10 +4318,10 @@ static const struct ata_blacklist_entry ata_device_blacklist [] = { { "ST380013AS", "3.20", ATA_HORKAGE_MAX_SEC_1024 }, /* - * Device times out with higher max sects. + * These devices time out with higher max sects. * https://bugzilla.kernel.org/show_bug.cgi?id=121671 */ - { "LITEON CX1-JB256-HP", NULL, ATA_HORKAGE_MAX_SEC_1024 }, + { "LITEON CX1-JB*-HP", NULL, ATA_HORKAGE_MAX_SEC_1024 }, /* Devices we expect to fail diagnostics */ diff --git a/drivers/ata/sata_mv.c b/drivers/ata/sata_mv.c index 823e938c9a7877..2f32782cea6d9c 100644 --- a/drivers/ata/sata_mv.c +++ b/drivers/ata/sata_mv.c @@ -4132,6 +4132,9 @@ static int mv_platform_probe(struct platform_device *pdev) host->iomap = NULL; hpriv->base = devm_ioremap(&pdev->dev, res->start, resource_size(res)); + if (!hpriv->base) + return -ENOMEM; + hpriv->base -= SATAHC0_REG_BASE; hpriv->clk = clk_get(&pdev->dev, NULL); diff --git a/drivers/base/firmware_class.c b/drivers/base/firmware_class.c index 22d1760a42789f..a95e1e572697d1 100644 --- a/drivers/base/firmware_class.c +++ b/drivers/base/firmware_class.c @@ -955,13 +955,14 @@ static int _request_firmware_load(struct firmware_priv *fw_priv, timeout = MAX_JIFFY_OFFSET; } - retval = wait_for_completion_interruptible_timeout(&buf->completion, + timeout = wait_for_completion_interruptible_timeout(&buf->completion, timeout); - if (retval == -ERESTARTSYS || !retval) { + if (timeout == -ERESTARTSYS || !timeout) { + retval = timeout; mutex_lock(&fw_lock); fw_load_abort(fw_priv); mutex_unlock(&fw_lock); - } else if (retval > 0) { + } else if (timeout > 0) { retval = 0; } diff --git a/drivers/base/memory.c b/drivers/base/memory.c index 62c63c0c5c22dc..c5cdd190b7816c 100644 --- a/drivers/base/memory.c +++ b/drivers/base/memory.c @@ -391,33 +391,33 @@ static ssize_t show_valid_zones(struct device *dev, { struct memory_block *mem = to_memory_block(dev); unsigned long start_pfn, end_pfn; + unsigned long valid_start, valid_end, valid_pages; unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block; - struct page *first_page; struct zone *zone; int zone_shift = 0; start_pfn = section_nr_to_pfn(mem->start_section_nr); end_pfn = start_pfn + nr_pages; - first_page = pfn_to_page(start_pfn); /* The block contains more than one zone can not be offlined. */ - if (!test_pages_in_a_zone(start_pfn, end_pfn)) + if (!test_pages_in_a_zone(start_pfn, end_pfn, &valid_start, &valid_end)) return sprintf(buf, "none\n"); - zone = page_zone(first_page); + zone = page_zone(pfn_to_page(valid_start)); + valid_pages = valid_end - valid_start; /* MMOP_ONLINE_KEEP */ sprintf(buf, "%s", zone->name); /* MMOP_ONLINE_KERNEL */ - zone_shift = zone_can_shift(start_pfn, nr_pages, ZONE_NORMAL); + zone_can_shift(valid_start, valid_pages, ZONE_NORMAL, &zone_shift); if (zone_shift) { strcat(buf, " "); strcat(buf, (zone + zone_shift)->name); } /* MMOP_ONLINE_MOVABLE */ - zone_shift = zone_can_shift(start_pfn, nr_pages, ZONE_MOVABLE); + zone_can_shift(valid_start, valid_pages, ZONE_MOVABLE, &zone_shift); if (zone_shift) { strcat(buf, " "); strcat(buf, (zone + zone_shift)->name); diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c index 2932a5bd892f7e..4f991013f6822e 100644 --- a/drivers/base/power/main.c +++ b/drivers/base/power/main.c @@ -33,6 +33,7 @@ #include #include #include +#include #include "../base.h" #include "power.h" @@ -1353,6 +1354,7 @@ static int __device_suspend(struct device *dev, pm_message_t state, bool async) pm_callback_t callback = NULL; char *info = NULL; int error = 0; + char suspend_abort[MAX_SUSPEND_ABORT_LEN]; DECLARE_DPM_WATCHDOG_ON_STACK(wd); TRACE_DEVICE(dev); @@ -1373,6 +1375,9 @@ static int __device_suspend(struct device *dev, pm_message_t state, bool async) pm_wakeup_event(dev, 0); if (pm_wakeup_pending()) { + pm_get_active_wakeup_sources(suspend_abort, + MAX_SUSPEND_ABORT_LEN); + log_suspend_abort_reason(suspend_abort); async_error = -EBUSY; goto Complete; } diff --git a/drivers/base/power/opp/core.c b/drivers/base/power/opp/core.c index 4c7c6da7a98918..6441dfda489f80 100644 --- a/drivers/base/power/opp/core.c +++ b/drivers/base/power/opp/core.c @@ -584,6 +584,7 @@ int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq) struct clk *clk; unsigned long freq, old_freq; unsigned long u_volt, u_volt_min, u_volt_max; + unsigned long old_u_volt, old_u_volt_min, old_u_volt_max; int ret; if (unlikely(!target_freq)) { @@ -633,6 +634,14 @@ int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq) return ret; } + if (IS_ERR(old_opp)) { + old_u_volt = 0; + } else { + old_u_volt = old_opp->u_volt; + old_u_volt_min = old_opp->u_volt_min; + old_u_volt_max = old_opp->u_volt_max; + } + u_volt = opp->u_volt; u_volt_min = opp->u_volt_min; u_volt_max = opp->u_volt_max; @@ -677,9 +686,10 @@ int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq) __func__, old_freq); restore_voltage: /* This shouldn't harm even if the voltages weren't updated earlier */ - if (!IS_ERR(old_opp)) - _set_opp_voltage(dev, reg, old_opp->u_volt, - old_opp->u_volt_min, old_opp->u_volt_max); + if (old_u_volt) { + _set_opp_voltage(dev, reg, old_u_volt, old_u_volt_min, + old_u_volt_max); + } return ret; } @@ -1316,7 +1326,7 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_put_prop_name); * that this function is *NOT* called under RCU protection or in contexts where * mutex cannot be locked. */ -int dev_pm_opp_set_regulator(struct device *dev, const char *name) +struct opp_table *dev_pm_opp_set_regulator(struct device *dev, const char *name) { struct opp_table *opp_table; struct regulator *reg; @@ -1354,20 +1364,20 @@ int dev_pm_opp_set_regulator(struct device *dev, const char *name) opp_table->regulator = reg; mutex_unlock(&opp_table_lock); - return 0; + return opp_table; err: _remove_opp_table(opp_table); unlock: mutex_unlock(&opp_table_lock); - return ret; + return ERR_PTR(ret); } EXPORT_SYMBOL_GPL(dev_pm_opp_set_regulator); /** * dev_pm_opp_put_regulator() - Releases resources blocked for regulator - * @dev: Device for which regulator was set. + * @opp_table: OPP table returned from dev_pm_opp_set_regulator(). * * Locking: The internal opp_table and opp structures are RCU protected. * Hence this function internally uses RCU updater strategy with mutex locks @@ -1375,22 +1385,12 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_set_regulator); * that this function is *NOT* called under RCU protection or in contexts where * mutex cannot be locked. */ -void dev_pm_opp_put_regulator(struct device *dev) +void dev_pm_opp_put_regulator(struct opp_table *opp_table) { - struct opp_table *opp_table; - mutex_lock(&opp_table_lock); - /* Check for existing table for 'dev' first */ - opp_table = _find_opp_table(dev); - if (IS_ERR(opp_table)) { - dev_err(dev, "Failed to find opp_table: %ld\n", - PTR_ERR(opp_table)); - goto unlock; - } - if (IS_ERR(opp_table->regulator)) { - dev_err(dev, "%s: Doesn't have regulator set\n", __func__); + pr_err("%s: Doesn't have regulator set\n", __func__); goto unlock; } diff --git a/drivers/base/power/power.h b/drivers/base/power/power.h index 50e30e7b059d18..a84332aefc2da7 100644 --- a/drivers/base/power/power.h +++ b/drivers/base/power/power.h @@ -21,14 +21,22 @@ extern void pm_runtime_init(struct device *dev); extern void pm_runtime_reinit(struct device *dev); extern void pm_runtime_remove(struct device *dev); +#define WAKE_IRQ_DEDICATED_ALLOCATED BIT(0) +#define WAKE_IRQ_DEDICATED_MANAGED BIT(1) +#define WAKE_IRQ_DEDICATED_MASK (WAKE_IRQ_DEDICATED_ALLOCATED | \ + WAKE_IRQ_DEDICATED_MANAGED) + struct wake_irq { struct device *dev; + unsigned int status; int irq; - bool dedicated_irq:1; }; extern void dev_pm_arm_wake_irq(struct wake_irq *wirq); extern void dev_pm_disarm_wake_irq(struct wake_irq *wirq); +extern void dev_pm_enable_wake_irq_check(struct device *dev, + bool can_change_status); +extern void dev_pm_disable_wake_irq_check(struct device *dev); #ifdef CONFIG_PM_SLEEP @@ -104,6 +112,15 @@ static inline void dev_pm_disarm_wake_irq(struct wake_irq *wirq) { } +static inline void dev_pm_enable_wake_irq_check(struct device *dev, + bool can_change_status) +{ +} + +static inline void dev_pm_disable_wake_irq_check(struct device *dev) +{ +} + #endif #ifdef CONFIG_PM_SLEEP diff --git a/drivers/base/power/runtime.c b/drivers/base/power/runtime.c index 82a081ea43176c..23f3b95a1158bc 100644 --- a/drivers/base/power/runtime.c +++ b/drivers/base/power/runtime.c @@ -515,7 +515,7 @@ static int rpm_suspend(struct device *dev, int rpmflags) callback = RPM_GET_CALLBACK(dev, runtime_suspend); - dev_pm_enable_wake_irq(dev); + dev_pm_enable_wake_irq_check(dev, true); retval = rpm_callback(callback, dev); if (retval) goto fail; @@ -554,7 +554,7 @@ static int rpm_suspend(struct device *dev, int rpmflags) return retval; fail: - dev_pm_disable_wake_irq(dev); + dev_pm_disable_wake_irq_check(dev); __update_runtime_status(dev, RPM_ACTIVE); dev->power.deferred_resume = false; wake_up_all(&dev->power.wait_queue); @@ -737,12 +737,12 @@ static int rpm_resume(struct device *dev, int rpmflags) callback = RPM_GET_CALLBACK(dev, runtime_resume); - dev_pm_disable_wake_irq(dev); + dev_pm_disable_wake_irq_check(dev); retval = rpm_callback(callback, dev); if (retval) { __update_runtime_status(dev, RPM_SUSPENDED); pm_runtime_cancel_pending(dev); - dev_pm_enable_wake_irq(dev); + dev_pm_enable_wake_irq_check(dev, false); } else { no_callback: __update_runtime_status(dev, RPM_ACTIVE); diff --git a/drivers/base/power/wakeirq.c b/drivers/base/power/wakeirq.c index 0d77cd6fd8d116..404d94c6c8bc6a 100644 --- a/drivers/base/power/wakeirq.c +++ b/drivers/base/power/wakeirq.c @@ -110,8 +110,10 @@ void dev_pm_clear_wake_irq(struct device *dev) dev->power.wakeirq = NULL; spin_unlock_irqrestore(&dev->power.lock, flags); - if (wirq->dedicated_irq) + if (wirq->status & WAKE_IRQ_DEDICATED_ALLOCATED) { free_irq(wirq->irq, wirq); + wirq->status &= ~WAKE_IRQ_DEDICATED_MASK; + } kfree(wirq); } EXPORT_SYMBOL_GPL(dev_pm_clear_wake_irq); @@ -179,7 +181,6 @@ int dev_pm_set_dedicated_wake_irq(struct device *dev, int irq) wirq->dev = dev; wirq->irq = irq; - wirq->dedicated_irq = true; irq_set_status_flags(irq, IRQ_NOAUTOEN); /* @@ -195,6 +196,8 @@ int dev_pm_set_dedicated_wake_irq(struct device *dev, int irq) if (err) goto err_free_irq; + wirq->status = WAKE_IRQ_DEDICATED_ALLOCATED; + return err; err_free_irq: @@ -210,9 +213,9 @@ EXPORT_SYMBOL_GPL(dev_pm_set_dedicated_wake_irq); * dev_pm_enable_wake_irq - Enable device wake-up interrupt * @dev: Device * - * Called from the bus code or the device driver for - * runtime_suspend() to enable the wake-up interrupt while - * the device is running. + * Optionally called from the bus code or the device driver for + * runtime_resume() to override the PM runtime core managed wake-up + * interrupt handling to enable the wake-up interrupt. * * Note that for runtime_suspend()) the wake-up interrupts * should be unconditionally enabled unlike for suspend() @@ -222,7 +225,7 @@ void dev_pm_enable_wake_irq(struct device *dev) { struct wake_irq *wirq = dev->power.wakeirq; - if (wirq && wirq->dedicated_irq) + if (wirq && (wirq->status & WAKE_IRQ_DEDICATED_ALLOCATED)) enable_irq(wirq->irq); } EXPORT_SYMBOL_GPL(dev_pm_enable_wake_irq); @@ -231,19 +234,72 @@ EXPORT_SYMBOL_GPL(dev_pm_enable_wake_irq); * dev_pm_disable_wake_irq - Disable device wake-up interrupt * @dev: Device * - * Called from the bus code or the device driver for - * runtime_resume() to disable the wake-up interrupt while - * the device is running. + * Optionally called from the bus code or the device driver for + * runtime_suspend() to override the PM runtime core managed wake-up + * interrupt handling to disable the wake-up interrupt. */ void dev_pm_disable_wake_irq(struct device *dev) { struct wake_irq *wirq = dev->power.wakeirq; - if (wirq && wirq->dedicated_irq) + if (wirq && (wirq->status & WAKE_IRQ_DEDICATED_ALLOCATED)) disable_irq_nosync(wirq->irq); } EXPORT_SYMBOL_GPL(dev_pm_disable_wake_irq); +/** + * dev_pm_enable_wake_irq_check - Checks and enables wake-up interrupt + * @dev: Device + * @can_change_status: Can change wake-up interrupt status + * + * Enables wakeirq conditionally. We need to enable wake-up interrupt + * lazily on the first rpm_suspend(). This is needed as the consumer device + * starts in RPM_SUSPENDED state, and the the first pm_runtime_get() would + * otherwise try to disable already disabled wakeirq. The wake-up interrupt + * starts disabled with IRQ_NOAUTOEN set. + * + * Should be only called from rpm_suspend() and rpm_resume() path. + * Caller must hold &dev->power.lock to change wirq->status + */ +void dev_pm_enable_wake_irq_check(struct device *dev, + bool can_change_status) +{ + struct wake_irq *wirq = dev->power.wakeirq; + + if (!wirq || !((wirq->status & WAKE_IRQ_DEDICATED_MASK))) + return; + + if (likely(wirq->status & WAKE_IRQ_DEDICATED_MANAGED)) { + goto enable; + } else if (can_change_status) { + wirq->status |= WAKE_IRQ_DEDICATED_MANAGED; + goto enable; + } + + return; + +enable: + enable_irq(wirq->irq); +} + +/** + * dev_pm_disable_wake_irq_check - Checks and disables wake-up interrupt + * @dev: Device + * + * Disables wake-up interrupt conditionally based on status. + * Should be only called from rpm_suspend() and rpm_resume() path. + */ +void dev_pm_disable_wake_irq_check(struct device *dev) +{ + struct wake_irq *wirq = dev->power.wakeirq; + + if (!wirq || !((wirq->status & WAKE_IRQ_DEDICATED_MASK))) + return; + + if (wirq->status & WAKE_IRQ_DEDICATED_MANAGED) + disable_irq_nosync(wirq->irq); +} + /** * dev_pm_arm_wake_irq - Arm device wake-up * @wirq: Device wake-up interrupt diff --git a/drivers/base/power/wakeup.c b/drivers/base/power/wakeup.c index 62e4de2aa8d159..16d307b13ccc90 100644 --- a/drivers/base/power/wakeup.c +++ b/drivers/base/power/wakeup.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include "power.h" @@ -802,6 +803,37 @@ void pm_wakeup_event(struct device *dev, unsigned int msec) } EXPORT_SYMBOL_GPL(pm_wakeup_event); +void pm_get_active_wakeup_sources(char *pending_wakeup_source, size_t max) +{ + struct wakeup_source *ws, *last_active_ws = NULL; + int len = 0; + bool active = false; + + rcu_read_lock(); + list_for_each_entry_rcu(ws, &wakeup_sources, entry) { + if (ws->active && len < max) { + if (!active) + len += scnprintf(pending_wakeup_source, max, + "Pending Wakeup Sources: "); + len += scnprintf(pending_wakeup_source + len, max - len, + "%s ", ws->name); + active = true; + } else if (!active && + (!last_active_ws || + ktime_to_ns(ws->last_time) > + ktime_to_ns(last_active_ws->last_time))) { + last_active_ws = ws; + } + } + if (!active && last_active_ws) { + scnprintf(pending_wakeup_source, max, + "Last active Wakeup Source: %s", + last_active_ws->name); + } + rcu_read_unlock(); +} +EXPORT_SYMBOL_GPL(pm_get_active_wakeup_sources); + void pm_print_active_wakeup_sources(void) { struct wakeup_source *ws; diff --git a/drivers/base/syscore.c b/drivers/base/syscore.c index 8d98a329f6ea63..96c34a95cc6259 100644 --- a/drivers/base/syscore.c +++ b/drivers/base/syscore.c @@ -11,6 +11,7 @@ #include #include #include +#include static LIST_HEAD(syscore_ops_list); static DEFINE_MUTEX(syscore_ops_lock); @@ -75,6 +76,8 @@ int syscore_suspend(void) return 0; err_out: + log_suspend_abort_reason("System core suspend callback %pF failed", + ops->suspend); pr_err("PM: System core suspend callback %pF failed.\n", ops->suspend); list_for_each_entry_continue(ops, &syscore_ops_list, node) diff --git a/drivers/bcma/bcma_private.h b/drivers/bcma/bcma_private.h index f642c4264c277b..168fa175d65a08 100644 --- a/drivers/bcma/bcma_private.h +++ b/drivers/bcma/bcma_private.h @@ -45,6 +45,9 @@ int bcma_sprom_get(struct bcma_bus *bus); void bcma_core_chipcommon_early_init(struct bcma_drv_cc *cc); void bcma_core_chipcommon_init(struct bcma_drv_cc *cc); void bcma_chipco_bcm4331_ext_pa_lines_ctl(struct bcma_drv_cc *cc, bool enable); +#ifdef CONFIG_BCMA_DRIVER_MIPS +void bcma_chipco_serial_init(struct bcma_drv_cc *cc); +#endif /* CONFIG_BCMA_DRIVER_MIPS */ /* driver_chipcommon_b.c */ int bcma_core_chipcommon_b_init(struct bcma_drv_cc_b *ccb); diff --git a/drivers/bcma/driver_chipcommon.c b/drivers/bcma/driver_chipcommon.c index b4f6520e74f05b..62f5bfa5065d91 100644 --- a/drivers/bcma/driver_chipcommon.c +++ b/drivers/bcma/driver_chipcommon.c @@ -15,8 +15,6 @@ #include #include -static void bcma_chipco_serial_init(struct bcma_drv_cc *cc); - static inline u32 bcma_cc_write32_masked(struct bcma_drv_cc *cc, u16 offset, u32 mask, u32 value) { @@ -186,9 +184,6 @@ void bcma_core_chipcommon_early_init(struct bcma_drv_cc *cc) if (cc->capabilities & BCMA_CC_CAP_PMU) bcma_pmu_early_init(cc); - if (IS_BUILTIN(CONFIG_BCM47XX) && bus->hosttype == BCMA_HOSTTYPE_SOC) - bcma_chipco_serial_init(cc); - if (bus->hosttype == BCMA_HOSTTYPE_SOC) bcma_core_chipcommon_flash_detect(cc); @@ -378,9 +373,9 @@ u32 bcma_chipco_gpio_pulldown(struct bcma_drv_cc *cc, u32 mask, u32 value) return res; } -static void bcma_chipco_serial_init(struct bcma_drv_cc *cc) +#ifdef CONFIG_BCMA_DRIVER_MIPS +void bcma_chipco_serial_init(struct bcma_drv_cc *cc) { -#if IS_BUILTIN(CONFIG_BCM47XX) unsigned int irq; u32 baud_base; u32 i; @@ -422,5 +417,5 @@ static void bcma_chipco_serial_init(struct bcma_drv_cc *cc) ports[i].baud_base = baud_base; ports[i].reg_shift = 0; } -#endif /* CONFIG_BCM47XX */ } +#endif /* CONFIG_BCMA_DRIVER_MIPS */ diff --git a/drivers/bcma/driver_mips.c b/drivers/bcma/driver_mips.c index 96f17132820080..89af807cf29ce4 100644 --- a/drivers/bcma/driver_mips.c +++ b/drivers/bcma/driver_mips.c @@ -278,9 +278,12 @@ static void bcma_core_mips_nvram_init(struct bcma_drv_mips *mcore) void bcma_core_mips_early_init(struct bcma_drv_mips *mcore) { + struct bcma_bus *bus = mcore->core->bus; + if (mcore->early_setup_done) return; + bcma_chipco_serial_init(&bus->drv_cc); bcma_core_mips_nvram_init(mcore); mcore->early_setup_done = true; diff --git a/drivers/block/loop.c b/drivers/block/loop.c index fa1b7a90ba11de..4af81876679766 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -1646,7 +1646,7 @@ static int loop_queue_rq(struct blk_mq_hw_ctx *hctx, blk_mq_start_request(bd->rq); if (lo->lo_state != Lo_bound) - return -EIO; + return BLK_MQ_RQ_QUEUE_ERROR; switch (req_op(cmd->rq)) { case REQ_OP_FLUSH: diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index 5545a679abd888..3c3b8f601469a1 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -56,6 +56,7 @@ struct virtblk_req { struct virtio_blk_outhdr out_hdr; struct virtio_scsi_inhdr in_hdr; u8 status; + u8 sense[SCSI_SENSE_BUFFERSIZE]; struct scatterlist sg[]; }; @@ -102,7 +103,8 @@ static int __virtblk_add_req(struct virtqueue *vq, } if (type == cpu_to_virtio32(vq->vdev, VIRTIO_BLK_T_SCSI_CMD)) { - sg_init_one(&sense, vbr->req->sense, SCSI_SENSE_BUFFERSIZE); + memcpy(vbr->sense, vbr->req->sense, SCSI_SENSE_BUFFERSIZE); + sg_init_one(&sense, vbr->sense, SCSI_SENSE_BUFFERSIZE); sgs[num_out + num_in++] = &sense; sg_init_one(&inhdr, &vbr->in_hdr, sizeof(vbr->in_hdr)); sgs[num_out + num_in++] = &inhdr; diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index 5497f7fc44d04c..d2ef51ca9cf47c 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -111,6 +112,14 @@ static inline bool is_partial_io(struct bio_vec *bvec) return bvec->bv_len != PAGE_SIZE; } +static void zram_revalidate_disk(struct zram *zram) +{ + revalidate_disk(zram->disk); + /* revalidate_disk reset the BDI_CAP_STABLE_WRITES so set again */ + zram->disk->queue->backing_dev_info.capabilities |= + BDI_CAP_STABLE_WRITES; +} + /* * Check if request is within bounds and aligned on zram logical blocks. */ @@ -1094,15 +1103,9 @@ static ssize_t disksize_store(struct device *dev, zram->comp = comp; zram->disksize = disksize; set_capacity(zram->disk, zram->disksize >> SECTOR_SHIFT); + zram_revalidate_disk(zram); up_write(&zram->init_lock); - /* - * Revalidate disk out of the init_lock to avoid lockdep splat. - * It's okay because disk's capacity is protected by init_lock - * so that revalidate_disk always sees up-to-date capacity. - */ - revalidate_disk(zram->disk); - return len; out_destroy_comp: @@ -1148,7 +1151,7 @@ static ssize_t reset_store(struct device *dev, /* Make sure all the pending I/O are finished */ fsync_bdev(bdev); zram_reset_device(zram); - revalidate_disk(zram->disk); + zram_revalidate_disk(zram); bdput(bdev); mutex_lock(&bdev->bd_mutex); diff --git a/drivers/bluetooth/btwilink.c b/drivers/bluetooth/btwilink.c index b6bb58c41df5b7..e1d087e1fbeada 100644 --- a/drivers/bluetooth/btwilink.c +++ b/drivers/bluetooth/btwilink.c @@ -30,6 +30,7 @@ #include #include +#include /* Bluetooth Driver Version */ #define VERSION "1.0" @@ -275,6 +276,14 @@ static int ti_st_send_frame(struct hci_dev *hdev, struct sk_buff *skb) return 0; } +static const struct of_device_id btwilink_of_match[] = { +{ + .compatible = "btwilink", + }, + {} +}; +MODULE_DEVICE_TABLE(of, btwilink_of_match); + static int bt_ti_probe(struct platform_device *pdev) { static struct ti_st *hst; @@ -338,6 +347,8 @@ static struct platform_driver btwilink_driver = { .remove = bt_ti_remove, .driver = { .name = "btwilink", + .owner = THIS_MODULE, + .of_match_table = of_match_ptr(btwilink_of_match), }, }; diff --git a/drivers/bus/arm-ccn.c b/drivers/bus/arm-ccn.c index d1074d9b38ba16..aee83462b796bd 100644 --- a/drivers/bus/arm-ccn.c +++ b/drivers/bus/arm-ccn.c @@ -1570,7 +1570,10 @@ static int __init arm_ccn_init(void) for (i = 0; i < ARRAY_SIZE(arm_ccn_pmu_events); i++) arm_ccn_pmu_events_attrs[i] = &arm_ccn_pmu_events[i].attr.attr; - return platform_driver_register(&arm_ccn_driver); + ret = platform_driver_register(&arm_ccn_driver); + if (ret) + cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_CCN_ONLINE); + return ret; } static void __exit arm_ccn_exit(void) diff --git a/drivers/bus/vexpress-config.c b/drivers/bus/vexpress-config.c index 9efdf1de4035e2..493e7b9fc81357 100644 --- a/drivers/bus/vexpress-config.c +++ b/drivers/bus/vexpress-config.c @@ -171,6 +171,7 @@ static int vexpress_config_populate(struct device_node *node) { struct device_node *bridge; struct device *parent; + int ret; bridge = of_parse_phandle(node, "arm,vexpress,config-bridge", 0); if (!bridge) @@ -182,7 +183,11 @@ static int vexpress_config_populate(struct device_node *node) if (WARN_ON(!parent)) return -ENODEV; - return of_platform_populate(node, NULL, NULL, parent); + ret = of_platform_populate(node, NULL, NULL, parent); + + put_device(parent); + + return ret; } static int __init vexpress_config_init(void) diff --git a/drivers/char/mem.c b/drivers/char/mem.c index 5bb1985ec484ae..6d9cc2d39d2230 100644 --- a/drivers/char/mem.c +++ b/drivers/char/mem.c @@ -381,9 +381,6 @@ static ssize_t read_kmem(struct file *file, char __user *buf, char *kbuf; /* k-addr because vread() takes vmlist_lock rwlock */ int err = 0; - if (!pfn_valid(PFN_DOWN(p))) - return -EIO; - read = 0; if (p < (unsigned long) high_memory) { low_count = count; @@ -412,6 +409,8 @@ static ssize_t read_kmem(struct file *file, char __user *buf, * by the kernel or data corruption may occur */ kbuf = xlate_dev_kmem_ptr((void *)p); + if (!virt_addr_valid(kbuf)) + return -ENXIO; if (copy_to_user(buf, kbuf, sz)) return -EFAULT; @@ -482,6 +481,8 @@ static ssize_t do_write_kmem(unsigned long p, const char __user *buf, * corruption may occur. */ ptr = xlate_dev_kmem_ptr((void *)p); + if (!virt_addr_valid(ptr)) + return -ENXIO; copied = copy_from_user(ptr, buf, sz); if (copied) { @@ -512,9 +513,6 @@ static ssize_t write_kmem(struct file *file, const char __user *buf, char *kbuf; /* k-addr because vwrite() takes vmlist_lock rwlock */ int err = 0; - if (!pfn_valid(PFN_DOWN(p))) - return -EIO; - if (p < (unsigned long) high_memory) { unsigned long to_write = min_t(unsigned long, count, (unsigned long)high_memory - p); diff --git a/drivers/char/tpm/tpm_tis_core.c b/drivers/char/tpm/tpm_tis_core.c index e3bf31b3713862..a1ce0607bf7b45 100644 --- a/drivers/char/tpm/tpm_tis_core.c +++ b/drivers/char/tpm/tpm_tis_core.c @@ -185,7 +185,12 @@ static int recv_data(struct tpm_chip *chip, u8 *buf, size_t count) TPM_STS_DATA_AVAIL | TPM_STS_VALID, chip->timeout_c, &priv->read_queue, true) == 0) { - burstcnt = min_t(int, get_burstcount(chip), count - size); + burstcnt = get_burstcount(chip); + if (burstcnt < 0) { + dev_err(&chip->dev, "Unable to read burstcount\n"); + return burstcnt; + } + burstcnt = min_t(int, burstcnt, count - size); rc = tpm_tis_read_bytes(priv, TPM_DATA_FIFO(priv->locality), burstcnt, buf + size); @@ -271,7 +276,13 @@ static int tpm_tis_send_data(struct tpm_chip *chip, u8 *buf, size_t len) } while (count < len - 1) { - burstcnt = min_t(int, get_burstcount(chip), len - count - 1); + burstcnt = get_burstcount(chip); + if (burstcnt < 0) { + dev_err(&chip->dev, "Unable to read burstcount\n"); + rc = burstcnt; + goto out_err; + } + burstcnt = min_t(int, burstcnt, len - count - 1); rc = tpm_tis_write_bytes(priv, TPM_DATA_FIFO(priv->locality), burstcnt, buf + count); if (rc < 0) diff --git a/drivers/char/tpm/xen-tpmfront.c b/drivers/char/tpm/xen-tpmfront.c index 62028f483bbaac..a2ab00831df1cc 100644 --- a/drivers/char/tpm/xen-tpmfront.c +++ b/drivers/char/tpm/xen-tpmfront.c @@ -307,7 +307,6 @@ static int tpmfront_probe(struct xenbus_device *dev, rv = setup_ring(dev, priv); if (rv) { chip = dev_get_drvdata(&dev->dev); - tpm_chip_unregister(chip); ring_free(priv); return rv; } diff --git a/drivers/clk/bcm/clk-bcm2835.c b/drivers/clk/bcm/clk-bcm2835.c index 8c7763fd9efc52..3bbd2a58db470a 100644 --- a/drivers/clk/bcm/clk-bcm2835.c +++ b/drivers/clk/bcm/clk-bcm2835.c @@ -751,7 +751,9 @@ static void bcm2835_pll_divider_off(struct clk_hw *hw) cprman_write(cprman, data->cm_reg, (cprman_read(cprman, data->cm_reg) & ~data->load_mask) | data->hold_mask); - cprman_write(cprman, data->a2w_reg, A2W_PLL_CHANNEL_DISABLE); + cprman_write(cprman, data->a2w_reg, + cprman_read(cprman, data->a2w_reg) | + A2W_PLL_CHANNEL_DISABLE); spin_unlock(&cprman->regs_lock); } diff --git a/drivers/clk/clk-wm831x.c b/drivers/clk/clk-wm831x.c index f4fdac55727c80..0621fbfb4bebfd 100644 --- a/drivers/clk/clk-wm831x.c +++ b/drivers/clk/clk-wm831x.c @@ -243,7 +243,7 @@ static int wm831x_clkout_is_prepared(struct clk_hw *hw) if (ret < 0) { dev_err(wm831x->dev, "Unable to read CLOCK_CONTROL_1: %d\n", ret); - return true; + return false; } return (ret & WM831X_CLKOUT_ENA) != 0; diff --git a/drivers/clk/clkdev.c b/drivers/clk/clkdev.c index 97ae60fa158499..bb8a77a5985f86 100644 --- a/drivers/clk/clkdev.c +++ b/drivers/clk/clkdev.c @@ -448,12 +448,20 @@ EXPORT_SYMBOL(clk_register_clkdev); * * con_id or dev_id may be NULL as a wildcard, just as in the rest of * clkdev. + * + * To make things easier for mass registration, we detect error clk_hws + * from a previous clk_hw_register_*() call, and return the error code for + * those. This is to permit this function to be called immediately + * after clk_hw_register_*(). */ int clk_hw_register_clkdev(struct clk_hw *hw, const char *con_id, const char *dev_id) { struct clk_lookup *cl; + if (IS_ERR(hw)) + return PTR_ERR(hw); + /* * Since dev_id can be NULL, and NULL is handled specially, we must * pass it as either a NULL format string, or with "%s". diff --git a/drivers/clk/imx/clk-imx31.c b/drivers/clk/imx/clk-imx31.c index 6a964144a5b5df..6a49ba2b96714f 100644 --- a/drivers/clk/imx/clk-imx31.c +++ b/drivers/clk/imx/clk-imx31.c @@ -157,10 +157,8 @@ static void __init _mx31_clocks_init(unsigned long fref) } } -int __init mx31_clocks_init(void) +int __init mx31_clocks_init(unsigned long fref) { - u32 fref = 26000000; /* default */ - _mx31_clocks_init(fref); clk_register_clkdev(clk[gpt_gate], "per", "imx-gpt.0"); diff --git a/drivers/clk/qcom/gcc-ipq806x.c b/drivers/clk/qcom/gcc-ipq806x.c index 52a7d3959875b2..28eb200d0f1ee0 100644 --- a/drivers/clk/qcom/gcc-ipq806x.c +++ b/drivers/clk/qcom/gcc-ipq806x.c @@ -2990,11 +2990,11 @@ static int gcc_ipq806x_probe(struct platform_device *pdev) struct regmap *regmap; int ret; - ret = qcom_cc_register_board_clk(dev, "cxo_board", "cxo", 19200000); + ret = qcom_cc_register_board_clk(dev, "cxo_board", "cxo", 25000000); if (ret) return ret; - ret = qcom_cc_register_board_clk(dev, "pxo_board", "pxo", 27000000); + ret = qcom_cc_register_board_clk(dev, "pxo_board", "pxo", 25000000); if (ret) return ret; diff --git a/drivers/clk/renesas/clk-mstp.c b/drivers/clk/renesas/clk-mstp.c index 9375777776d994..b533f99550e1b9 100644 --- a/drivers/clk/renesas/clk-mstp.c +++ b/drivers/clk/renesas/clk-mstp.c @@ -37,12 +37,14 @@ * @smstpcr: module stop control register * @mstpsr: module stop status register (optional) * @lock: protects writes to SMSTPCR + * @width_8bit: registers are 8-bit, not 32-bit */ struct mstp_clock_group { struct clk_onecell_data data; void __iomem *smstpcr; void __iomem *mstpsr; spinlock_t lock; + bool width_8bit; }; /** @@ -59,6 +61,18 @@ struct mstp_clock { #define to_mstp_clock(_hw) container_of(_hw, struct mstp_clock, hw) +static inline u32 cpg_mstp_read(struct mstp_clock_group *group, + u32 __iomem *reg) +{ + return group->width_8bit ? readb(reg) : clk_readl(reg); +} + +static inline void cpg_mstp_write(struct mstp_clock_group *group, u32 val, + u32 __iomem *reg) +{ + group->width_8bit ? writeb(val, reg) : clk_writel(val, reg); +} + static int cpg_mstp_clock_endisable(struct clk_hw *hw, bool enable) { struct mstp_clock *clock = to_mstp_clock(hw); @@ -70,12 +84,12 @@ static int cpg_mstp_clock_endisable(struct clk_hw *hw, bool enable) spin_lock_irqsave(&group->lock, flags); - value = clk_readl(group->smstpcr); + value = cpg_mstp_read(group, group->smstpcr); if (enable) value &= ~bitmask; else value |= bitmask; - clk_writel(value, group->smstpcr); + cpg_mstp_write(group, value, group->smstpcr); spin_unlock_irqrestore(&group->lock, flags); @@ -83,7 +97,7 @@ static int cpg_mstp_clock_endisable(struct clk_hw *hw, bool enable) return 0; for (i = 1000; i > 0; --i) { - if (!(clk_readl(group->mstpsr) & bitmask)) + if (!(cpg_mstp_read(group, group->mstpsr) & bitmask)) break; cpu_relax(); } @@ -114,9 +128,9 @@ static int cpg_mstp_clock_is_enabled(struct clk_hw *hw) u32 value; if (group->mstpsr) - value = clk_readl(group->mstpsr); + value = cpg_mstp_read(group, group->mstpsr); else - value = clk_readl(group->smstpcr); + value = cpg_mstp_read(group, group->smstpcr); return !(value & BIT(clock->bit_index)); } @@ -188,6 +202,9 @@ static void __init cpg_mstp_clocks_init(struct device_node *np) return; } + if (of_device_is_compatible(np, "renesas,r7s72100-mstp-clocks")) + group->width_8bit = true; + for (i = 0; i < MSTP_MAX_CLOCKS; ++i) clks[i] = ERR_PTR(-ENOENT); diff --git a/drivers/clk/renesas/renesas-cpg-mssr.c b/drivers/clk/renesas/renesas-cpg-mssr.c index e1365e7491ae02..25c41cd9cdfc76 100644 --- a/drivers/clk/renesas/renesas-cpg-mssr.c +++ b/drivers/clk/renesas/renesas-cpg-mssr.c @@ -33,9 +33,9 @@ #include "clk-div6.h" #ifdef DEBUG -#define WARN_DEBUG(x) do { } while (0) -#else #define WARN_DEBUG(x) WARN_ON(x) +#else +#define WARN_DEBUG(x) do { } while (0) #endif diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-a23.c b/drivers/clk/sunxi-ng/ccu-sun8i-a23.c index 2646d980087bf3..5c6d37bdf247cf 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-a23.c +++ b/drivers/clk/sunxi-ng/ccu-sun8i-a23.c @@ -344,10 +344,10 @@ static SUNXI_CCU_MP_WITH_MUX_GATE(spi1_clk, "spi1", mod0_default_parents, 0x0a4, static const char * const i2s_parents[] = { "pll-audio-8x", "pll-audio-4x", "pll-audio-2x", "pll-audio" }; static SUNXI_CCU_MUX_WITH_GATE(i2s0_clk, "i2s0", i2s_parents, - 0x0b0, 16, 2, BIT(31), 0); + 0x0b0, 16, 2, BIT(31), CLK_SET_RATE_PARENT); static SUNXI_CCU_MUX_WITH_GATE(i2s1_clk, "i2s1", i2s_parents, - 0x0b4, 16, 2, BIT(31), 0); + 0x0b4, 16, 2, BIT(31), CLK_SET_RATE_PARENT); /* TODO: the parent for most of the USB clocks is not known */ static SUNXI_CCU_GATE(usb_phy0_clk, "usb-phy0", "osc24M", @@ -415,7 +415,7 @@ static SUNXI_CCU_M_WITH_GATE(ve_clk, "ve", "pll-ve", 0x13c, 16, 3, BIT(31), CLK_SET_RATE_PARENT); static SUNXI_CCU_GATE(ac_dig_clk, "ac-dig", "pll-audio", - 0x140, BIT(31), 0); + 0x140, BIT(31), CLK_SET_RATE_PARENT); static SUNXI_CCU_GATE(avs_clk, "avs", "osc24M", 0x144, BIT(31), 0); diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-h3.c b/drivers/clk/sunxi-ng/ccu-sun8i-h3.c index 4d70590f05e36e..21c427d86f289b 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-h3.c +++ b/drivers/clk/sunxi-ng/ccu-sun8i-h3.c @@ -394,16 +394,16 @@ static SUNXI_CCU_MP_WITH_MUX_GATE(spi1_clk, "spi1", mod0_default_parents, 0x0a4, static const char * const i2s_parents[] = { "pll-audio-8x", "pll-audio-4x", "pll-audio-2x", "pll-audio" }; static SUNXI_CCU_MUX_WITH_GATE(i2s0_clk, "i2s0", i2s_parents, - 0x0b0, 16, 2, BIT(31), 0); + 0x0b0, 16, 2, BIT(31), CLK_SET_RATE_PARENT); static SUNXI_CCU_MUX_WITH_GATE(i2s1_clk, "i2s1", i2s_parents, - 0x0b4, 16, 2, BIT(31), 0); + 0x0b4, 16, 2, BIT(31), CLK_SET_RATE_PARENT); static SUNXI_CCU_MUX_WITH_GATE(i2s2_clk, "i2s2", i2s_parents, - 0x0b8, 16, 2, BIT(31), 0); + 0x0b8, 16, 2, BIT(31), CLK_SET_RATE_PARENT); static SUNXI_CCU_M_WITH_GATE(spdif_clk, "spdif", "pll-audio", - 0x0c0, 0, 4, BIT(31), 0); + 0x0c0, 0, 4, BIT(31), CLK_SET_RATE_PARENT); static SUNXI_CCU_GATE(usb_phy0_clk, "usb-phy0", "osc24M", 0x0cc, BIT(8), 0); @@ -466,7 +466,7 @@ static SUNXI_CCU_M_WITH_GATE(ve_clk, "ve", "pll-ve", 0x13c, 16, 3, BIT(31), 0); static SUNXI_CCU_GATE(ac_dig_clk, "ac-dig", "pll-audio", - 0x140, BIT(31), 0); + 0x140, BIT(31), CLK_SET_RATE_PARENT); static SUNXI_CCU_GATE(avs_clk, "avs", "osc24M", 0x144, BIT(31), 0); diff --git a/drivers/clk/ti/clk-3xxx.c b/drivers/clk/ti/clk-3xxx.c index 8831e1a05367ad..11d8aa3ec18604 100644 --- a/drivers/clk/ti/clk-3xxx.c +++ b/drivers/clk/ti/clk-3xxx.c @@ -22,13 +22,6 @@ #include "clock.h" -/* - * DPLL5_FREQ_FOR_USBHOST: USBHOST and USBTLL are the only clocks - * that are sourced by DPLL5, and both of these require this clock - * to be at 120 MHz for proper operation. - */ -#define DPLL5_FREQ_FOR_USBHOST 120000000 - #define OMAP3430ES2_ST_DSS_IDLE_SHIFT 1 #define OMAP3430ES2_ST_HSOTGUSB_IDLE_SHIFT 5 #define OMAP3430ES2_ST_SSI_IDLE_SHIFT 8 @@ -546,14 +539,21 @@ void __init omap3_clk_lock_dpll5(void) struct clk *dpll5_clk; struct clk *dpll5_m2_clk; + /* + * Errata sprz319f advisory 2.1 documents a USB host clock drift issue + * that can be worked around using specially crafted dpll5 settings + * with a dpll5_m2 divider set to 8. Set the dpll5 rate to 8x the USB + * host clock rate, its .set_rate handler() will detect that frequency + * and use the errata settings. + */ dpll5_clk = clk_get(NULL, "dpll5_ck"); - clk_set_rate(dpll5_clk, DPLL5_FREQ_FOR_USBHOST); + clk_set_rate(dpll5_clk, OMAP3_DPLL5_FREQ_FOR_USBHOST * 8); clk_prepare_enable(dpll5_clk); - /* Program dpll5_m2_clk divider for no division */ + /* Program dpll5_m2_clk divider */ dpll5_m2_clk = clk_get(NULL, "dpll5_m2_ck"); clk_prepare_enable(dpll5_m2_clk); - clk_set_rate(dpll5_m2_clk, DPLL5_FREQ_FOR_USBHOST); + clk_set_rate(dpll5_m2_clk, OMAP3_DPLL5_FREQ_FOR_USBHOST); clk_disable_unprepare(dpll5_m2_clk); clk_disable_unprepare(dpll5_clk); diff --git a/drivers/clk/ti/clk-7xx.c b/drivers/clk/ti/clk-7xx.c index bfa17d33ef3b94..9fd6043314eb39 100644 --- a/drivers/clk/ti/clk-7xx.c +++ b/drivers/clk/ti/clk-7xx.c @@ -201,7 +201,6 @@ static struct ti_dt_clk dra7xx_clks[] = { DT_CLK(NULL, "atl_dpll_clk_mux", "atl_dpll_clk_mux"), DT_CLK(NULL, "atl_gfclk_mux", "atl_gfclk_mux"), DT_CLK(NULL, "dcan1_sys_clk_mux", "dcan1_sys_clk_mux"), - DT_CLK(NULL, "gmac_gmii_ref_clk_div", "gmac_gmii_ref_clk_div"), DT_CLK(NULL, "gmac_rft_clk_mux", "gmac_rft_clk_mux"), DT_CLK(NULL, "gpu_core_gclk_mux", "gpu_core_gclk_mux"), DT_CLK(NULL, "gpu_hyd_gclk_mux", "gpu_hyd_gclk_mux"), diff --git a/drivers/clk/ti/clock.h b/drivers/clk/ti/clock.h index 90f3f472ae1c4c..13c37f48d9d69d 100644 --- a/drivers/clk/ti/clock.h +++ b/drivers/clk/ti/clock.h @@ -257,11 +257,20 @@ long omap2_dpll_round_rate(struct clk_hw *hw, unsigned long target_rate, unsigned long omap3_clkoutx2_recalc(struct clk_hw *hw, unsigned long parent_rate); +/* + * OMAP3_DPLL5_FREQ_FOR_USBHOST: USBHOST and USBTLL are the only clocks + * that are sourced by DPLL5, and both of these require this clock + * to be at 120 MHz for proper operation. + */ +#define OMAP3_DPLL5_FREQ_FOR_USBHOST 120000000 + unsigned long omap3_dpll_recalc(struct clk_hw *hw, unsigned long parent_rate); int omap3_dpll4_set_rate(struct clk_hw *clk, unsigned long rate, unsigned long parent_rate); int omap3_dpll4_set_rate_and_parent(struct clk_hw *hw, unsigned long rate, unsigned long parent_rate, u8 index); +int omap3_dpll5_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate); void omap3_clk_lock_dpll5(void); unsigned long omap4_dpll_regm4xen_recalc(struct clk_hw *hw, diff --git a/drivers/clk/ti/dpll.c b/drivers/clk/ti/dpll.c index 9fc8754a6e6160..4b9a419d8e1413 100644 --- a/drivers/clk/ti/dpll.c +++ b/drivers/clk/ti/dpll.c @@ -114,6 +114,18 @@ static const struct clk_ops omap3_dpll_ck_ops = { .round_rate = &omap2_dpll_round_rate, }; +static const struct clk_ops omap3_dpll5_ck_ops = { + .enable = &omap3_noncore_dpll_enable, + .disable = &omap3_noncore_dpll_disable, + .get_parent = &omap2_init_dpll_parent, + .recalc_rate = &omap3_dpll_recalc, + .set_rate = &omap3_dpll5_set_rate, + .set_parent = &omap3_noncore_dpll_set_parent, + .set_rate_and_parent = &omap3_noncore_dpll_set_rate_and_parent, + .determine_rate = &omap3_noncore_dpll_determine_rate, + .round_rate = &omap2_dpll_round_rate, +}; + static const struct clk_ops omap3_dpll_per_ck_ops = { .enable = &omap3_noncore_dpll_enable, .disable = &omap3_noncore_dpll_disable, @@ -474,7 +486,12 @@ static void __init of_ti_omap3_dpll_setup(struct device_node *node) .modes = (1 << DPLL_LOW_POWER_BYPASS) | (1 << DPLL_LOCKED), }; - of_ti_dpll_setup(node, &omap3_dpll_ck_ops, &dd); + if ((of_machine_is_compatible("ti,omap3630") || + of_machine_is_compatible("ti,omap36xx")) && + !strcmp(node->name, "dpll5_ck")) + of_ti_dpll_setup(node, &omap3_dpll5_ck_ops, &dd); + else + of_ti_dpll_setup(node, &omap3_dpll_ck_ops, &dd); } CLK_OF_DECLARE(ti_omap3_dpll_clock, "ti,omap3-dpll-clock", of_ti_omap3_dpll_setup); diff --git a/drivers/clk/ti/dpll3xxx.c b/drivers/clk/ti/dpll3xxx.c index 88f2ce81ba5598..4cdd28a2558442 100644 --- a/drivers/clk/ti/dpll3xxx.c +++ b/drivers/clk/ti/dpll3xxx.c @@ -838,3 +838,70 @@ int omap3_dpll4_set_rate_and_parent(struct clk_hw *hw, unsigned long rate, return omap3_noncore_dpll_set_rate_and_parent(hw, rate, parent_rate, index); } + +/* Apply DM3730 errata sprz319 advisory 2.1. */ +static bool omap3_dpll5_apply_errata(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct omap3_dpll5_settings { + unsigned int rate, m, n; + }; + + static const struct omap3_dpll5_settings precomputed[] = { + /* + * From DM3730 errata advisory 2.1, table 35 and 36. + * The N value is increased by 1 compared to the tables as the + * errata lists register values while last_rounded_field is the + * real divider value. + */ + { 12000000, 80, 0 + 1 }, + { 13000000, 443, 5 + 1 }, + { 19200000, 50, 0 + 1 }, + { 26000000, 443, 11 + 1 }, + { 38400000, 25, 0 + 1 } + }; + + const struct omap3_dpll5_settings *d; + struct clk_hw_omap *clk = to_clk_hw_omap(hw); + struct dpll_data *dd; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(precomputed); ++i) { + if (parent_rate == precomputed[i].rate) + break; + } + + if (i == ARRAY_SIZE(precomputed)) + return false; + + d = &precomputed[i]; + + /* Update the M, N and rounded rate values and program the DPLL. */ + dd = clk->dpll_data; + dd->last_rounded_m = d->m; + dd->last_rounded_n = d->n; + dd->last_rounded_rate = div_u64((u64)parent_rate * d->m, d->n); + omap3_noncore_dpll_program(clk, 0); + + return true; +} + +/** + * omap3_dpll5_set_rate - set rate for omap3 dpll5 + * @hw: clock to change + * @rate: target rate for clock + * @parent_rate: rate of the parent clock + * + * Set rate for the DPLL5 clock. Apply the sprz319 advisory 2.1 on OMAP36xx if + * the DPLL is used for USB host (detected through the requested rate). + */ +int omap3_dpll5_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + if (rate == OMAP3_DPLL5_FREQ_FOR_USBHOST * 8) { + if (omap3_dpll5_apply_errata(hw, parent_rate)) + return 0; + } + + return omap3_noncore_dpll_set_rate(hw, rate, parent_rate); +} diff --git a/drivers/clocksource/exynos_mct.c b/drivers/clocksource/exynos_mct.c index 8f3488b808964b..7f6fed9f070353 100644 --- a/drivers/clocksource/exynos_mct.c +++ b/drivers/clocksource/exynos_mct.c @@ -495,6 +495,7 @@ static int exynos4_mct_dying_cpu(unsigned int cpu) if (mct_int_type == MCT_INT_SPI) { if (evt->irq != -1) disable_irq_nosync(evt->irq); + exynos4_mct_write(0x1, mevt->base + MCT_L_INT_CSTAT_OFFSET); } else { disable_percpu_irq(mct_irqs[MCT_L0_IRQ]); } diff --git a/drivers/cpufreq/Kconfig b/drivers/cpufreq/Kconfig index d8b164a7c4e517..17d7f54b4318e6 100644 --- a/drivers/cpufreq/Kconfig +++ b/drivers/cpufreq/Kconfig @@ -102,15 +102,23 @@ config CPU_FREQ_DEFAULT_GOV_CONSERVATIVE governor. If unsure have a look at the help section of the driver. Fallback governor will be the performance governor. -config CPU_FREQ_DEFAULT_GOV_SCHEDUTIL - bool "schedutil" - depends on SMP - select CPU_FREQ_GOV_SCHEDUTIL +config CPU_FREQ_DEFAULT_GOV_SCHED + bool "sched" + select CPU_FREQ_GOV_SCHED + help + Use the CPUfreq governor 'sched' as default. This scales + cpu frequency using CPU utilization estimates from the + scheduler. + +config CPU_FREQ_DEFAULT_GOV_INTERACTIVE + bool "interactive" + select CPU_FREQ_GOV_INTERACTIVE select CPU_FREQ_GOV_PERFORMANCE help - Use the 'schedutil' CPUFreq governor by default. If unsure, - have a look at the help section of that governor. The fallback - governor will be 'performance'. + Use the CPUFreq governor 'interactive' as default. This allows + you to get a full dynamic cpu frequency capable system by simply + loading your cpufreq low-level hardware driver, using the + 'interactive' governor for latency-sensitive workloads. endchoice @@ -193,20 +201,36 @@ config CPU_FREQ_GOV_CONSERVATIVE If in doubt, say N. -config CPU_FREQ_GOV_SCHEDUTIL - bool "'schedutil' cpufreq policy governor" - depends on CPU_FREQ && SMP +config CPU_FREQ_GOV_SCHED + bool "'sched' cpufreq governor" + depends on CPU_FREQ + depends on SMP + select CPU_FREQ_GOV_COMMON + help + 'sched' - this governor scales cpu frequency from the + scheduler as a function of cpu capacity utilization. It does + not evaluate utilization on a periodic basis (as ondemand + does) but instead is event-driven by the scheduler. + + If in doubt, say N. + +config CPU_FREQ_GOV_INTERACTIVE + tristate "'interactive' cpufreq policy governor" + depends on CPU_FREQ select CPU_FREQ_GOV_ATTR_SET select IRQ_WORK help - This governor makes decisions based on the utilization data provided - by the scheduler. It sets the CPU frequency to be proportional to - the utilization/capacity ratio coming from the scheduler. If the - utilization is frequency-invariant, the new frequency is also - proportional to the maximum available frequency. If that is not the - case, it is proportional to the current frequency of the CPU. The - frequency tipping point is at utilization/capacity equal to 80% in - both cases. + 'interactive' - This driver adds a dynamic cpufreq policy governor + designed for latency-sensitive workloads. + + This governor attempts to reduce the latency of clock + increases so that the system is more responsive to + interactive workloads. + + To compile this driver as a module, choose M here: the + module will be called cpufreq_interactive. + + For details, take a look at linux/Documentation/cpu-freq. If in doubt, say N. diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile index 0a9b6a093646bd..f0c9905d68a56f 100644 --- a/drivers/cpufreq/Makefile +++ b/drivers/cpufreq/Makefile @@ -10,6 +10,7 @@ obj-$(CONFIG_CPU_FREQ_GOV_POWERSAVE) += cpufreq_powersave.o obj-$(CONFIG_CPU_FREQ_GOV_USERSPACE) += cpufreq_userspace.o obj-$(CONFIG_CPU_FREQ_GOV_ONDEMAND) += cpufreq_ondemand.o obj-$(CONFIG_CPU_FREQ_GOV_CONSERVATIVE) += cpufreq_conservative.o +obj-$(CONFIG_CPU_FREQ_GOV_INTERACTIVE) += cpufreq_interactive.o obj-$(CONFIG_CPU_FREQ_GOV_COMMON) += cpufreq_governor.o obj-$(CONFIG_CPU_FREQ_GOV_ATTR_SET) += cpufreq_governor_attr_set.o diff --git a/drivers/cpufreq/cpufreq-dt.c b/drivers/cpufreq/cpufreq-dt.c index 5c07ae05d69ac8..4d3ec92cbabfe7 100644 --- a/drivers/cpufreq/cpufreq-dt.c +++ b/drivers/cpufreq/cpufreq-dt.c @@ -28,6 +28,7 @@ #include "cpufreq-dt.h" struct private_data { + struct opp_table *opp_table; struct device *cpu_dev; struct thermal_cooling_device *cdev; const char *reg_name; @@ -143,6 +144,7 @@ static int resources_available(void) static int cpufreq_init(struct cpufreq_policy *policy) { struct cpufreq_frequency_table *freq_table; + struct opp_table *opp_table = NULL; struct private_data *priv; struct device *cpu_dev; struct clk *cpu_clk; @@ -186,8 +188,9 @@ static int cpufreq_init(struct cpufreq_policy *policy) */ name = find_supply_name(cpu_dev); if (name) { - ret = dev_pm_opp_set_regulator(cpu_dev, name); - if (ret) { + opp_table = dev_pm_opp_set_regulator(cpu_dev, name); + if (IS_ERR(opp_table)) { + ret = PTR_ERR(opp_table); dev_err(cpu_dev, "Failed to set regulator for cpu%d: %d\n", policy->cpu, ret); goto out_put_clk; @@ -237,6 +240,7 @@ static int cpufreq_init(struct cpufreq_policy *policy) } priv->reg_name = name; + priv->opp_table = opp_table; ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table); if (ret) { @@ -285,7 +289,7 @@ static int cpufreq_init(struct cpufreq_policy *policy) out_free_opp: dev_pm_opp_of_cpumask_remove_table(policy->cpus); if (name) - dev_pm_opp_put_regulator(cpu_dev); + dev_pm_opp_put_regulator(opp_table); out_put_clk: clk_put(cpu_clk); @@ -300,7 +304,7 @@ static int cpufreq_exit(struct cpufreq_policy *policy) dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table); dev_pm_opp_of_cpumask_remove_table(policy->related_cpus); if (priv->reg_name) - dev_pm_opp_put_regulator(priv->cpu_dev); + dev_pm_opp_put_regulator(priv->opp_table); clk_put(policy->clk); kfree(priv); diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 6e6c1fb60fbc77..019e81797418af 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -29,6 +29,9 @@ #include #include #include +#ifdef CONFIG_SMP +#include +#endif #include static LIST_HEAD(cpufreq_policy_list); @@ -117,6 +120,12 @@ bool have_governor_per_policy(void) } EXPORT_SYMBOL_GPL(have_governor_per_policy); +bool cpufreq_driver_is_slow(void) +{ + return !(cpufreq_driver->flags & CPUFREQ_DRIVER_FAST); +} +EXPORT_SYMBOL_GPL(cpufreq_driver_is_slow); + struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy) { if (have_governor_per_policy()) @@ -301,6 +310,50 @@ static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci) #endif } +/********************************************************************* + * FREQUENCY INVARIANT CPU CAPACITY * + *********************************************************************/ + +static DEFINE_PER_CPU(unsigned long, freq_scale) = SCHED_CAPACITY_SCALE; +static DEFINE_PER_CPU(unsigned long, max_freq_scale) = SCHED_CAPACITY_SCALE; + +static void +scale_freq_capacity(struct cpufreq_policy *policy, struct cpufreq_freqs *freqs) +{ + unsigned long cur = freqs ? freqs->new : policy->cur; + unsigned long scale = (cur << SCHED_CAPACITY_SHIFT) / policy->max; + struct cpufreq_cpuinfo *cpuinfo = &policy->cpuinfo; + int cpu; + + pr_debug("cpus %*pbl cur/cur max freq %lu/%u kHz freq scale %lu\n", + cpumask_pr_args(policy->cpus), cur, policy->max, scale); + + for_each_cpu(cpu, policy->cpus) + per_cpu(freq_scale, cpu) = scale; + + if (freqs) + return; + + scale = (policy->max << SCHED_CAPACITY_SHIFT) / cpuinfo->max_freq; + + pr_debug("cpus %*pbl cur max/max freq %u/%u kHz max freq scale %lu\n", + cpumask_pr_args(policy->cpus), policy->max, cpuinfo->max_freq, + scale); + + for_each_cpu(cpu, policy->cpus) + per_cpu(max_freq_scale, cpu) = scale; +} + +unsigned long cpufreq_scale_freq_capacity(struct sched_domain *sd, int cpu) +{ + return per_cpu(freq_scale, cpu); +} + +unsigned long cpufreq_scale_max_freq_capacity(int cpu) +{ + return per_cpu(max_freq_scale, cpu); +} + static void __cpufreq_notify_transition(struct cpufreq_policy *policy, struct cpufreq_freqs *freqs, unsigned int state) { @@ -378,6 +431,9 @@ static void cpufreq_notify_post_transition(struct cpufreq_policy *policy, void cpufreq_freq_transition_begin(struct cpufreq_policy *policy, struct cpufreq_freqs *freqs) { +#ifdef CONFIG_SMP + int cpu; +#endif /* * Catch double invocations of _begin() which lead to self-deadlock. @@ -405,6 +461,12 @@ void cpufreq_freq_transition_begin(struct cpufreq_policy *policy, spin_unlock(&policy->transition_lock); + scale_freq_capacity(policy, freqs); +#ifdef CONFIG_SMP + for_each_cpu(cpu, policy->cpus) + trace_cpu_capacity(capacity_curr_of(cpu), cpu); +#endif + cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE); } EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin); @@ -2192,8 +2254,11 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy, blocking_notifier_call_chain(&cpufreq_policy_notifier_list, CPUFREQ_NOTIFY, new_policy); + scale_freq_capacity(new_policy, NULL); + policy->min = new_policy->min; policy->max = new_policy->max; + trace_cpu_frequency_limits(policy->max, policy->min, policy->cpu); policy->cached_target_freq = UINT_MAX; diff --git a/drivers/cpufreq/cpufreq_conservative.c b/drivers/cpufreq/cpufreq_conservative.c index 13475890d792d7..e8e16a5dbd1e4d 100644 --- a/drivers/cpufreq/cpufreq_conservative.c +++ b/drivers/cpufreq/cpufreq_conservative.c @@ -302,7 +302,10 @@ static void cs_start(struct cpufreq_policy *policy) dbs_info->requested_freq = policy->cur; } -static struct dbs_governor cs_governor = { +#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_CONSERVATIVE +static +#endif +struct dbs_governor cs_governor = { .gov = CPUFREQ_DBS_GOVERNOR_INITIALIZER("conservative"), .kobj_type = { .default_attrs = cs_attributes }, .gov_dbs_timer = cs_dbs_timer, diff --git a/drivers/cpufreq/cpufreq_interactive.c b/drivers/cpufreq/cpufreq_interactive.c new file mode 100644 index 00000000000000..d6cac0e8e7a688 --- /dev/null +++ b/drivers/cpufreq/cpufreq_interactive.c @@ -0,0 +1,1410 @@ +/* + * drivers/cpufreq/cpufreq_interactive.c + * + * Copyright (C) 2010-2016 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Author: Mike Chan (mike@android.com) + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define CREATE_TRACE_POINTS +#include + +#define gov_attr_ro(_name) \ +static struct governor_attr _name = \ +__ATTR(_name, 0444, show_##_name, NULL) + +#define gov_attr_wo(_name) \ +static struct governor_attr _name = \ +__ATTR(_name, 0200, NULL, store_##_name) + +#define gov_attr_rw(_name) \ +static struct governor_attr _name = \ +__ATTR(_name, 0644, show_##_name, store_##_name) + +/* Separate instance required for each 'interactive' directory in sysfs */ +struct interactive_tunables { + struct gov_attr_set attr_set; + + /* Hi speed to bump to from lo speed when load burst (default max) */ + unsigned int hispeed_freq; + + /* Go to hi speed when CPU load at or above this value. */ +#define DEFAULT_GO_HISPEED_LOAD 99 + unsigned long go_hispeed_load; + + /* Target load. Lower values result in higher CPU speeds. */ + spinlock_t target_loads_lock; + unsigned int *target_loads; + int ntarget_loads; + + /* + * The minimum amount of time to spend at a frequency before we can ramp + * down. + */ +#define DEFAULT_MIN_SAMPLE_TIME (80 * USEC_PER_MSEC) + unsigned long min_sample_time; + + /* The sample rate of the timer used to increase frequency */ + unsigned long sampling_rate; + + /* + * Wait this long before raising speed above hispeed, by default a + * single timer interval. + */ + spinlock_t above_hispeed_delay_lock; + unsigned int *above_hispeed_delay; + int nabove_hispeed_delay; + + /* Non-zero means indefinite speed boost active */ + int boost; + /* Duration of a boot pulse in usecs */ + int boostpulse_duration; + /* End time of boost pulse in ktime converted to usecs */ + u64 boostpulse_endtime; + bool boosted; + + /* + * Max additional time to wait in idle, beyond sampling_rate, at speeds + * above minimum before wakeup to reduce speed, or -1 if unnecessary. + */ +#define DEFAULT_TIMER_SLACK (4 * DEFAULT_SAMPLING_RATE) + unsigned long timer_slack_delay; + unsigned long timer_slack; + bool io_is_busy; +}; + +/* Separate instance required for each 'struct cpufreq_policy' */ +struct interactive_policy { + struct cpufreq_policy *policy; + struct interactive_tunables *tunables; + struct list_head tunables_hook; +}; + +/* Separate instance required for each CPU */ +struct interactive_cpu { + struct update_util_data update_util; + struct interactive_policy *ipolicy; + + struct irq_work irq_work; + u64 last_sample_time; + unsigned long next_sample_jiffies; + bool work_in_progress; + + struct rw_semaphore enable_sem; + struct timer_list slack_timer; + + spinlock_t load_lock; /* protects the next 4 fields */ + u64 time_in_idle; + u64 time_in_idle_timestamp; + u64 cputime_speedadj; + u64 cputime_speedadj_timestamp; + + spinlock_t target_freq_lock; /*protects target freq */ + unsigned int target_freq; + + unsigned int floor_freq; + u64 pol_floor_val_time; /* policy floor_validate_time */ + u64 loc_floor_val_time; /* per-cpu floor_validate_time */ + u64 pol_hispeed_val_time; /* policy hispeed_validate_time */ + u64 loc_hispeed_val_time; /* per-cpu hispeed_validate_time */ +}; + +static DEFINE_PER_CPU(struct interactive_cpu, interactive_cpu); + +/* Realtime thread handles frequency scaling */ +static struct task_struct *speedchange_task; +static cpumask_t speedchange_cpumask; +static spinlock_t speedchange_cpumask_lock; + +/* Target load. Lower values result in higher CPU speeds. */ +#define DEFAULT_TARGET_LOAD 90 +static unsigned int default_target_loads[] = {DEFAULT_TARGET_LOAD}; + +#define DEFAULT_SAMPLING_RATE (20 * USEC_PER_MSEC) +#define DEFAULT_ABOVE_HISPEED_DELAY DEFAULT_SAMPLING_RATE +static unsigned int default_above_hispeed_delay[] = { + DEFAULT_ABOVE_HISPEED_DELAY +}; + +/* Iterate over interactive policies for tunables */ +#define for_each_ipolicy(__ip) \ + list_for_each_entry(__ip, &tunables->attr_set.policy_list, tunables_hook) + +static struct interactive_tunables *global_tunables; +static DEFINE_MUTEX(global_tunables_lock); + +static inline void update_slack_delay(struct interactive_tunables *tunables) +{ + tunables->timer_slack_delay = usecs_to_jiffies(tunables->timer_slack + + tunables->sampling_rate); +} + +static bool timer_slack_required(struct interactive_cpu *icpu) +{ + struct interactive_policy *ipolicy = icpu->ipolicy; + struct interactive_tunables *tunables = ipolicy->tunables; + + if (tunables->timer_slack < 0) + return false; + + if (icpu->target_freq > ipolicy->policy->min) + return true; + + return false; +} + +static void gov_slack_timer_start(struct interactive_cpu *icpu, int cpu) +{ + struct interactive_tunables *tunables = icpu->ipolicy->tunables; + + icpu->slack_timer.expires = jiffies + tunables->timer_slack_delay; + add_timer_on(&icpu->slack_timer, cpu); +} + +static void gov_slack_timer_modify(struct interactive_cpu *icpu) +{ + struct interactive_tunables *tunables = icpu->ipolicy->tunables; + + mod_timer(&icpu->slack_timer, jiffies + tunables->timer_slack_delay); +} + +static void slack_timer_resched(struct interactive_cpu *icpu, int cpu, + bool modify) +{ + struct interactive_tunables *tunables = icpu->ipolicy->tunables; + unsigned long flags; + + spin_lock_irqsave(&icpu->load_lock, flags); + + icpu->time_in_idle = get_cpu_idle_time(cpu, + &icpu->time_in_idle_timestamp, + tunables->io_is_busy); + icpu->cputime_speedadj = 0; + icpu->cputime_speedadj_timestamp = icpu->time_in_idle_timestamp; + + if (timer_slack_required(icpu)) { + if (modify) + gov_slack_timer_modify(icpu); + else + gov_slack_timer_start(icpu, cpu); + } + + spin_unlock_irqrestore(&icpu->load_lock, flags); +} + +static unsigned int +freq_to_above_hispeed_delay(struct interactive_tunables *tunables, + unsigned int freq) +{ + unsigned long flags; + unsigned int ret; + int i; + + spin_lock_irqsave(&tunables->above_hispeed_delay_lock, flags); + + for (i = 0; i < tunables->nabove_hispeed_delay - 1 && + freq >= tunables->above_hispeed_delay[i + 1]; i += 2) + ; + + ret = tunables->above_hispeed_delay[i]; + spin_unlock_irqrestore(&tunables->above_hispeed_delay_lock, flags); + + return ret; +} + +static unsigned int freq_to_targetload(struct interactive_tunables *tunables, + unsigned int freq) +{ + unsigned long flags; + unsigned int ret; + int i; + + spin_lock_irqsave(&tunables->target_loads_lock, flags); + + for (i = 0; i < tunables->ntarget_loads - 1 && + freq >= tunables->target_loads[i + 1]; i += 2) + ; + + ret = tunables->target_loads[i]; + spin_unlock_irqrestore(&tunables->target_loads_lock, flags); + return ret; +} + +/* + * If increasing frequencies never map to a lower target load then + * choose_freq() will find the minimum frequency that does not exceed its + * target load given the current load. + */ +static unsigned int choose_freq(struct interactive_cpu *icpu, + unsigned int loadadjfreq) +{ + struct cpufreq_policy *policy = icpu->ipolicy->policy; + struct cpufreq_frequency_table *freq_table = policy->freq_table; + unsigned int prevfreq, freqmin = 0, freqmax = UINT_MAX, tl; + unsigned int freq = policy->cur; + int index; + + do { + prevfreq = freq; + tl = freq_to_targetload(icpu->ipolicy->tunables, freq); + + /* + * Find the lowest frequency where the computed load is less + * than or equal to the target load. + */ + + index = cpufreq_frequency_table_target(policy, loadadjfreq / tl, + CPUFREQ_RELATION_L); + + freq = freq_table[index].frequency; + + if (freq > prevfreq) { + /* The previous frequency is too low */ + freqmin = prevfreq; + + if (freq < freqmax) + continue; + + /* Find highest frequency that is less than freqmax */ + index = cpufreq_frequency_table_target(policy, + freqmax - 1, CPUFREQ_RELATION_H); + + freq = freq_table[index].frequency; + + if (freq == freqmin) { + /* + * The first frequency below freqmax has already + * been found to be too low. freqmax is the + * lowest speed we found that is fast enough. + */ + freq = freqmax; + break; + } + } else if (freq < prevfreq) { + /* The previous frequency is high enough. */ + freqmax = prevfreq; + + if (freq > freqmin) + continue; + + /* Find lowest frequency that is higher than freqmin */ + index = cpufreq_frequency_table_target(policy, + freqmin + 1, CPUFREQ_RELATION_L); + + freq = freq_table[index].frequency; + + /* + * If freqmax is the first frequency above + * freqmin then we have already found that + * this speed is fast enough. + */ + if (freq == freqmax) + break; + } + + /* If same frequency chosen as previous then done. */ + } while (freq != prevfreq); + + return freq; +} + +static u64 update_load(struct interactive_cpu *icpu, int cpu) +{ + struct interactive_tunables *tunables = icpu->ipolicy->tunables; + unsigned int delta_idle, delta_time; + u64 now_idle, now, active_time; + + now_idle = get_cpu_idle_time(cpu, &now, tunables->io_is_busy); + delta_idle = (unsigned int)(now_idle - icpu->time_in_idle); + delta_time = (unsigned int)(now - icpu->time_in_idle_timestamp); + + if (delta_time <= delta_idle) + active_time = 0; + else + active_time = delta_time - delta_idle; + + icpu->cputime_speedadj += active_time * icpu->ipolicy->policy->cur; + + icpu->time_in_idle = now_idle; + icpu->time_in_idle_timestamp = now; + + return now; +} + +/* Re-evaluate load to see if a frequency change is required or not */ +static void eval_target_freq(struct interactive_cpu *icpu) +{ + struct interactive_tunables *tunables = icpu->ipolicy->tunables; + struct cpufreq_policy *policy = icpu->ipolicy->policy; + struct cpufreq_frequency_table *freq_table = policy->freq_table; + u64 cputime_speedadj, now, max_fvtime; + unsigned int new_freq, loadadjfreq, index, delta_time; + unsigned long flags; + int cpu_load; + int cpu = smp_processor_id(); + + spin_lock_irqsave(&icpu->load_lock, flags); + now = update_load(icpu, smp_processor_id()); + delta_time = (unsigned int)(now - icpu->cputime_speedadj_timestamp); + cputime_speedadj = icpu->cputime_speedadj; + spin_unlock_irqrestore(&icpu->load_lock, flags); + + if (WARN_ON_ONCE(!delta_time)) + return; + + spin_lock_irqsave(&icpu->target_freq_lock, flags); + do_div(cputime_speedadj, delta_time); + loadadjfreq = (unsigned int)cputime_speedadj * 100; + cpu_load = loadadjfreq / policy->cur; + tunables->boosted = tunables->boost || + now < tunables->boostpulse_endtime; + + if (cpu_load >= tunables->go_hispeed_load || tunables->boosted) { + if (policy->cur < tunables->hispeed_freq) { + new_freq = tunables->hispeed_freq; + } else { + new_freq = choose_freq(icpu, loadadjfreq); + + if (new_freq < tunables->hispeed_freq) + new_freq = tunables->hispeed_freq; + } + } else { + new_freq = choose_freq(icpu, loadadjfreq); + if (new_freq > tunables->hispeed_freq && + policy->cur < tunables->hispeed_freq) + new_freq = tunables->hispeed_freq; + } + + if (policy->cur >= tunables->hispeed_freq && + new_freq > policy->cur && + now - icpu->pol_hispeed_val_time < freq_to_above_hispeed_delay(tunables, policy->cur)) { + trace_cpufreq_interactive_notyet(cpu, cpu_load, + icpu->target_freq, policy->cur, new_freq); + goto exit; + } + + icpu->loc_hispeed_val_time = now; + + index = cpufreq_frequency_table_target(policy, new_freq, + CPUFREQ_RELATION_L); + new_freq = freq_table[index].frequency; + + /* + * Do not scale below floor_freq unless we have been at or above the + * floor frequency for the minimum sample time since last validated. + */ + max_fvtime = max(icpu->pol_floor_val_time, icpu->loc_floor_val_time); + if (new_freq < icpu->floor_freq && icpu->target_freq >= policy->cur) { + if (now - max_fvtime < tunables->min_sample_time) { + trace_cpufreq_interactive_notyet(cpu, cpu_load, + icpu->target_freq, policy->cur, new_freq); + goto exit; + } + } + + /* + * Update the timestamp for checking whether speed has been held at + * or above the selected frequency for a minimum of min_sample_time, + * if not boosted to hispeed_freq. If boosted to hispeed_freq then we + * allow the speed to drop as soon as the boostpulse duration expires + * (or the indefinite boost is turned off). + */ + + if (!tunables->boosted || new_freq > tunables->hispeed_freq) { + icpu->floor_freq = new_freq; + if (icpu->target_freq >= policy->cur || new_freq >= policy->cur) + icpu->loc_floor_val_time = now; + } + + if (icpu->target_freq == new_freq && + icpu->target_freq <= policy->cur) { + trace_cpufreq_interactive_already(cpu, cpu_load, + icpu->target_freq, policy->cur, new_freq); + goto exit; + } + + trace_cpufreq_interactive_target(cpu, cpu_load, icpu->target_freq, + policy->cur, new_freq); + + icpu->target_freq = new_freq; + spin_unlock_irqrestore(&icpu->target_freq_lock, flags); + + spin_lock_irqsave(&speedchange_cpumask_lock, flags); + cpumask_set_cpu(cpu, &speedchange_cpumask); + spin_unlock_irqrestore(&speedchange_cpumask_lock, flags); + + wake_up_process(speedchange_task); + return; + +exit: + spin_unlock_irqrestore(&icpu->target_freq_lock, flags); +} + +static void cpufreq_interactive_update(struct interactive_cpu *icpu) +{ + eval_target_freq(icpu); + slack_timer_resched(icpu, smp_processor_id(), true); +} + +static void cpufreq_interactive_idle_end(void) +{ + struct interactive_cpu *icpu = &per_cpu(interactive_cpu, + smp_processor_id()); + + if (!down_read_trylock(&icpu->enable_sem)) + return; + + if (icpu->ipolicy) { + /* + * We haven't sampled load for more than sampling_rate time, do + * it right now. + */ + if (time_after_eq(jiffies, icpu->next_sample_jiffies)) + cpufreq_interactive_update(icpu); + } + + up_read(&icpu->enable_sem); +} + +static void cpufreq_interactive_get_policy_info(struct cpufreq_policy *policy, + unsigned int *pmax_freq, + u64 *phvt, u64 *pfvt) +{ + struct interactive_cpu *icpu; + u64 hvt = ~0ULL, fvt = 0; + unsigned int max_freq = 0, i; + + for_each_cpu(i, policy->cpus) { + icpu = &per_cpu(interactive_cpu, i); + + fvt = max(fvt, icpu->loc_floor_val_time); + if (icpu->target_freq > max_freq) { + max_freq = icpu->target_freq; + hvt = icpu->loc_hispeed_val_time; + } else if (icpu->target_freq == max_freq) { + hvt = min(hvt, icpu->loc_hispeed_val_time); + } + } + + *pmax_freq = max_freq; + *phvt = hvt; + *pfvt = fvt; +} + +static void cpufreq_interactive_adjust_cpu(unsigned int cpu, + struct cpufreq_policy *policy) +{ + struct interactive_cpu *icpu; + u64 hvt, fvt; + unsigned int max_freq; + int i; + + cpufreq_interactive_get_policy_info(policy, &max_freq, &hvt, &fvt); + + for_each_cpu(i, policy->cpus) { + icpu = &per_cpu(interactive_cpu, i); + icpu->pol_floor_val_time = fvt; + } + + if (max_freq != policy->cur) { + __cpufreq_driver_target(policy, max_freq, CPUFREQ_RELATION_H); + for_each_cpu(i, policy->cpus) { + icpu = &per_cpu(interactive_cpu, i); + icpu->pol_hispeed_val_time = hvt; + } + } + + trace_cpufreq_interactive_setspeed(cpu, max_freq, policy->cur); +} + +static int cpufreq_interactive_speedchange_task(void *data) +{ + unsigned int cpu; + cpumask_t tmp_mask; + unsigned long flags; + +again: + set_current_state(TASK_INTERRUPTIBLE); + spin_lock_irqsave(&speedchange_cpumask_lock, flags); + + if (cpumask_empty(&speedchange_cpumask)) { + spin_unlock_irqrestore(&speedchange_cpumask_lock, flags); + schedule(); + + if (kthread_should_stop()) + return 0; + + spin_lock_irqsave(&speedchange_cpumask_lock, flags); + } + + set_current_state(TASK_RUNNING); + tmp_mask = speedchange_cpumask; + cpumask_clear(&speedchange_cpumask); + spin_unlock_irqrestore(&speedchange_cpumask_lock, flags); + + for_each_cpu(cpu, &tmp_mask) { + struct interactive_cpu *icpu = &per_cpu(interactive_cpu, cpu); + struct cpufreq_policy *policy = icpu->ipolicy->policy; + + if (unlikely(!down_read_trylock(&icpu->enable_sem))) + continue; + + if (likely(icpu->ipolicy)) + cpufreq_interactive_adjust_cpu(cpu, policy); + + up_read(&icpu->enable_sem); + } + + goto again; +} + +static void cpufreq_interactive_boost(struct interactive_tunables *tunables) +{ + struct interactive_policy *ipolicy; + struct cpufreq_policy *policy; + struct interactive_cpu *icpu; + unsigned long flags[2]; + bool wakeup = false; + int i; + + tunables->boosted = true; + + spin_lock_irqsave(&speedchange_cpumask_lock, flags[0]); + + for_each_ipolicy(ipolicy) { + policy = ipolicy->policy; + + for_each_cpu(i, policy->cpus) { + icpu = &per_cpu(interactive_cpu, i); + + if (!down_read_trylock(&icpu->enable_sem)) + continue; + + if (!icpu->ipolicy) { + up_read(&icpu->enable_sem); + continue; + } + + spin_lock_irqsave(&icpu->target_freq_lock, flags[1]); + if (icpu->target_freq < tunables->hispeed_freq) { + icpu->target_freq = tunables->hispeed_freq; + cpumask_set_cpu(i, &speedchange_cpumask); + icpu->pol_hispeed_val_time = ktime_to_us(ktime_get()); + wakeup = true; + } + spin_unlock_irqrestore(&icpu->target_freq_lock, flags[1]); + + up_read(&icpu->enable_sem); + } + } + + spin_unlock_irqrestore(&speedchange_cpumask_lock, flags[0]); + + if (wakeup) + wake_up_process(speedchange_task); +} + +static int cpufreq_interactive_notifier(struct notifier_block *nb, + unsigned long val, void *data) +{ + struct cpufreq_freqs *freq = data; + struct interactive_cpu *icpu = &per_cpu(interactive_cpu, freq->cpu); + unsigned long flags; + + if (val != CPUFREQ_POSTCHANGE) + return 0; + + if (!down_read_trylock(&icpu->enable_sem)) + return 0; + + if (!icpu->ipolicy) { + up_read(&icpu->enable_sem); + return 0; + } + + spin_lock_irqsave(&icpu->load_lock, flags); + update_load(icpu, freq->cpu); + spin_unlock_irqrestore(&icpu->load_lock, flags); + + up_read(&icpu->enable_sem); + + return 0; +} + +static struct notifier_block cpufreq_notifier_block = { + .notifier_call = cpufreq_interactive_notifier, +}; + +static unsigned int *get_tokenized_data(const char *buf, int *num_tokens) +{ + const char *cp = buf; + int ntokens = 1, i = 0; + unsigned int *tokenized_data; + int err = -EINVAL; + + while ((cp = strpbrk(cp + 1, " :"))) + ntokens++; + + if (!(ntokens & 0x1)) + goto err; + + tokenized_data = kcalloc(ntokens, sizeof(*tokenized_data), GFP_KERNEL); + if (!tokenized_data) { + err = -ENOMEM; + goto err; + } + + cp = buf; + while (i < ntokens) { + if (kstrtouint(cp, 0, &tokenized_data[i++]) < 0) + goto err_kfree; + + cp = strpbrk(cp, " :"); + if (!cp) + break; + cp++; + } + + if (i != ntokens) + goto err_kfree; + + *num_tokens = ntokens; + return tokenized_data; + +err_kfree: + kfree(tokenized_data); +err: + return ERR_PTR(err); +} + +/* Interactive governor sysfs interface */ +static struct interactive_tunables *to_tunables(struct gov_attr_set *attr_set) +{ + return container_of(attr_set, struct interactive_tunables, attr_set); +} + +#define show_one(file_name, type) \ +static ssize_t show_##file_name(struct gov_attr_set *attr_set, char *buf) \ +{ \ + struct interactive_tunables *tunables = to_tunables(attr_set); \ + return sprintf(buf, type "\n", tunables->file_name); \ +} + +static ssize_t show_target_loads(struct gov_attr_set *attr_set, char *buf) +{ + struct interactive_tunables *tunables = to_tunables(attr_set); + unsigned long flags; + ssize_t ret = 0; + int i; + + spin_lock_irqsave(&tunables->target_loads_lock, flags); + + for (i = 0; i < tunables->ntarget_loads; i++) + ret += sprintf(buf + ret, "%u%s", tunables->target_loads[i], + i & 0x1 ? ":" : " "); + + sprintf(buf + ret - 1, "\n"); + spin_unlock_irqrestore(&tunables->target_loads_lock, flags); + + return ret; +} + +static ssize_t store_target_loads(struct gov_attr_set *attr_set, + const char *buf, size_t count) +{ + struct interactive_tunables *tunables = to_tunables(attr_set); + unsigned int *new_target_loads; + unsigned long flags; + int ntokens; + + new_target_loads = get_tokenized_data(buf, &ntokens); + if (IS_ERR(new_target_loads)) + return PTR_ERR(new_target_loads); + + spin_lock_irqsave(&tunables->target_loads_lock, flags); + if (tunables->target_loads != default_target_loads) + kfree(tunables->target_loads); + tunables->target_loads = new_target_loads; + tunables->ntarget_loads = ntokens; + spin_unlock_irqrestore(&tunables->target_loads_lock, flags); + + return count; +} + +static ssize_t show_above_hispeed_delay(struct gov_attr_set *attr_set, + char *buf) +{ + struct interactive_tunables *tunables = to_tunables(attr_set); + unsigned long flags; + ssize_t ret = 0; + int i; + + spin_lock_irqsave(&tunables->above_hispeed_delay_lock, flags); + + for (i = 0; i < tunables->nabove_hispeed_delay; i++) + ret += sprintf(buf + ret, "%u%s", + tunables->above_hispeed_delay[i], + i & 0x1 ? ":" : " "); + + sprintf(buf + ret - 1, "\n"); + spin_unlock_irqrestore(&tunables->above_hispeed_delay_lock, flags); + + return ret; +} + +static ssize_t store_above_hispeed_delay(struct gov_attr_set *attr_set, + const char *buf, size_t count) +{ + struct interactive_tunables *tunables = to_tunables(attr_set); + unsigned int *new_above_hispeed_delay = NULL; + unsigned long flags; + int ntokens; + + new_above_hispeed_delay = get_tokenized_data(buf, &ntokens); + if (IS_ERR(new_above_hispeed_delay)) + return PTR_ERR(new_above_hispeed_delay); + + spin_lock_irqsave(&tunables->above_hispeed_delay_lock, flags); + if (tunables->above_hispeed_delay != default_above_hispeed_delay) + kfree(tunables->above_hispeed_delay); + tunables->above_hispeed_delay = new_above_hispeed_delay; + tunables->nabove_hispeed_delay = ntokens; + spin_unlock_irqrestore(&tunables->above_hispeed_delay_lock, flags); + + return count; +} + +static ssize_t store_hispeed_freq(struct gov_attr_set *attr_set, + const char *buf, size_t count) +{ + struct interactive_tunables *tunables = to_tunables(attr_set); + unsigned long int val; + int ret; + + ret = kstrtoul(buf, 0, &val); + if (ret < 0) + return ret; + + tunables->hispeed_freq = val; + + return count; +} + +static ssize_t store_go_hispeed_load(struct gov_attr_set *attr_set, + const char *buf, size_t count) +{ + struct interactive_tunables *tunables = to_tunables(attr_set); + unsigned long val; + int ret; + + ret = kstrtoul(buf, 0, &val); + if (ret < 0) + return ret; + + tunables->go_hispeed_load = val; + + return count; +} + +static ssize_t store_min_sample_time(struct gov_attr_set *attr_set, + const char *buf, size_t count) +{ + struct interactive_tunables *tunables = to_tunables(attr_set); + unsigned long val; + int ret; + + ret = kstrtoul(buf, 0, &val); + if (ret < 0) + return ret; + + tunables->min_sample_time = val; + + return count; +} + +static ssize_t show_timer_rate(struct gov_attr_set *attr_set, char *buf) +{ + struct interactive_tunables *tunables = to_tunables(attr_set); + + return sprintf(buf, "%lu\n", tunables->sampling_rate); +} + +static ssize_t store_timer_rate(struct gov_attr_set *attr_set, const char *buf, + size_t count) +{ + struct interactive_tunables *tunables = to_tunables(attr_set); + unsigned long val, val_round; + int ret; + + ret = kstrtoul(buf, 0, &val); + if (ret < 0) + return ret; + + val_round = jiffies_to_usecs(usecs_to_jiffies(val)); + if (val != val_round) + pr_warn("timer_rate not aligned to jiffy. Rounded up to %lu\n", + val_round); + + tunables->sampling_rate = val_round; + + return count; +} + +static ssize_t store_timer_slack(struct gov_attr_set *attr_set, const char *buf, + size_t count) +{ + struct interactive_tunables *tunables = to_tunables(attr_set); + unsigned long val; + int ret; + + ret = kstrtol(buf, 10, &val); + if (ret < 0) + return ret; + + tunables->timer_slack = val; + update_slack_delay(tunables); + + return count; +} + +static ssize_t store_boost(struct gov_attr_set *attr_set, const char *buf, + size_t count) +{ + struct interactive_tunables *tunables = to_tunables(attr_set); + unsigned long val; + int ret; + + ret = kstrtoul(buf, 0, &val); + if (ret < 0) + return ret; + + tunables->boost = val; + + if (tunables->boost) { + trace_cpufreq_interactive_boost("on"); + if (!tunables->boosted) + cpufreq_interactive_boost(tunables); + } else { + tunables->boostpulse_endtime = ktime_to_us(ktime_get()); + trace_cpufreq_interactive_unboost("off"); + } + + return count; +} + +static ssize_t store_boostpulse(struct gov_attr_set *attr_set, const char *buf, + size_t count) +{ + struct interactive_tunables *tunables = to_tunables(attr_set); + unsigned long val; + int ret; + + ret = kstrtoul(buf, 0, &val); + if (ret < 0) + return ret; + + tunables->boostpulse_endtime = ktime_to_us(ktime_get()) + + tunables->boostpulse_duration; + trace_cpufreq_interactive_boost("pulse"); + if (!tunables->boosted) + cpufreq_interactive_boost(tunables); + + return count; +} + +static ssize_t store_boostpulse_duration(struct gov_attr_set *attr_set, + const char *buf, size_t count) +{ + struct interactive_tunables *tunables = to_tunables(attr_set); + unsigned long val; + int ret; + + ret = kstrtoul(buf, 0, &val); + if (ret < 0) + return ret; + + tunables->boostpulse_duration = val; + + return count; +} + +static ssize_t store_io_is_busy(struct gov_attr_set *attr_set, const char *buf, + size_t count) +{ + struct interactive_tunables *tunables = to_tunables(attr_set); + unsigned long val; + int ret; + + ret = kstrtoul(buf, 0, &val); + if (ret < 0) + return ret; + + tunables->io_is_busy = val; + + return count; +} + +show_one(hispeed_freq, "%u"); +show_one(go_hispeed_load, "%lu"); +show_one(min_sample_time, "%lu"); +show_one(timer_slack, "%lu"); +show_one(boost, "%u"); +show_one(boostpulse_duration, "%u"); +show_one(io_is_busy, "%u"); + +gov_attr_rw(target_loads); +gov_attr_rw(above_hispeed_delay); +gov_attr_rw(hispeed_freq); +gov_attr_rw(go_hispeed_load); +gov_attr_rw(min_sample_time); +gov_attr_rw(timer_rate); +gov_attr_rw(timer_slack); +gov_attr_rw(boost); +gov_attr_wo(boostpulse); +gov_attr_rw(boostpulse_duration); +gov_attr_rw(io_is_busy); + +static struct attribute *interactive_attributes[] = { + &target_loads.attr, + &above_hispeed_delay.attr, + &hispeed_freq.attr, + &go_hispeed_load.attr, + &min_sample_time.attr, + &timer_rate.attr, + &timer_slack.attr, + &boost.attr, + &boostpulse.attr, + &boostpulse_duration.attr, + &io_is_busy.attr, + NULL +}; + +static struct kobj_type interactive_tunables_ktype = { + .default_attrs = interactive_attributes, + .sysfs_ops = &governor_sysfs_ops, +}; + +static int cpufreq_interactive_idle_notifier(struct notifier_block *nb, + unsigned long val, void *data) +{ + if (val == IDLE_END) + cpufreq_interactive_idle_end(); + + return 0; +} + +static struct notifier_block cpufreq_interactive_idle_nb = { + .notifier_call = cpufreq_interactive_idle_notifier, +}; + +/* Interactive Governor callbacks */ +struct interactive_governor { + struct cpufreq_governor gov; + unsigned int usage_count; +}; + +static struct interactive_governor interactive_gov; + +#define CPU_FREQ_GOV_INTERACTIVE (&interactive_gov.gov) + +static void irq_work(struct irq_work *irq_work) +{ + struct interactive_cpu *icpu = container_of(irq_work, struct + interactive_cpu, irq_work); + + cpufreq_interactive_update(icpu); + icpu->work_in_progress = false; +} + +static void update_util_handler(struct update_util_data *data, u64 time, + unsigned int flags) +{ + struct interactive_cpu *icpu = container_of(data, + struct interactive_cpu, update_util); + struct interactive_policy *ipolicy = icpu->ipolicy; + struct interactive_tunables *tunables = ipolicy->tunables; + u64 delta_ns; + + /* + * The irq-work may not be allowed to be queued up right now. + * Possible reasons: + * - Work has already been queued up or is in progress. + * - It is too early (too little time from the previous sample). + */ + if (icpu->work_in_progress) + return; + + delta_ns = time - icpu->last_sample_time; + if ((s64)delta_ns < tunables->sampling_rate * NSEC_PER_USEC) + return; + + icpu->last_sample_time = time; + icpu->next_sample_jiffies = usecs_to_jiffies(tunables->sampling_rate) + + jiffies; + + icpu->work_in_progress = true; + irq_work_queue(&icpu->irq_work); +} + +static void gov_set_update_util(struct interactive_policy *ipolicy) +{ + struct cpufreq_policy *policy = ipolicy->policy; + struct interactive_cpu *icpu; + int cpu; + + for_each_cpu(cpu, policy->cpus) { + icpu = &per_cpu(interactive_cpu, cpu); + + icpu->last_sample_time = 0; + icpu->next_sample_jiffies = 0; + cpufreq_add_update_util_hook(cpu, &icpu->update_util, + update_util_handler); + } +} + +static inline void gov_clear_update_util(struct cpufreq_policy *policy) +{ + int i; + + for_each_cpu(i, policy->cpus) + cpufreq_remove_update_util_hook(i); + + synchronize_sched(); +} + +static void icpu_cancel_work(struct interactive_cpu *icpu) +{ + irq_work_sync(&icpu->irq_work); + icpu->work_in_progress = false; + del_timer_sync(&icpu->slack_timer); +} + +static struct interactive_policy * +interactive_policy_alloc(struct cpufreq_policy *policy) +{ + struct interactive_policy *ipolicy; + + ipolicy = kzalloc(sizeof(*ipolicy), GFP_KERNEL); + if (!ipolicy) + return NULL; + + ipolicy->policy = policy; + + return ipolicy; +} + +static void interactive_policy_free(struct interactive_policy *ipolicy) +{ + kfree(ipolicy); +} + +static struct interactive_tunables * +interactive_tunables_alloc(struct interactive_policy *ipolicy) +{ + struct interactive_tunables *tunables; + + tunables = kzalloc(sizeof(*tunables), GFP_KERNEL); + if (!tunables) + return NULL; + + gov_attr_set_init(&tunables->attr_set, &ipolicy->tunables_hook); + if (!have_governor_per_policy()) + global_tunables = tunables; + + ipolicy->tunables = tunables; + + return tunables; +} + +static void interactive_tunables_free(struct interactive_tunables *tunables) +{ + if (!have_governor_per_policy()) + global_tunables = NULL; + + kfree(tunables); +} + +int cpufreq_interactive_init(struct cpufreq_policy *policy) +{ + struct interactive_policy *ipolicy; + struct interactive_tunables *tunables; + int ret; + + /* State should be equivalent to EXIT */ + if (policy->governor_data) + return -EBUSY; + + ipolicy = interactive_policy_alloc(policy); + if (!ipolicy) + return -ENOMEM; + + mutex_lock(&global_tunables_lock); + + if (global_tunables) { + if (WARN_ON(have_governor_per_policy())) { + ret = -EINVAL; + goto free_int_policy; + } + + policy->governor_data = ipolicy; + ipolicy->tunables = global_tunables; + + gov_attr_set_get(&global_tunables->attr_set, + &ipolicy->tunables_hook); + goto out; + } + + tunables = interactive_tunables_alloc(ipolicy); + if (!tunables) { + ret = -ENOMEM; + goto free_int_policy; + } + + tunables->hispeed_freq = policy->max; + tunables->above_hispeed_delay = default_above_hispeed_delay; + tunables->nabove_hispeed_delay = + ARRAY_SIZE(default_above_hispeed_delay); + tunables->go_hispeed_load = DEFAULT_GO_HISPEED_LOAD; + tunables->target_loads = default_target_loads; + tunables->ntarget_loads = ARRAY_SIZE(default_target_loads); + tunables->min_sample_time = DEFAULT_MIN_SAMPLE_TIME; + tunables->boostpulse_duration = DEFAULT_MIN_SAMPLE_TIME; + tunables->sampling_rate = DEFAULT_SAMPLING_RATE; + tunables->timer_slack = DEFAULT_TIMER_SLACK; + update_slack_delay(tunables); + + spin_lock_init(&tunables->target_loads_lock); + spin_lock_init(&tunables->above_hispeed_delay_lock); + + policy->governor_data = ipolicy; + + ret = kobject_init_and_add(&tunables->attr_set.kobj, + &interactive_tunables_ktype, + get_governor_parent_kobj(policy), "%s", + interactive_gov.gov.name); + if (ret) + goto fail; + + /* One time initialization for governor */ + if (!interactive_gov.usage_count++) { + idle_notifier_register(&cpufreq_interactive_idle_nb); + cpufreq_register_notifier(&cpufreq_notifier_block, + CPUFREQ_TRANSITION_NOTIFIER); + } + + out: + mutex_unlock(&global_tunables_lock); + return 0; + + fail: + policy->governor_data = NULL; + interactive_tunables_free(tunables); + + free_int_policy: + mutex_unlock(&global_tunables_lock); + + interactive_policy_free(ipolicy); + pr_err("governor initialization failed (%d)\n", ret); + + return ret; +} + +void cpufreq_interactive_exit(struct cpufreq_policy *policy) +{ + struct interactive_policy *ipolicy = policy->governor_data; + struct interactive_tunables *tunables = ipolicy->tunables; + unsigned int count; + + mutex_lock(&global_tunables_lock); + + /* Last policy using the governor ? */ + if (!--interactive_gov.usage_count) { + cpufreq_unregister_notifier(&cpufreq_notifier_block, + CPUFREQ_TRANSITION_NOTIFIER); + idle_notifier_unregister(&cpufreq_interactive_idle_nb); + } + + count = gov_attr_set_put(&tunables->attr_set, &ipolicy->tunables_hook); + policy->governor_data = NULL; + if (!count) + interactive_tunables_free(tunables); + + mutex_unlock(&global_tunables_lock); + + interactive_policy_free(ipolicy); +} + +int cpufreq_interactive_start(struct cpufreq_policy *policy) +{ + struct interactive_policy *ipolicy = policy->governor_data; + struct interactive_cpu *icpu; + unsigned int cpu; + + for_each_cpu(cpu, policy->cpus) { + icpu = &per_cpu(interactive_cpu, cpu); + + icpu->target_freq = policy->cur; + icpu->floor_freq = icpu->target_freq; + icpu->pol_floor_val_time = ktime_to_us(ktime_get()); + icpu->loc_floor_val_time = icpu->pol_floor_val_time; + icpu->pol_hispeed_val_time = icpu->pol_floor_val_time; + icpu->loc_hispeed_val_time = icpu->pol_floor_val_time; + + down_write(&icpu->enable_sem); + icpu->ipolicy = ipolicy; + up_write(&icpu->enable_sem); + + slack_timer_resched(icpu, cpu, false); + } + + gov_set_update_util(ipolicy); + return 0; +} + +void cpufreq_interactive_stop(struct cpufreq_policy *policy) +{ + struct interactive_policy *ipolicy = policy->governor_data; + struct interactive_cpu *icpu; + unsigned int cpu; + + gov_clear_update_util(ipolicy->policy); + + for_each_cpu(cpu, policy->cpus) { + icpu = &per_cpu(interactive_cpu, cpu); + + icpu_cancel_work(icpu); + + down_write(&icpu->enable_sem); + icpu->ipolicy = NULL; + up_write(&icpu->enable_sem); + } +} + +void cpufreq_interactive_limits(struct cpufreq_policy *policy) +{ + struct interactive_cpu *icpu; + unsigned int cpu; + unsigned long flags; + + cpufreq_policy_apply_limits(policy); + + for_each_cpu(cpu, policy->cpus) { + icpu = &per_cpu(interactive_cpu, cpu); + + spin_lock_irqsave(&icpu->target_freq_lock, flags); + + if (policy->max < icpu->target_freq) + icpu->target_freq = policy->max; + else if (policy->min > icpu->target_freq) + icpu->target_freq = policy->min; + + spin_unlock_irqrestore(&icpu->target_freq_lock, flags); + } +} + +static struct interactive_governor interactive_gov = { + .gov = { + .name = "interactive", + .max_transition_latency = TRANSITION_LATENCY_LIMIT, + .owner = THIS_MODULE, + .init = cpufreq_interactive_init, + .exit = cpufreq_interactive_exit, + .start = cpufreq_interactive_start, + .stop = cpufreq_interactive_stop, + .limits = cpufreq_interactive_limits, + } +}; + +static void cpufreq_interactive_nop_timer(unsigned long data) +{ + /* + * The purpose of slack-timer is to wake up the CPU from IDLE, in order + * to decrease its frequency if it is not set to minimum already. + * + * This is important for platforms where CPU with higher frequencies + * consume higher power even at IDLE. + */ +} + +static int __init cpufreq_interactive_gov_init(void) +{ + struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 }; + struct interactive_cpu *icpu; + unsigned int cpu; + + for_each_possible_cpu(cpu) { + icpu = &per_cpu(interactive_cpu, cpu); + + init_irq_work(&icpu->irq_work, irq_work); + spin_lock_init(&icpu->load_lock); + spin_lock_init(&icpu->target_freq_lock); + init_rwsem(&icpu->enable_sem); + + /* Initialize per-cpu slack-timer */ + init_timer_pinned(&icpu->slack_timer); + icpu->slack_timer.function = cpufreq_interactive_nop_timer; + } + + spin_lock_init(&speedchange_cpumask_lock); + speedchange_task = kthread_create(cpufreq_interactive_speedchange_task, + NULL, "cfinteractive"); + if (IS_ERR(speedchange_task)) + return PTR_ERR(speedchange_task); + + sched_setscheduler_nocheck(speedchange_task, SCHED_FIFO, ¶m); + get_task_struct(speedchange_task); + + /* wake up so the thread does not look hung to the freezer */ + wake_up_process(speedchange_task); + + return cpufreq_register_governor(CPU_FREQ_GOV_INTERACTIVE); +} + +#ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE +struct cpufreq_governor *cpufreq_default_governor(void) +{ + return CPU_FREQ_GOV_INTERACTIVE; +} + +fs_initcall(cpufreq_interactive_gov_init); +#else +module_init(cpufreq_interactive_gov_init); +#endif + +static void __exit cpufreq_interactive_gov_exit(void) +{ + cpufreq_unregister_governor(CPU_FREQ_GOV_INTERACTIVE); + kthread_stop(speedchange_task); + put_task_struct(speedchange_task); +} +module_exit(cpufreq_interactive_gov_exit); + +MODULE_AUTHOR("Mike Chan "); +MODULE_DESCRIPTION("'cpufreq_interactive' - A dynamic cpufreq governor for Latency sensitive workloads"); +MODULE_LICENSE("GPL"); diff --git a/drivers/cpufreq/cpufreq_performance.c b/drivers/cpufreq/cpufreq_performance.c index dafb679adc589e..399428e40e8910 100644 --- a/drivers/cpufreq/cpufreq_performance.c +++ b/drivers/cpufreq/cpufreq_performance.c @@ -22,7 +22,10 @@ static void cpufreq_gov_performance_limits(struct cpufreq_policy *policy) __cpufreq_driver_target(policy, policy->max, CPUFREQ_RELATION_H); } -static struct cpufreq_governor cpufreq_gov_performance = { +#ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE_MODULE +static +#endif +struct cpufreq_governor cpufreq_gov_performance = { .name = "performance", .owner = THIS_MODULE, .limits = cpufreq_gov_performance_limits, diff --git a/drivers/cpufreq/cpufreq_powersave.c b/drivers/cpufreq/cpufreq_powersave.c index 78a651038faf2d..5daa500fb0a9e1 100644 --- a/drivers/cpufreq/cpufreq_powersave.c +++ b/drivers/cpufreq/cpufreq_powersave.c @@ -22,7 +22,10 @@ static void cpufreq_gov_powersave_limits(struct cpufreq_policy *policy) __cpufreq_driver_target(policy, policy->min, CPUFREQ_RELATION_L); } -static struct cpufreq_governor cpufreq_gov_powersave = { +#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_POWERSAVE +static +#endif +struct cpufreq_governor cpufreq_gov_powersave = { .name = "powersave", .limits = cpufreq_gov_powersave_limits, .owner = THIS_MODULE, diff --git a/drivers/cpufreq/cpufreq_userspace.c b/drivers/cpufreq/cpufreq_userspace.c index bd897e3e134de0..765166d881bb34 100644 --- a/drivers/cpufreq/cpufreq_userspace.c +++ b/drivers/cpufreq/cpufreq_userspace.c @@ -118,7 +118,10 @@ static void cpufreq_userspace_policy_limits(struct cpufreq_policy *policy) mutex_unlock(&userspace_mutex); } -static struct cpufreq_governor cpufreq_gov_userspace = { +#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_USERSPACE +static +#endif +struct cpufreq_governor cpufreq_gov_userspace = { .name = "userspace", .init = cpufreq_userspace_policy_init, .exit = cpufreq_userspace_policy_exit, diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 4737520ec8230a..80fa656da5ab43 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -820,6 +820,25 @@ static void intel_pstate_hwp_enable(struct cpudata *cpudata) wrmsrl_on_cpu(cpudata->cpu, MSR_PM_ENABLE, 0x1); } +#define MSR_IA32_POWER_CTL_BIT_EE 19 + +/* Disable energy efficiency optimization */ +static void intel_pstate_disable_ee(int cpu) +{ + u64 power_ctl; + int ret; + + ret = rdmsrl_on_cpu(cpu, MSR_IA32_POWER_CTL, &power_ctl); + if (ret) + return; + + if (!(power_ctl & BIT(MSR_IA32_POWER_CTL_BIT_EE))) { + pr_info("Disabling energy efficiency optimization\n"); + power_ctl |= BIT(MSR_IA32_POWER_CTL_BIT_EE); + wrmsrl_on_cpu(cpu, MSR_IA32_POWER_CTL, power_ctl); + } +} + static int atom_get_min_pstate(void) { u64 value; @@ -1420,6 +1439,11 @@ static const struct x86_cpu_id intel_pstate_cpu_oob_ids[] __initconst = { {} }; +static const struct x86_cpu_id intel_pstate_cpu_ee_disable_ids[] = { + ICPU(INTEL_FAM6_KABYLAKE_DESKTOP, core_params), + {} +}; + static int intel_pstate_init_cpu(unsigned int cpunum) { struct cpudata *cpu; @@ -1435,6 +1459,12 @@ static int intel_pstate_init_cpu(unsigned int cpunum) cpu->cpu = cpunum; if (hwp_active) { + const struct x86_cpu_id *id; + + id = x86_match_cpu(intel_pstate_cpu_ee_disable_ids); + if (id) + intel_pstate_disable_ee(cpunum); + intel_pstate_hwp_enable(cpu); pid_params.sample_rate_ms = 50; pid_params.sample_rate_ns = 50 * NSEC_PER_MSEC; diff --git a/drivers/cpufreq/powernv-cpufreq.c b/drivers/cpufreq/powernv-cpufreq.c index d3ffde8066298f..a84724eabfb899 100644 --- a/drivers/cpufreq/powernv-cpufreq.c +++ b/drivers/cpufreq/powernv-cpufreq.c @@ -647,8 +647,14 @@ static int powernv_cpufreq_target_index(struct cpufreq_policy *policy, if (unlikely(rebooting) && new_index != get_nominal_index()) return 0; - if (!throttled) + if (!throttled) { + /* we don't want to be preempted while + * checking if the CPU frequency has been throttled + */ + preempt_disable(); powernv_cpufreq_throttle_check(NULL); + preempt_enable(); + } cur_msec = jiffies_to_msecs(get_jiffies_64()); diff --git a/drivers/cpuidle/cpuidle.c b/drivers/cpuidle/cpuidle.c index c73207abb5a44a..78ab946d946a7c 100644 --- a/drivers/cpuidle/cpuidle.c +++ b/drivers/cpuidle/cpuidle.c @@ -192,7 +192,7 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv, } /* Take note of the planned idle state. */ - sched_idle_set_state(target_state); + sched_idle_set_state(target_state, index); trace_cpu_idle_rcuidle(index, dev->cpu); time_start = ns_to_ktime(local_clock()); @@ -205,7 +205,7 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv, trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, dev->cpu); /* The cpu is no longer idle or about to enter idle. */ - sched_idle_set_state(NULL); + sched_idle_set_state(NULL, -1); if (broadcast) { if (WARN_ON_ONCE(!irqs_disabled())) diff --git a/drivers/cpuidle/governors/menu.c b/drivers/cpuidle/governors/menu.c index 03d38c291de62d..65bb6fd70439f3 100644 --- a/drivers/cpuidle/governors/menu.c +++ b/drivers/cpuidle/governors/menu.c @@ -178,7 +178,12 @@ static inline int performance_multiplier(unsigned long nr_iowaiters, unsigned lo /* for higher loadavg, we are more reluctant */ - mult += 2 * get_loadavg(load); + /* + * this doesn't work as intended - it is almost always 0, but can + * sometimes, depending on workload, spike very high into the hundreds + * even when the average cpu load is under 10%. + */ + /* mult += 2 * get_loadavg(); */ /* for IO wait tasks (per cpu!) we add 5x each */ mult += 10 * nr_iowaiters; diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 954a64c7757b10..c310318b34ddce 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -736,7 +736,9 @@ static int aead_set_sh_desc(struct crypto_aead *aead) /* Will read cryptlen */ append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); - aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2); + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | KEY_VLF | + FIFOLD_TYPE_MSG1OUT2 | FIFOLD_TYPE_LASTBOTH); + append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); /* Write ICV */ append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB | diff --git a/drivers/crypto/ccp/ccp-dev-v5.c b/drivers/crypto/ccp/ccp-dev-v5.c index faf3cb3ddce230..a388bf2d67f47a 100644 --- a/drivers/crypto/ccp/ccp-dev-v5.c +++ b/drivers/crypto/ccp/ccp-dev-v5.c @@ -955,7 +955,7 @@ static irqreturn_t ccp5_irq_handler(int irq, void *data) static void ccp5_config(struct ccp_device *ccp) { /* Public side */ - iowrite32(0x00001249, ccp->io_regs + CMD5_REQID_CONFIG_OFFSET); + iowrite32(0x0, ccp->io_regs + CMD5_REQID_CONFIG_OFFSET); } static void ccp5other_config(struct ccp_device *ccp) diff --git a/drivers/crypto/ccp/ccp-dev.h b/drivers/crypto/ccp/ccp-dev.h index da5f4a67808344..340aef14d616fd 100644 --- a/drivers/crypto/ccp/ccp-dev.h +++ b/drivers/crypto/ccp/ccp-dev.h @@ -238,6 +238,7 @@ struct ccp_dma_chan { struct ccp_device *ccp; spinlock_t lock; + struct list_head created; struct list_head pending; struct list_head active; struct list_head complete; diff --git a/drivers/crypto/ccp/ccp-dmaengine.c b/drivers/crypto/ccp/ccp-dmaengine.c index 6553912804f73f..e5d9278f401974 100644 --- a/drivers/crypto/ccp/ccp-dmaengine.c +++ b/drivers/crypto/ccp/ccp-dmaengine.c @@ -63,6 +63,7 @@ static void ccp_free_chan_resources(struct dma_chan *dma_chan) ccp_free_desc_resources(chan->ccp, &chan->complete); ccp_free_desc_resources(chan->ccp, &chan->active); ccp_free_desc_resources(chan->ccp, &chan->pending); + ccp_free_desc_resources(chan->ccp, &chan->created); spin_unlock_irqrestore(&chan->lock, flags); } @@ -273,6 +274,7 @@ static dma_cookie_t ccp_tx_submit(struct dma_async_tx_descriptor *tx_desc) spin_lock_irqsave(&chan->lock, flags); cookie = dma_cookie_assign(tx_desc); + list_del(&desc->entry); list_add_tail(&desc->entry, &chan->pending); spin_unlock_irqrestore(&chan->lock, flags); @@ -426,7 +428,7 @@ static struct ccp_dma_desc *ccp_create_desc(struct dma_chan *dma_chan, spin_lock_irqsave(&chan->lock, sflags); - list_add_tail(&desc->entry, &chan->pending); + list_add_tail(&desc->entry, &chan->created); spin_unlock_irqrestore(&chan->lock, sflags); @@ -610,6 +612,7 @@ static int ccp_terminate_all(struct dma_chan *dma_chan) /*TODO: Purge the complete list? */ ccp_free_desc_resources(chan->ccp, &chan->active); ccp_free_desc_resources(chan->ccp, &chan->pending); + ccp_free_desc_resources(chan->ccp, &chan->created); spin_unlock_irqrestore(&chan->lock, flags); @@ -679,6 +682,7 @@ int ccp_dmaengine_register(struct ccp_device *ccp) chan->ccp = ccp; spin_lock_init(&chan->lock); + INIT_LIST_HEAD(&chan->created); INIT_LIST_HEAD(&chan->pending); INIT_LIST_HEAD(&chan->active); INIT_LIST_HEAD(&chan->complete); diff --git a/drivers/crypto/chelsio/chcr_core.c b/drivers/crypto/chelsio/chcr_core.c index fb5f9bbfa09cd3..6aece3f25b0800 100644 --- a/drivers/crypto/chelsio/chcr_core.c +++ b/drivers/crypto/chelsio/chcr_core.c @@ -51,6 +51,7 @@ static struct cxgb4_uld_info chcr_uld_info = { int assign_chcr_device(struct chcr_dev **dev) { struct uld_ctx *u_ctx; + int ret = -ENXIO; /* * Which device to use if multiple devices are available TODO @@ -58,15 +59,14 @@ int assign_chcr_device(struct chcr_dev **dev) * must go to the same device to maintain the ordering. */ mutex_lock(&dev_mutex); /* TODO ? */ - u_ctx = list_first_entry(&uld_ctx_list, struct uld_ctx, entry); - if (!u_ctx) { - mutex_unlock(&dev_mutex); - return -ENXIO; + list_for_each_entry(u_ctx, &uld_ctx_list, entry) + if (u_ctx && u_ctx->dev) { + *dev = u_ctx->dev; + ret = 0; + break; } - - *dev = u_ctx->dev; mutex_unlock(&dev_mutex); - return 0; + return ret; } static int chcr_dev_add(struct uld_ctx *u_ctx) @@ -203,10 +203,8 @@ static int chcr_uld_state_change(void *handle, enum cxgb4_state state) static int __init chcr_crypto_init(void) { - if (cxgb4_register_uld(CXGB4_ULD_CRYPTO, &chcr_uld_info)) { + if (cxgb4_register_uld(CXGB4_ULD_CRYPTO, &chcr_uld_info)) pr_err("ULD register fail: No chcr crypto support in cxgb4"); - return -1; - } return 0; } diff --git a/drivers/crypto/qat/qat_c62x/adf_drv.c b/drivers/crypto/qat/qat_c62x/adf_drv.c index bc5cbc193aae5c..5b2d78a5b5aaaf 100644 --- a/drivers/crypto/qat/qat_c62x/adf_drv.c +++ b/drivers/crypto/qat/qat_c62x/adf_drv.c @@ -233,7 +233,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) &hw_data->accel_capabilities_mask); /* Find and map all the device's BARS */ - i = 0; + i = (hw_data->fuses & ADF_DEVICE_FUSECTL_MASK) ? 1 : 0; bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); for_each_set_bit(bar_nr, (const unsigned long *)&bar_mask, ADF_PCI_MAX_BARS * 2) { diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index e8822536530b99..33f0a6251e385c 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -69,6 +69,7 @@ #define ADF_ERRSOU5 (0x3A000 + 0xD8) #define ADF_DEVICE_FUSECTL_OFFSET 0x40 #define ADF_DEVICE_LEGFUSE_OFFSET 0x4C +#define ADF_DEVICE_FUSECTL_MASK 0x80000000 #define ADF_PCI_MAX_BARS 3 #define ADF_DEVICE_NAME_LENGTH 32 #define ADF_ETR_MAX_RINGS_PER_BANK 16 diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index 1e480f14066353..8c4fd255a601b2 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -456,7 +456,7 @@ static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle) unsigned int csr_val; int times = 30; - if (handle->pci_dev->device == ADF_C3XXX_PCI_DEVICE_ID) + if (handle->pci_dev->device != ADF_DH895XCC_PCI_DEVICE_ID) return 0; csr_val = ADF_CSR_RD(csr_addr, 0); @@ -716,7 +716,7 @@ int qat_hal_init(struct adf_accel_dev *accel_dev) (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v + LOCAL_TO_XFER_REG_OFFSET); handle->pci_dev = pci_info->pci_dev; - if (handle->pci_dev->device != ADF_C3XXX_PCI_DEVICE_ID) { + if (handle->pci_dev->device == ADF_DH895XCC_PCI_DEVICE_ID) { sram_bar = &pci_info->pci_bars[hw_data->get_sram_bar_id(hw_data)]; handle->hal_sram_addr_v = sram_bar->virt_addr; diff --git a/drivers/devfreq/devfreq.c b/drivers/devfreq/devfreq.c index bf3ea7603a58a9..712592cef1a2e7 100644 --- a/drivers/devfreq/devfreq.c +++ b/drivers/devfreq/devfreq.c @@ -593,11 +593,16 @@ struct devfreq *devfreq_add_device(struct device *dev, list_add(&devfreq->node, &devfreq_list); governor = find_devfreq_governor(devfreq->governor_name); - if (!IS_ERR(governor)) - devfreq->governor = governor; - if (devfreq->governor) - err = devfreq->governor->event_handler(devfreq, - DEVFREQ_GOV_START, NULL); + if (IS_ERR(governor)) { + dev_err(dev, "%s: Unable to find governor for the device\n", + __func__); + err = PTR_ERR(governor); + goto err_init; + } + + devfreq->governor = governor; + err = devfreq->governor->event_handler(devfreq, DEVFREQ_GOV_START, + NULL); if (err) { dev_err(dev, "%s: Unable to start governor for the device\n", __func__); diff --git a/drivers/devfreq/exynos-bus.c b/drivers/devfreq/exynos-bus.c index 29866f7e6d7e76..1b21bb60e7975a 100644 --- a/drivers/devfreq/exynos-bus.c +++ b/drivers/devfreq/exynos-bus.c @@ -498,7 +498,7 @@ static int exynos_bus_probe(struct platform_device *pdev) if (IS_ERR(bus->devfreq)) { dev_err(dev, "failed to add devfreq dev with passive governor\n"); - ret = -EPROBE_DEFER; + ret = PTR_ERR(bus->devfreq); goto err; } diff --git a/drivers/dma-buf/Makefile b/drivers/dma-buf/Makefile index 210a10bfad2b81..ee418efd2fff1b 100644 --- a/drivers/dma-buf/Makefile +++ b/drivers/dma-buf/Makefile @@ -1,3 +1,3 @@ obj-y := dma-buf.o fence.o reservation.o seqno-fence.o fence-array.o obj-$(CONFIG_SYNC_FILE) += sync_file.o -obj-$(CONFIG_SW_SYNC) += sw_sync.o sync_debug.o +obj-$(CONFIG_SW_SYNC) += sync_debug.o diff --git a/drivers/dma-buf/fence.c b/drivers/dma-buf/fence.c index 4d51f9e83fa83a..2453e076ee365a 100644 --- a/drivers/dma-buf/fence.c +++ b/drivers/dma-buf/fence.c @@ -304,8 +304,12 @@ fence_remove_callback(struct fence *fence, struct fence_cb *cb) spin_lock_irqsave(fence->lock, flags); ret = !list_empty(&cb->node); - if (ret) + if (ret) { list_del_init(&cb->node); + if (list_empty(&fence->cb_list)) + if (fence->ops->disable_signaling) + fence->ops->disable_signaling(fence); + } spin_unlock_irqrestore(fence->lock, flags); diff --git a/drivers/dma-buf/sw_sync.c b/drivers/dma-buf/sw_sync.c deleted file mode 100644 index 62e8e6dc7953bb..00000000000000 --- a/drivers/dma-buf/sw_sync.c +++ /dev/null @@ -1,375 +0,0 @@ -/* - * Sync File validation framework - * - * Copyright (C) 2012 Google, Inc. - * - * This software is licensed under the terms of the GNU General Public - * License version 2, as published by the Free Software Foundation, and - * may be copied, distributed, and modified under those terms. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - */ - -#include -#include -#include -#include -#include - -#include "sync_debug.h" - -#define CREATE_TRACE_POINTS -#include "sync_trace.h" - -/* - * SW SYNC validation framework - * - * A sync object driver that uses a 32bit counter to coordinate - * synchronization. Useful when there is no hardware primitive backing - * the synchronization. - * - * To start the framework just open: - * - * /sync/sw_sync - * - * That will create a sync timeline, all fences created under this timeline - * file descriptor will belong to the this timeline. - * - * The 'sw_sync' file can be opened many times as to create different - * timelines. - * - * Fences can be created with SW_SYNC_IOC_CREATE_FENCE ioctl with struct - * sw_sync_ioctl_create_fence as parameter. - * - * To increment the timeline counter, SW_SYNC_IOC_INC ioctl should be used - * with the increment as u32. This will update the last signaled value - * from the timeline and signal any fence that has a seqno smaller or equal - * to it. - * - * struct sw_sync_ioctl_create_fence - * @value: the seqno to initialise the fence with - * @name: the name of the new sync point - * @fence: return the fd of the new sync_file with the created fence - */ -struct sw_sync_create_fence_data { - __u32 value; - char name[32]; - __s32 fence; /* fd of new fence */ -}; - -#define SW_SYNC_IOC_MAGIC 'W' - -#define SW_SYNC_IOC_CREATE_FENCE _IOWR(SW_SYNC_IOC_MAGIC, 0,\ - struct sw_sync_create_fence_data) - -#define SW_SYNC_IOC_INC _IOW(SW_SYNC_IOC_MAGIC, 1, __u32) - -static const struct fence_ops timeline_fence_ops; - -static inline struct sync_pt *fence_to_sync_pt(struct fence *fence) -{ - if (fence->ops != &timeline_fence_ops) - return NULL; - return container_of(fence, struct sync_pt, base); -} - -/** - * sync_timeline_create() - creates a sync object - * @name: sync_timeline name - * - * Creates a new sync_timeline. Returns the sync_timeline object or NULL in - * case of error. - */ -struct sync_timeline *sync_timeline_create(const char *name) -{ - struct sync_timeline *obj; - - obj = kzalloc(sizeof(*obj), GFP_KERNEL); - if (!obj) - return NULL; - - kref_init(&obj->kref); - obj->context = fence_context_alloc(1); - strlcpy(obj->name, name, sizeof(obj->name)); - - INIT_LIST_HEAD(&obj->child_list_head); - INIT_LIST_HEAD(&obj->active_list_head); - spin_lock_init(&obj->child_list_lock); - - sync_timeline_debug_add(obj); - - return obj; -} - -static void sync_timeline_free(struct kref *kref) -{ - struct sync_timeline *obj = - container_of(kref, struct sync_timeline, kref); - - sync_timeline_debug_remove(obj); - - kfree(obj); -} - -static void sync_timeline_get(struct sync_timeline *obj) -{ - kref_get(&obj->kref); -} - -static void sync_timeline_put(struct sync_timeline *obj) -{ - kref_put(&obj->kref, sync_timeline_free); -} - -/** - * sync_timeline_signal() - signal a status change on a sync_timeline - * @obj: sync_timeline to signal - * @inc: num to increment on timeline->value - * - * A sync implementation should call this any time one of it's fences - * has signaled or has an error condition. - */ -static void sync_timeline_signal(struct sync_timeline *obj, unsigned int inc) -{ - unsigned long flags; - struct sync_pt *pt, *next; - - trace_sync_timeline(obj); - - spin_lock_irqsave(&obj->child_list_lock, flags); - - obj->value += inc; - - list_for_each_entry_safe(pt, next, &obj->active_list_head, - active_list) { - if (fence_is_signaled_locked(&pt->base)) - list_del_init(&pt->active_list); - } - - spin_unlock_irqrestore(&obj->child_list_lock, flags); -} - -/** - * sync_pt_create() - creates a sync pt - * @parent: fence's parent sync_timeline - * @size: size to allocate for this pt - * @inc: value of the fence - * - * Creates a new sync_pt as a child of @parent. @size bytes will be - * allocated allowing for implementation specific data to be kept after - * the generic sync_timeline struct. Returns the sync_pt object or - * NULL in case of error. - */ -static struct sync_pt *sync_pt_create(struct sync_timeline *obj, int size, - unsigned int value) -{ - unsigned long flags; - struct sync_pt *pt; - - if (size < sizeof(*pt)) - return NULL; - - pt = kzalloc(size, GFP_KERNEL); - if (!pt) - return NULL; - - spin_lock_irqsave(&obj->child_list_lock, flags); - sync_timeline_get(obj); - fence_init(&pt->base, &timeline_fence_ops, &obj->child_list_lock, - obj->context, value); - list_add_tail(&pt->child_list, &obj->child_list_head); - INIT_LIST_HEAD(&pt->active_list); - spin_unlock_irqrestore(&obj->child_list_lock, flags); - return pt; -} - -static const char *timeline_fence_get_driver_name(struct fence *fence) -{ - return "sw_sync"; -} - -static const char *timeline_fence_get_timeline_name(struct fence *fence) -{ - struct sync_timeline *parent = fence_parent(fence); - - return parent->name; -} - -static void timeline_fence_release(struct fence *fence) -{ - struct sync_pt *pt = fence_to_sync_pt(fence); - struct sync_timeline *parent = fence_parent(fence); - unsigned long flags; - - spin_lock_irqsave(fence->lock, flags); - list_del(&pt->child_list); - if (!list_empty(&pt->active_list)) - list_del(&pt->active_list); - spin_unlock_irqrestore(fence->lock, flags); - - sync_timeline_put(parent); - fence_free(fence); -} - -static bool timeline_fence_signaled(struct fence *fence) -{ - struct sync_timeline *parent = fence_parent(fence); - - return (fence->seqno > parent->value) ? false : true; -} - -static bool timeline_fence_enable_signaling(struct fence *fence) -{ - struct sync_pt *pt = fence_to_sync_pt(fence); - struct sync_timeline *parent = fence_parent(fence); - - if (timeline_fence_signaled(fence)) - return false; - - list_add_tail(&pt->active_list, &parent->active_list_head); - return true; -} - -static void timeline_fence_value_str(struct fence *fence, - char *str, int size) -{ - snprintf(str, size, "%d", fence->seqno); -} - -static void timeline_fence_timeline_value_str(struct fence *fence, - char *str, int size) -{ - struct sync_timeline *parent = fence_parent(fence); - - snprintf(str, size, "%d", parent->value); -} - -static const struct fence_ops timeline_fence_ops = { - .get_driver_name = timeline_fence_get_driver_name, - .get_timeline_name = timeline_fence_get_timeline_name, - .enable_signaling = timeline_fence_enable_signaling, - .signaled = timeline_fence_signaled, - .wait = fence_default_wait, - .release = timeline_fence_release, - .fence_value_str = timeline_fence_value_str, - .timeline_value_str = timeline_fence_timeline_value_str, -}; - -/* - * *WARNING* - * - * improper use of this can result in deadlocking kernel drivers from userspace. - */ - -/* opening sw_sync create a new sync obj */ -static int sw_sync_debugfs_open(struct inode *inode, struct file *file) -{ - struct sync_timeline *obj; - char task_comm[TASK_COMM_LEN]; - - get_task_comm(task_comm, current); - - obj = sync_timeline_create(task_comm); - if (!obj) - return -ENOMEM; - - file->private_data = obj; - - return 0; -} - -static int sw_sync_debugfs_release(struct inode *inode, struct file *file) -{ - struct sync_timeline *obj = file->private_data; - - smp_wmb(); - - sync_timeline_put(obj); - return 0; -} - -static long sw_sync_ioctl_create_fence(struct sync_timeline *obj, - unsigned long arg) -{ - int fd = get_unused_fd_flags(O_CLOEXEC); - int err; - struct sync_pt *pt; - struct sync_file *sync_file; - struct sw_sync_create_fence_data data; - - if (fd < 0) - return fd; - - if (copy_from_user(&data, (void __user *)arg, sizeof(data))) { - err = -EFAULT; - goto err; - } - - pt = sync_pt_create(obj, sizeof(*pt), data.value); - if (!pt) { - err = -ENOMEM; - goto err; - } - - sync_file = sync_file_create(&pt->base); - if (!sync_file) { - fence_put(&pt->base); - err = -ENOMEM; - goto err; - } - - data.fence = fd; - if (copy_to_user((void __user *)arg, &data, sizeof(data))) { - fput(sync_file->file); - err = -EFAULT; - goto err; - } - - fd_install(fd, sync_file->file); - - return 0; - -err: - put_unused_fd(fd); - return err; -} - -static long sw_sync_ioctl_inc(struct sync_timeline *obj, unsigned long arg) -{ - u32 value; - - if (copy_from_user(&value, (void __user *)arg, sizeof(value))) - return -EFAULT; - - sync_timeline_signal(obj, value); - - return 0; -} - -static long sw_sync_ioctl(struct file *file, unsigned int cmd, - unsigned long arg) -{ - struct sync_timeline *obj = file->private_data; - - switch (cmd) { - case SW_SYNC_IOC_CREATE_FENCE: - return sw_sync_ioctl_create_fence(obj, arg); - - case SW_SYNC_IOC_INC: - return sw_sync_ioctl_inc(obj, arg); - - default: - return -ENOTTY; - } -} - -const struct file_operations sw_sync_debugfs_fops = { - .open = sw_sync_debugfs_open, - .release = sw_sync_debugfs_release, - .unlocked_ioctl = sw_sync_ioctl, - .compat_ioctl = sw_sync_ioctl, -}; diff --git a/drivers/dma-buf/sync_debug.c b/drivers/dma-buf/sync_debug.c index 2dd4c3db6caa83..c07fcec147a0a8 100644 --- a/drivers/dma-buf/sync_debug.c +++ b/drivers/dma-buf/sync_debug.c @@ -15,14 +15,25 @@ */ #include -#include "sync_debug.h" - -static struct dentry *dbgfs; +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +//#include "sync.h" + +#ifdef CONFIG_DEBUG_FS static LIST_HEAD(sync_timeline_list_head); static DEFINE_SPINLOCK(sync_timeline_list_lock); -static LIST_HEAD(sync_file_list_head); -static DEFINE_SPINLOCK(sync_file_list_lock); +static LIST_HEAD(sync_fence_list_head); +static DEFINE_SPINLOCK(sync_fence_list_lock); void sync_timeline_debug_add(struct sync_timeline *obj) { @@ -42,22 +53,22 @@ void sync_timeline_debug_remove(struct sync_timeline *obj) spin_unlock_irqrestore(&sync_timeline_list_lock, flags); } -void sync_file_debug_add(struct sync_file *sync_file) +void sync_fence_debug_add(struct sync_fence *fence) { unsigned long flags; - spin_lock_irqsave(&sync_file_list_lock, flags); - list_add_tail(&sync_file->sync_file_list, &sync_file_list_head); - spin_unlock_irqrestore(&sync_file_list_lock, flags); + spin_lock_irqsave(&sync_fence_list_lock, flags); + list_add_tail(&fence->sync_fence_list, &sync_fence_list_head); + spin_unlock_irqrestore(&sync_fence_list_lock, flags); } -void sync_file_debug_remove(struct sync_file *sync_file) +void sync_fence_debug_remove(struct sync_fence *fence) { unsigned long flags; - spin_lock_irqsave(&sync_file_list_lock, flags); - list_del(&sync_file->sync_file_list); - spin_unlock_irqrestore(&sync_file_list_lock, flags); + spin_lock_irqsave(&sync_fence_list_lock, flags); + list_del(&fence->sync_fence_list); + spin_unlock_irqrestore(&sync_fence_list_lock, flags); } static const char *sync_status_str(int status) @@ -71,39 +82,39 @@ static const char *sync_status_str(int status) return "error"; } -static void sync_print_fence(struct seq_file *s, struct fence *fence, bool show) +static void sync_print_pt(struct seq_file *s, struct fence *pt, bool fence) { int status = 1; - struct sync_timeline *parent = fence_parent(fence); - if (fence_is_signaled_locked(fence)) - status = fence->status; + if (fence_is_signaled_locked(pt)) + status = pt->status; - seq_printf(s, " %s%sfence %s", - show ? parent->name : "", - show ? "_" : "", + seq_printf(s, " %s%spt %s", + fence && pt->ops->get_timeline_name ? + pt->ops->get_timeline_name(pt) : "", + fence ? "_" : "", sync_status_str(status)); if (status <= 0) { struct timespec64 ts64 = - ktime_to_timespec64(fence->timestamp); + ktime_to_timespec64(pt->timestamp); seq_printf(s, "@%lld.%09ld", (s64)ts64.tv_sec, ts64.tv_nsec); } - if (fence->ops->timeline_value_str && - fence->ops->fence_value_str) { + if ((!fence || pt->ops->timeline_value_str) && + pt->ops->fence_value_str) { char value[64]; bool success; - fence->ops->fence_value_str(fence, value, sizeof(value)); + pt->ops->fence_value_str(pt, value, sizeof(value)); success = strlen(value); - if (success) { + if (success) seq_printf(s, ": %s", value); - fence->ops->timeline_value_str(fence, value, - sizeof(value)); + if (success && fence) { + pt->ops->timeline_value_str(pt, value, sizeof(value)); if (strlen(value)) seq_printf(s, " / %s", value); @@ -118,33 +129,51 @@ static void sync_print_obj(struct seq_file *s, struct sync_timeline *obj) struct list_head *pos; unsigned long flags; - seq_printf(s, "%s: %d\n", obj->name, obj->value); + seq_printf(s, "%s %s", obj->name, obj->ops->driver_name); + + if (obj->ops->timeline_value_str) { + char value[64]; + + obj->ops->timeline_value_str(obj, value, sizeof(value)); + seq_printf(s, ": %s", value); + } + + seq_puts(s, "\n"); spin_lock_irqsave(&obj->child_list_lock, flags); list_for_each(pos, &obj->child_list_head) { struct sync_pt *pt = container_of(pos, struct sync_pt, child_list); - sync_print_fence(s, &pt->base, false); + sync_print_pt(s, &pt->base, false); } spin_unlock_irqrestore(&obj->child_list_lock, flags); } -static void sync_print_sync_file(struct seq_file *s, - struct sync_file *sync_file) +static void sync_print_fence(struct seq_file *s, struct sync_fence *fence) { + wait_queue_t *pos; + unsigned long flags; int i; - seq_printf(s, "[%p] %s: %s\n", sync_file, sync_file->name, - sync_status_str(!fence_is_signaled(sync_file->fence))); + seq_printf(s, "[%p] %s: %s\n", fence, fence->name, + sync_status_str(atomic_read(&fence->status))); + + for (i = 0; i < fence->num_fences; ++i) { + sync_print_pt(s, fence->cbs[i].sync_pt, true); + } + + spin_lock_irqsave(&fence->wq.lock, flags); + list_for_each_entry(pos, &fence->wq.task_list, task_list) { + struct sync_fence_waiter *waiter; - if (fence_is_array(sync_file->fence)) { - struct fence_array *array = to_fence_array(sync_file->fence); + if (pos->func != &sync_fence_wake_up_wq) + continue; - for (i = 0; i < array->num_fences; ++i) - sync_print_fence(s, array->fences[i], true); - } else { - sync_print_fence(s, sync_file->fence, true); + waiter = container_of(pos, struct sync_fence_waiter, work); + + seq_printf(s, "waiter %pF\n", waiter->callback); } + spin_unlock_irqrestore(&fence->wq.lock, flags); } static int sync_debugfs_show(struct seq_file *s, void *unused) @@ -167,25 +196,25 @@ static int sync_debugfs_show(struct seq_file *s, void *unused) seq_puts(s, "fences:\n--------------\n"); - spin_lock_irqsave(&sync_file_list_lock, flags); - list_for_each(pos, &sync_file_list_head) { - struct sync_file *sync_file = - container_of(pos, struct sync_file, sync_file_list); + spin_lock_irqsave(&sync_fence_list_lock, flags); + list_for_each(pos, &sync_fence_list_head) { + struct sync_fence *fence = + container_of(pos, struct sync_fence, sync_fence_list); - sync_print_sync_file(s, sync_file); + sync_print_fence(s, fence); seq_puts(s, "\n"); } - spin_unlock_irqrestore(&sync_file_list_lock, flags); + spin_unlock_irqrestore(&sync_fence_list_lock, flags); return 0; } -static int sync_info_debugfs_open(struct inode *inode, struct file *file) +static int sync_debugfs_open(struct inode *inode, struct file *file) { return single_open(file, sync_debugfs_show, inode->i_private); } -static const struct file_operations sync_info_debugfs_fops = { - .open = sync_info_debugfs_open, +static const struct file_operations sync_debugfs_fops = { + .open = sync_debugfs_open, .read = seq_read, .llseek = seq_lseek, .release = single_release, @@ -193,18 +222,7 @@ static const struct file_operations sync_info_debugfs_fops = { static __init int sync_debugfs_init(void) { - dbgfs = debugfs_create_dir("sync", NULL); - - /* - * The debugfs files won't ever get removed and thus, there is - * no need to protect it against removal races. The use of - * debugfs_create_file_unsafe() is actually safe here. - */ - debugfs_create_file_unsafe("info", 0444, dbgfs, NULL, - &sync_info_debugfs_fops); - debugfs_create_file_unsafe("sw_sync", 0644, dbgfs, NULL, - &sw_sync_debugfs_fops); - + debugfs_create_file("sync", S_IRUGO, NULL, NULL, &sync_debugfs_fops); return 0; } late_initcall(sync_debugfs_init); @@ -234,3 +252,5 @@ void sync_dump(void) } } } + +#endif diff --git a/drivers/dma-buf/sync_trace.h b/drivers/dma-buf/sync_trace.h index d13d59ff1b85fa..bcc7ab0d93e927 100644 --- a/drivers/dma-buf/sync_trace.h +++ b/drivers/dma-buf/sync_trace.h @@ -15,15 +15,65 @@ TRACE_EVENT(sync_timeline, TP_STRUCT__entry( __string(name, timeline->name) - __field(u32, value) + __array(char, value, 32) ), TP_fast_assign( __assign_str(name, timeline->name); - __entry->value = timeline->value; + if (timeline->ops->timeline_value_str) { + timeline->ops->timeline_value_str(timeline, + __entry->value, + sizeof(__entry->value)); + } else { + __entry->value[0] = '\0'; + } ), - TP_printk("name=%s value=%d", __get_str(name), __entry->value) + TP_printk("name=%s value=%s", __get_str(name), __entry->value) +); + +TRACE_EVENT(sync_wait, + TP_PROTO(struct sync_fence *fence, int begin), + + TP_ARGS(fence, begin), + + TP_STRUCT__entry( + __string(name, fence->name) + __field(s32, status) + __field(u32, begin) + ), + + TP_fast_assign( + __assign_str(name, fence->name); + __entry->status = atomic_read(&fence->status); + __entry->begin = begin; + ), + + TP_printk("%s name=%s state=%d", __entry->begin ? "begin" : "end", + __get_str(name), __entry->status) +); + +TRACE_EVENT(sync_pt, + TP_PROTO(struct fence *pt), + + TP_ARGS(pt), + + TP_STRUCT__entry( + __string(timeline, pt->ops->get_timeline_name(pt)) + __array(char, value, 32) + ), + + TP_fast_assign( + __assign_str(timeline, pt->ops->get_timeline_name(pt)); + if (pt->ops->fence_value_str) { + pt->ops->fence_value_str(pt, __entry->value, + sizeof(__entry->value)); + } else { + __entry->value[0] = '\0'; + } + ), + + TP_printk("name=%s value=%s", __get_str(timeline), __entry->value) ); #endif /* if !defined(_TRACE_SYNC_H) || defined(TRACE_HEADER_MULTI_READ) */ diff --git a/drivers/dma/cppi41.c b/drivers/dma/cppi41.c index d5ba43a87a682b..55c1782e36239c 100644 --- a/drivers/dma/cppi41.c +++ b/drivers/dma/cppi41.c @@ -153,6 +153,8 @@ struct cppi41_dd { /* context for suspend/resume */ unsigned int dma_tdfdq; + + bool is_suspended; }; #define FIST_COMPLETION_QUEUE 93 @@ -257,6 +259,10 @@ static struct cppi41_channel *desc_to_chan(struct cppi41_dd *cdd, u32 desc) BUG_ON(desc_num >= ALLOC_DECS_NUM); c = cdd->chan_busy[desc_num]; cdd->chan_busy[desc_num] = NULL; + + /* Usecount for chan_busy[], paired with push_desc_queue() */ + pm_runtime_put(cdd->ddev.dev); + return c; } @@ -447,6 +453,15 @@ static void push_desc_queue(struct cppi41_channel *c) */ __iowmb(); + /* + * DMA transfers can take at least 200ms to complete with USB mass + * storage connected. To prevent autosuspend timeouts, we must use + * pm_runtime_get/put() when chan_busy[] is modified. This will get + * cleared in desc_to_chan() or cppi41_stop_chan() depending on the + * outcome of the transfer. + */ + pm_runtime_get(cdd->ddev.dev); + desc_phys = lower_32_bits(c->desc_phys); desc_num = (desc_phys - cdd->descs_phys) / sizeof(struct cppi41_desc); WARN_ON(cdd->chan_busy[desc_num]); @@ -457,20 +472,26 @@ static void push_desc_queue(struct cppi41_channel *c) cppi_writel(reg, cdd->qmgr_mem + QMGR_QUEUE_D(c->q_num)); } -static void pending_desc(struct cppi41_channel *c) +/* + * Caller must hold cdd->lock to prevent push_desc_queue() + * getting called out of order. We have both cppi41_dma_issue_pending() + * and cppi41_runtime_resume() call this function. + */ +static void cppi41_run_queue(struct cppi41_dd *cdd) { - struct cppi41_dd *cdd = c->cdd; - unsigned long flags; + struct cppi41_channel *c, *_c; - spin_lock_irqsave(&cdd->lock, flags); - list_add_tail(&c->node, &cdd->pending); - spin_unlock_irqrestore(&cdd->lock, flags); + list_for_each_entry_safe(c, _c, &cdd->pending, node) { + push_desc_queue(c); + list_del(&c->node); + } } static void cppi41_dma_issue_pending(struct dma_chan *chan) { struct cppi41_channel *c = to_cpp41_chan(chan); struct cppi41_dd *cdd = c->cdd; + unsigned long flags; int error; error = pm_runtime_get(cdd->ddev.dev); @@ -482,10 +503,11 @@ static void cppi41_dma_issue_pending(struct dma_chan *chan) return; } - if (likely(pm_runtime_active(cdd->ddev.dev))) - push_desc_queue(c); - else - pending_desc(c); + spin_lock_irqsave(&cdd->lock, flags); + list_add_tail(&c->node, &cdd->pending); + if (!cdd->is_suspended) + cppi41_run_queue(cdd); + spin_unlock_irqrestore(&cdd->lock, flags); pm_runtime_mark_last_busy(cdd->ddev.dev); pm_runtime_put_autosuspend(cdd->ddev.dev); @@ -705,6 +727,9 @@ static int cppi41_stop_chan(struct dma_chan *chan) WARN_ON(!cdd->chan_busy[desc_num]); cdd->chan_busy[desc_num] = NULL; + /* Usecount for chan_busy[], paired with push_desc_queue() */ + pm_runtime_put(cdd->ddev.dev); + return 0; } @@ -1150,8 +1175,12 @@ static int __maybe_unused cppi41_resume(struct device *dev) static int __maybe_unused cppi41_runtime_suspend(struct device *dev) { struct cppi41_dd *cdd = dev_get_drvdata(dev); + unsigned long flags; + spin_lock_irqsave(&cdd->lock, flags); + cdd->is_suspended = true; WARN_ON(!list_empty(&cdd->pending)); + spin_unlock_irqrestore(&cdd->lock, flags); return 0; } @@ -1159,14 +1188,11 @@ static int __maybe_unused cppi41_runtime_suspend(struct device *dev) static int __maybe_unused cppi41_runtime_resume(struct device *dev) { struct cppi41_dd *cdd = dev_get_drvdata(dev); - struct cppi41_channel *c, *_c; unsigned long flags; spin_lock_irqsave(&cdd->lock, flags); - list_for_each_entry_safe(c, _c, &cdd->pending, node) { - push_desc_queue(c); - list_del(&c->node); - } + cdd->is_suspended = false; + cppi41_run_queue(cdd); spin_unlock_irqrestore(&cdd->lock, flags); return 0; diff --git a/drivers/dma/omap-dma.c b/drivers/dma/omap-dma.c index 7ca27d4b1c5432..6b16ce390dce18 100644 --- a/drivers/dma/omap-dma.c +++ b/drivers/dma/omap-dma.c @@ -1339,6 +1339,7 @@ static int omap_dma_probe(struct platform_device *pdev) struct omap_dmadev *od; struct resource *res; int rc, i, irq; + u32 lch_count; od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL); if (!od) @@ -1381,20 +1382,31 @@ static int omap_dma_probe(struct platform_device *pdev) spin_lock_init(&od->lock); spin_lock_init(&od->irq_lock); - if (!pdev->dev.of_node) { - od->dma_requests = od->plat->dma_attr->lch_count; - if (unlikely(!od->dma_requests)) - od->dma_requests = OMAP_SDMA_REQUESTS; - } else if (of_property_read_u32(pdev->dev.of_node, "dma-requests", - &od->dma_requests)) { + /* Number of DMA requests */ + od->dma_requests = OMAP_SDMA_REQUESTS; + if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node, + "dma-requests", + &od->dma_requests)) { dev_info(&pdev->dev, "Missing dma-requests property, using %u.\n", OMAP_SDMA_REQUESTS); - od->dma_requests = OMAP_SDMA_REQUESTS; } - od->lch_map = devm_kcalloc(&pdev->dev, od->dma_requests, - sizeof(*od->lch_map), GFP_KERNEL); + /* Number of available logical channels */ + if (!pdev->dev.of_node) { + lch_count = od->plat->dma_attr->lch_count; + if (unlikely(!lch_count)) + lch_count = OMAP_SDMA_CHANNELS; + } else if (of_property_read_u32(pdev->dev.of_node, "dma-channels", + &lch_count)) { + dev_info(&pdev->dev, + "Missing dma-channels property, using %u.\n", + OMAP_SDMA_CHANNELS); + lch_count = OMAP_SDMA_CHANNELS; + } + + od->lch_map = devm_kcalloc(&pdev->dev, lch_count, sizeof(*od->lch_map), + GFP_KERNEL); if (!od->lch_map) return -ENOMEM; diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c index 030fe05ed43b1e..9f3dbc8c63d27e 100644 --- a/drivers/dma/pl330.c +++ b/drivers/dma/pl330.c @@ -448,6 +448,9 @@ struct dma_pl330_chan { /* for cyclic capability */ bool cyclic; + + /* for runtime pm tracking */ + bool active; }; struct pl330_dmac { @@ -2031,6 +2034,7 @@ static void pl330_tasklet(unsigned long data) _stop(pch->thread); spin_unlock(&pch->thread->dmac->lock); power_down = true; + pch->active = false; } else { /* Make sure the PL330 Channel thread is active */ spin_lock(&pch->thread->dmac->lock); @@ -2050,6 +2054,7 @@ static void pl330_tasklet(unsigned long data) desc->status = PREP; list_move_tail(&desc->node, &pch->work_list); if (power_down) { + pch->active = true; spin_lock(&pch->thread->dmac->lock); _start(pch->thread); spin_unlock(&pch->thread->dmac->lock); @@ -2164,6 +2169,7 @@ static int pl330_terminate_all(struct dma_chan *chan) unsigned long flags; struct pl330_dmac *pl330 = pch->dmac; LIST_HEAD(list); + bool power_down = false; pm_runtime_get_sync(pl330->ddma.dev); spin_lock_irqsave(&pch->lock, flags); @@ -2174,6 +2180,8 @@ static int pl330_terminate_all(struct dma_chan *chan) pch->thread->req[0].desc = NULL; pch->thread->req[1].desc = NULL; pch->thread->req_running = -1; + power_down = pch->active; + pch->active = false; /* Mark all desc done */ list_for_each_entry(desc, &pch->submitted_list, node) { @@ -2191,6 +2199,8 @@ static int pl330_terminate_all(struct dma_chan *chan) list_splice_tail_init(&pch->completed_list, &pl330->desc_pool); spin_unlock_irqrestore(&pch->lock, flags); pm_runtime_mark_last_busy(pl330->ddma.dev); + if (power_down) + pm_runtime_put_autosuspend(pl330->ddma.dev); pm_runtime_put_autosuspend(pl330->ddma.dev); return 0; @@ -2350,6 +2360,7 @@ static void pl330_issue_pending(struct dma_chan *chan) * updated on work_list emptiness status. */ WARN_ON(list_empty(&pch->submitted_list)); + pch->active = true; pm_runtime_get_sync(pch->dmac->ddma.dev); } list_splice_tail_init(&pch->submitted_list, &pch->work_list); diff --git a/drivers/dma/sh/rcar-dmac.c b/drivers/dma/sh/rcar-dmac.c index 2e441d0ccd79a3..4c357d47546594 100644 --- a/drivers/dma/sh/rcar-dmac.c +++ b/drivers/dma/sh/rcar-dmac.c @@ -986,6 +986,7 @@ static void rcar_dmac_free_chan_resources(struct dma_chan *chan) { struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); struct rcar_dmac *dmac = to_rcar_dmac(chan->device); + struct rcar_dmac_chan_map *map = &rchan->map; struct rcar_dmac_desc_page *page, *_page; struct rcar_dmac_desc *desc; LIST_HEAD(list); @@ -1019,6 +1020,13 @@ static void rcar_dmac_free_chan_resources(struct dma_chan *chan) free_page((unsigned long)page); } + /* Remove slave mapping if present. */ + if (map->slave.xfer_size) { + dma_unmap_resource(chan->device->dev, map->addr, + map->slave.xfer_size, map->dir, 0); + map->slave.xfer_size = 0; + } + pm_runtime_put(chan->device->dev); } diff --git a/drivers/extcon/extcon.c b/drivers/extcon/extcon.c index 78298460d1686a..7c1e3a7b14e0c1 100644 --- a/drivers/extcon/extcon.c +++ b/drivers/extcon/extcon.c @@ -453,7 +453,7 @@ int extcon_sync(struct extcon_dev *edev, unsigned int id) dev_err(&edev->dev, "out of memory in extcon_set_state\n"); kobject_uevent(&edev->dev.kobj, KOBJ_CHANGE); - return 0; + return -ENOMEM; } length = name_show(&edev->dev, NULL, prop_buf); diff --git a/drivers/firmware/efi/efi.c b/drivers/firmware/efi/efi.c index 1ac199cd75e7bd..a4944e22f294b3 100644 --- a/drivers/firmware/efi/efi.c +++ b/drivers/firmware/efi/efi.c @@ -259,8 +259,10 @@ static __init int efivar_ssdt_load(void) } data = kmalloc(size, GFP_KERNEL); - if (!data) + if (!data) { + ret = -ENOMEM; goto free_entry; + } ret = efivar_entry_get(entry, NULL, &size, data); if (ret) { diff --git a/drivers/firmware/efi/fake_mem.c b/drivers/firmware/efi/fake_mem.c index 520a40e5e0e431..6c7d60c239b5b4 100644 --- a/drivers/firmware/efi/fake_mem.c +++ b/drivers/firmware/efi/fake_mem.c @@ -71,8 +71,7 @@ void __init efi_fake_memmap(void) } /* allocate memory for new EFI memmap */ - new_memmap_phy = memblock_alloc(efi.memmap.desc_size * new_nr_map, - PAGE_SIZE); + new_memmap_phy = efi_memmap_alloc(new_nr_map); if (!new_memmap_phy) return; diff --git a/drivers/firmware/efi/libstub/efistub.h b/drivers/firmware/efi/libstub/efistub.h index ee49cd23ee636d..fac67992bede58 100644 --- a/drivers/firmware/efi/libstub/efistub.h +++ b/drivers/firmware/efi/libstub/efistub.h @@ -30,14 +30,6 @@ efi_status_t efi_file_close(void *handle); unsigned long get_dram_base(efi_system_table_t *sys_table_arg); -efi_status_t update_fdt(efi_system_table_t *sys_table, void *orig_fdt, - unsigned long orig_fdt_size, - void *fdt, int new_fdt_size, char *cmdline_ptr, - u64 initrd_addr, u64 initrd_size, - efi_memory_desc_t *memory_map, - unsigned long map_size, unsigned long desc_size, - u32 desc_ver); - efi_status_t allocate_new_fdt_and_exit_boot(efi_system_table_t *sys_table, void *handle, unsigned long *new_fdt_addr, diff --git a/drivers/firmware/efi/libstub/fdt.c b/drivers/firmware/efi/libstub/fdt.c index a6a93116a8f053..260c4b4b492ec3 100644 --- a/drivers/firmware/efi/libstub/fdt.c +++ b/drivers/firmware/efi/libstub/fdt.c @@ -16,13 +16,10 @@ #include "efistub.h" -efi_status_t update_fdt(efi_system_table_t *sys_table, void *orig_fdt, - unsigned long orig_fdt_size, - void *fdt, int new_fdt_size, char *cmdline_ptr, - u64 initrd_addr, u64 initrd_size, - efi_memory_desc_t *memory_map, - unsigned long map_size, unsigned long desc_size, - u32 desc_ver) +static efi_status_t update_fdt(efi_system_table_t *sys_table, void *orig_fdt, + unsigned long orig_fdt_size, + void *fdt, int new_fdt_size, char *cmdline_ptr, + u64 initrd_addr, u64 initrd_size) { int node, num_rsv; int status; @@ -101,25 +98,23 @@ efi_status_t update_fdt(efi_system_table_t *sys_table, void *orig_fdt, if (status) goto fdt_set_fail; - fdt_val64 = cpu_to_fdt64((u64)(unsigned long)memory_map); + fdt_val64 = U64_MAX; /* placeholder */ status = fdt_setprop(fdt, node, "linux,uefi-mmap-start", &fdt_val64, sizeof(fdt_val64)); if (status) goto fdt_set_fail; - fdt_val32 = cpu_to_fdt32(map_size); + fdt_val32 = U32_MAX; /* placeholder */ status = fdt_setprop(fdt, node, "linux,uefi-mmap-size", &fdt_val32, sizeof(fdt_val32)); if (status) goto fdt_set_fail; - fdt_val32 = cpu_to_fdt32(desc_size); status = fdt_setprop(fdt, node, "linux,uefi-mmap-desc-size", &fdt_val32, sizeof(fdt_val32)); if (status) goto fdt_set_fail; - fdt_val32 = cpu_to_fdt32(desc_ver); status = fdt_setprop(fdt, node, "linux,uefi-mmap-desc-ver", &fdt_val32, sizeof(fdt_val32)); if (status) @@ -148,6 +143,43 @@ efi_status_t update_fdt(efi_system_table_t *sys_table, void *orig_fdt, return EFI_LOAD_ERROR; } +static efi_status_t update_fdt_memmap(void *fdt, struct efi_boot_memmap *map) +{ + int node = fdt_path_offset(fdt, "/chosen"); + u64 fdt_val64; + u32 fdt_val32; + int err; + + if (node < 0) + return EFI_LOAD_ERROR; + + fdt_val64 = cpu_to_fdt64((unsigned long)*map->map); + err = fdt_setprop_inplace(fdt, node, "linux,uefi-mmap-start", + &fdt_val64, sizeof(fdt_val64)); + if (err) + return EFI_LOAD_ERROR; + + fdt_val32 = cpu_to_fdt32(*map->map_size); + err = fdt_setprop_inplace(fdt, node, "linux,uefi-mmap-size", + &fdt_val32, sizeof(fdt_val32)); + if (err) + return EFI_LOAD_ERROR; + + fdt_val32 = cpu_to_fdt32(*map->desc_size); + err = fdt_setprop_inplace(fdt, node, "linux,uefi-mmap-desc-size", + &fdt_val32, sizeof(fdt_val32)); + if (err) + return EFI_LOAD_ERROR; + + fdt_val32 = cpu_to_fdt32(*map->desc_ver); + err = fdt_setprop_inplace(fdt, node, "linux,uefi-mmap-desc-ver", + &fdt_val32, sizeof(fdt_val32)); + if (err) + return EFI_LOAD_ERROR; + + return EFI_SUCCESS; +} + #ifndef EFI_FDT_ALIGN #define EFI_FDT_ALIGN EFI_PAGE_SIZE #endif @@ -155,6 +187,7 @@ efi_status_t update_fdt(efi_system_table_t *sys_table, void *orig_fdt, struct exit_boot_struct { efi_memory_desc_t *runtime_map; int *runtime_entry_count; + void *new_fdt_addr; }; static efi_status_t exit_boot_func(efi_system_table_t *sys_table_arg, @@ -170,7 +203,7 @@ static efi_status_t exit_boot_func(efi_system_table_t *sys_table_arg, efi_get_virtmap(*map->map, *map->map_size, *map->desc_size, p->runtime_map, p->runtime_entry_count); - return EFI_SUCCESS; + return update_fdt_memmap(p->new_fdt_addr, map); } /* @@ -243,20 +276,10 @@ efi_status_t allocate_new_fdt_and_exit_boot(efi_system_table_t *sys_table, goto fail; } - /* - * Now that we have done our final memory allocation (and free) - * we can get the memory map key needed for - * exit_boot_services(). - */ - status = efi_get_memory_map(sys_table, &map); - if (status != EFI_SUCCESS) - goto fail_free_new_fdt; - status = update_fdt(sys_table, (void *)fdt_addr, fdt_size, (void *)*new_fdt_addr, new_fdt_size, - cmdline_ptr, initrd_addr, initrd_size, - memory_map, map_size, desc_size, desc_ver); + cmdline_ptr, initrd_addr, initrd_size); /* Succeeding the first time is the expected case. */ if (status == EFI_SUCCESS) @@ -266,22 +289,19 @@ efi_status_t allocate_new_fdt_and_exit_boot(efi_system_table_t *sys_table, /* * We need to allocate more space for the new * device tree, so free existing buffer that is - * too small. Also free memory map, as we will need - * to get new one that reflects the free/alloc we do - * on the device tree buffer. + * too small. */ efi_free(sys_table, new_fdt_size, *new_fdt_addr); - sys_table->boottime->free_pool(memory_map); new_fdt_size += EFI_PAGE_SIZE; } else { pr_efi_err(sys_table, "Unable to construct new device tree.\n"); - goto fail_free_mmap; + goto fail_free_new_fdt; } } - sys_table->boottime->free_pool(memory_map); priv.runtime_map = runtime_map; priv.runtime_entry_count = &runtime_entry_count; + priv.new_fdt_addr = (void *)*new_fdt_addr; status = efi_exit_boot_services(sys_table, handle, &map, &priv, exit_boot_func); @@ -319,9 +339,6 @@ efi_status_t allocate_new_fdt_and_exit_boot(efi_system_table_t *sys_table, pr_efi_err(sys_table, "Exit boot services failed.\n"); -fail_free_mmap: - sys_table->boottime->free_pool(memory_map); - fail_free_new_fdt: efi_free(sys_table, new_fdt_size, *new_fdt_addr); diff --git a/drivers/firmware/efi/memmap.c b/drivers/firmware/efi/memmap.c index f03ddecd232b54..78686443cb378a 100644 --- a/drivers/firmware/efi/memmap.c +++ b/drivers/firmware/efi/memmap.c @@ -9,6 +9,44 @@ #include #include #include +#include +#include + +static phys_addr_t __init __efi_memmap_alloc_early(unsigned long size) +{ + return memblock_alloc(size, 0); +} + +static phys_addr_t __init __efi_memmap_alloc_late(unsigned long size) +{ + unsigned int order = get_order(size); + struct page *p = alloc_pages(GFP_KERNEL, order); + + if (!p) + return 0; + + return PFN_PHYS(page_to_pfn(p)); +} + +/** + * efi_memmap_alloc - Allocate memory for the EFI memory map + * @num_entries: Number of entries in the allocated map. + * + * Depending on whether mm_init() has already been invoked or not, + * either memblock or "normal" page allocation is used. + * + * Returns the physical address of the allocated memory map on + * success, zero on failure. + */ +phys_addr_t __init efi_memmap_alloc(unsigned int num_entries) +{ + unsigned long size = num_entries * efi.memmap.desc_size; + + if (slab_is_available()) + return __efi_memmap_alloc_late(size); + + return __efi_memmap_alloc_early(size); +} /** * __efi_memmap_init - Common code for mapping the EFI memory map diff --git a/drivers/gpio/gpio-stmpe.c b/drivers/gpio/gpio-stmpe.c index 5b0042776ec708..adba614b396567 100644 --- a/drivers/gpio/gpio-stmpe.c +++ b/drivers/gpio/gpio-stmpe.c @@ -413,7 +413,7 @@ static irqreturn_t stmpe_gpio_irq(int irq, void *dev) stmpe->partnum != STMPE1801) { stmpe_reg_write(stmpe, statmsbreg + i, status[i]); stmpe_reg_write(stmpe, - stmpe->regs[STMPE_IDX_GPEDR_LSB + i], + stmpe->regs[STMPE_IDX_GPEDR_MSB] + i, status[i]); } } diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 868128a676bae8..92159313361b1a 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -986,7 +986,8 @@ static int gpio_chrdev_open(struct inode *inode, struct file *filp) return -ENODEV; get_device(&gdev->dev); filp->private_data = gdev; - return 0; + + return nonseekable_open(inode, filp); } /** @@ -1011,7 +1012,7 @@ static const struct file_operations gpio_fileops = { .release = gpio_chrdev_release, .open = gpio_chrdev_open, .owner = THIS_MODULE, - .llseek = noop_llseek, + .llseek = no_llseek, .unlocked_ioctl = gpio_ioctl, #ifdef CONFIG_COMPAT .compat_ioctl = gpio_ioctl_compat, @@ -1316,12 +1317,12 @@ void gpiochip_remove(struct gpio_chip *chip) /* FIXME: should the legacy sysfs handling be moved to gpio_device? */ gpiochip_sysfs_unregister(gdev); + gpiochip_free_hogs(chip); /* Numb the device, cancelling all outstanding operations */ gdev->chip = NULL; gpiochip_irqchip_remove(chip); acpi_gpiochip_remove(chip); gpiochip_remove_pin_ranges(chip); - gpiochip_free_hogs(chip); of_gpiochip_remove(chip); /* * We accept no more calls into the driver from this point, so diff --git a/drivers/gpu/Makefile b/drivers/gpu/Makefile index e9ed439a5b6532..66386b42a1f452 100644 --- a/drivers/gpu/Makefile +++ b/drivers/gpu/Makefile @@ -2,5 +2,5 @@ # taken to initialize them in the correct order. Link order is the only way # to ensure this currently. obj-$(CONFIG_TEGRA_HOST1X) += host1x/ -obj-y += drm/ vga/ +obj-y += drm/ vga/ arm/ obj-$(CONFIG_IMX_IPUV3_CORE) += ipu-v3/ diff --git a/drivers/gpu/arm/Kconfig b/drivers/gpu/arm/Kconfig new file mode 100644 index 00000000000000..255cc81c7d231d --- /dev/null +++ b/drivers/gpu/arm/Kconfig @@ -0,0 +1 @@ +source "drivers/gpu/arm/utgard/Kconfig" diff --git a/drivers/gpu/arm/Makefile b/drivers/gpu/arm/Makefile new file mode 100644 index 00000000000000..e4dcf28c56f45a --- /dev/null +++ b/drivers/gpu/arm/Makefile @@ -0,0 +1 @@ +obj-y += utgard/ diff --git a/drivers/gpu/arm/utgard/Kbuild b/drivers/gpu/arm/utgard/Kbuild new file mode 100644 index 00000000000000..bf247bcd3e79c8 --- /dev/null +++ b/drivers/gpu/arm/utgard/Kbuild @@ -0,0 +1,207 @@ +# +# Copyright (C) 2010-2011 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the GNU General Public License version 2 +# as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained from Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# + +# This file is called by the Linux build system. + +# set up defaults if not defined by the user +TIMESTAMP ?= default +OS_MEMORY_KERNEL_BUFFER_SIZE_IN_MB ?= 16 +USING_GPU_UTILIZATION ?= 0 +PROFILING_SKIP_PP_JOBS ?= 0 +PROFILING_SKIP_PP_AND_GP_JOBS ?= 0 +MALI_PP_SCHEDULER_FORCE_NO_JOB_OVERLAP ?= 0 +MALI_PP_SCHEDULER_KEEP_SUB_JOB_STARTS_ALIGNED ?= 0 +MALI_PP_SCHEDULER_FORCE_NO_JOB_OVERLAP_BETWEEN_APPS ?= 0 +MALI_UPPER_HALF_SCHEDULING ?= 1 +MALI_ENABLE_CPU_CYCLES ?= 0 + +# For customer releases the Linux Device Drivers will be provided as ARM proprietary and GPL releases: +# The in-tree driver will only use the GPL releases. +ccflags-y += -I$(src)/linux/license/gpl + +ifeq ($(USING_GPU_UTILIZATION), 1) + ifeq ($(USING_DVFS), 1) + $(error USING_GPU_UTILIZATION conflict with USING_DVFS you can read the Integration Guide to choose which one do you need) + endif +endif + +mali-y += \ + linux/mali_osk_atomics.o \ + linux/mali_osk_irq.o \ + linux/mali_osk_wq.o \ + linux/mali_osk_locks.o \ + linux/mali_osk_wait_queue.o \ + linux/mali_osk_low_level_mem.o \ + linux/mali_osk_math.o \ + linux/mali_osk_memory.o \ + linux/mali_osk_misc.o \ + linux/mali_osk_mali.o \ + linux/mali_osk_notification.o \ + linux/mali_osk_time.o \ + linux/mali_osk_timers.o \ + linux/mali_osk_bitmap.o + +mali-y += linux/mali_memory.o linux/mali_memory_os_alloc.o +mali-y += linux/mali_memory_external.o +mali-y += linux/mali_memory_block_alloc.o +mali-y += linux/mali_memory_swap_alloc.o + +mali-y += \ + linux/mali_memory_manager.o \ + linux/mali_memory_virtual.o \ + linux/mali_memory_util.o \ + linux/mali_memory_cow.o \ + linux/mali_memory_defer_bind.o + +mali-y += \ + linux/mali_ukk_mem.o \ + linux/mali_ukk_gp.o \ + linux/mali_ukk_pp.o \ + linux/mali_ukk_core.o \ + linux/mali_ukk_soft_job.o \ + linux/mali_ukk_timeline.o + +# Source files which always are included in a build +mali-y += \ + common/mali_kernel_core.o \ + linux/mali_kernel_linux.o \ + common/mali_session.o \ + linux/mali_device_pause_resume.o \ + common/mali_kernel_vsync.o \ + linux/mali_ukk_vsync.o \ + linux/mali_kernel_sysfs.o \ + common/mali_mmu.o \ + common/mali_mmu_page_directory.o \ + common/mali_mem_validation.o \ + common/mali_hw_core.o \ + common/mali_gp.o \ + common/mali_pp.o \ + common/mali_pp_job.o \ + common/mali_gp_job.o \ + common/mali_soft_job.o \ + common/mali_scheduler.o \ + common/mali_executor.o \ + common/mali_group.o \ + common/mali_dlbu.o \ + common/mali_broadcast.o \ + common/mali_pm.o \ + common/mali_pmu.o \ + common/mali_user_settings_db.o \ + common/mali_kernel_utilization.o \ + common/mali_control_timer.o \ + common/mali_l2_cache.o \ + common/mali_timeline.o \ + common/mali_timeline_fence_wait.o \ + common/mali_timeline_sync_fence.o \ + common/mali_spinlock_reentrant.o \ + common/mali_pm_domain.o \ + linux/mali_osk_pm.o \ + linux/mali_pmu_power_up_down.o \ + __malidrv_build_info.o + +EXTRA_DEFINES += -DMALI_FAKE_PLATFORM_DEVICE=1 +mali-y += platform/hikey/mali_hikey.o + +mali-$(CONFIG_MALI400_PROFILING) += linux/mali_ukk_profiling.o +mali-$(CONFIG_MALI400_PROFILING) += linux/mali_osk_profiling.o + +mali-$(CONFIG_MALI400_INTERNAL_PROFILING) += linux/mali_profiling_internal.o timestamp-$(TIMESTAMP)/mali_timestamp.o +ccflags-$(CONFIG_MALI400_INTERNAL_PROFILING) += -I$(src)/timestamp-$(TIMESTAMP) + +mali-$(CONFIG_DMA_SHARED_BUFFER) += linux/mali_memory_dma_buf.o +mali-$(CONFIG_SYNC) += linux/mali_sync.o +ccflags-$(CONFIG_SYNC) += -Idrivers/staging/android + +mali-$(CONFIG_MALI400_UMP) += linux/mali_memory_ump.o + +mali-$(CONFIG_MALI_DVFS) += common/mali_dvfs_policy.o + +# Tell the Linux build system from which .o file to create the kernel module +obj-$(CONFIG_MALI400) := mali.o + +ccflags-y += $(EXTRA_DEFINES) + +# Set up our defines, which will be passed to gcc +ccflags-y += -DMALI_PP_SCHEDULER_FORCE_NO_JOB_OVERLAP=$(MALI_PP_SCHEDULER_FORCE_NO_JOB_OVERLAP) +ccflags-y += -DMALI_PP_SCHEDULER_KEEP_SUB_JOB_STARTS_ALIGNED=$(MALI_PP_SCHEDULER_KEEP_SUB_JOB_STARTS_ALIGNED) +ccflags-y += -DMALI_PP_SCHEDULER_FORCE_NO_JOB_OVERLAP_BETWEEN_APPS=$(MALI_PP_SCHEDULER_FORCE_NO_JOB_OVERLAP_BETWEEN_APPS) +ccflags-y += -DMALI_STATE_TRACKING=1 +ccflags-y += -DMALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_MB=$(OS_MEMORY_KERNEL_BUFFER_SIZE_IN_MB) +ccflags-y += -DUSING_GPU_UTILIZATION=$(USING_GPU_UTILIZATION) +ccflags-y += -DMALI_ENABLE_CPU_CYCLES=$(MALI_ENABLE_CPU_CYCLES) + +ifeq ($(MALI_UPPER_HALF_SCHEDULING),1) + ccflags-y += -DMALI_UPPER_HALF_SCHEDULING +endif + +ccflags-$(CONFIG_MALI400_UMP) += -I$(src)/../../ump/include/ump +ccflags-$(CONFIG_MALI400_DEBUG) += -DDEBUG + +# Use our defines when compiling +ccflags-y += -I$(src) -I$(src)/include -I$(src)/common -I$(src)/linux -I$(src)/platform -Wno-date-time + +# Get subversion revision number, fall back to only ${MALI_RELEASE_NAME} if no svn info is available +MALI_RELEASE_NAME=$(shell cat $(src)/.version 2> /dev/null) + +SVN_INFO = (cd $(src); svn info 2>/dev/null) + +ifneq ($(shell $(SVN_INFO) 2>/dev/null),) +# SVN detected +SVN_REV := $(shell $(SVN_INFO) | grep '^Revision: '| sed -e 's/^Revision: //' 2>/dev/null) +DRIVER_REV := $(MALI_RELEASE_NAME)-r$(SVN_REV) +CHANGE_DATE := $(shell $(SVN_INFO) | grep '^Last Changed Date: ' | cut -d: -f2- | cut -b2-) +CHANGED_REVISION := $(shell $(SVN_INFO) | grep '^Last Changed Rev: ' | cut -d: -f2- | cut -b2-) +REPO_URL := $(shell $(SVN_INFO) | grep '^URL: ' | cut -d: -f2- | cut -b2-) + +else # SVN +GIT_REV := $(shell cd $(src); git describe --always 2>/dev/null) +ifneq ($(GIT_REV),) +# Git detected +DRIVER_REV := $(MALI_RELEASE_NAME)-$(GIT_REV) +CHANGE_DATE := $(shell cd $(src); git log -1 --format="%ci") +CHANGED_REVISION := $(GIT_REV) +REPO_URL := $(shell cd $(src); git describe --all --always 2>/dev/null) + +else # Git +# No Git or SVN detected +DRIVER_REV := $(MALI_RELEASE_NAME) +CHANGE_DATE := $(MALI_RELEASE_NAME) +CHANGED_REVISION := $(MALI_RELEASE_NAME) +endif +endif + +ccflags-y += -DSVN_REV_STRING=\"$(DRIVER_REV)\" + +VERSION_STRINGS := +VERSION_STRINGS += API_VERSION=$(shell cd $(src); grep "\#define _MALI_API_VERSION" $(FILES_PREFIX)include/linux/mali/mali_utgard_uk_types.h | cut -d' ' -f 3 ) +VERSION_STRINGS += REPO_URL=$(REPO_URL) +VERSION_STRINGS += REVISION=$(DRIVER_REV) +VERSION_STRINGS += CHANGED_REVISION=$(CHANGED_REVISION) +VERSION_STRINGS += CHANGE_DATE=$(CHANGE_DATE) +VERSION_STRINGS += BUILD_DATE=$(shell date) +ifdef CONFIG_MALI400_DEBUG +VERSION_STRINGS += BUILD=debug +else +VERSION_STRINGS += BUILD=release +endif +VERSION_STRINGS += TARGET_PLATFORM=$(TARGET_PLATFORM) +VERSION_STRINGS += MALI_PLATFORM=$(MALI_PLATFORM) +VERSION_STRINGS += KDIR=$(KDIR) +VERSION_STRINGS += OS_MEMORY_KERNEL_BUFFER_SIZE_IN_MB=$(OS_MEMORY_KERNEL_BUFFER_SIZE_IN_MB) +VERSION_STRINGS += USING_UMP=$(CONFIG_MALI400_UMP) +VERSION_STRINGS += USING_PROFILING=$(CONFIG_MALI400_PROFILING) +VERSION_STRINGS += USING_INTERNAL_PROFILING=$(CONFIG_MALI400_INTERNAL_PROFILING) +VERSION_STRINGS += USING_GPU_UTILIZATION=$(USING_GPU_UTILIZATION) +VERSION_STRINGS += USING_DVFS=$(CONFIG_MALI_DVFS) +VERSION_STRINGS += MALI_UPPER_HALF_SCHEDULING=$(MALI_UPPER_HALF_SCHEDULING) + +# Create file with Mali driver configuration +$(src)/__malidrv_build_info.c: + @echo 'const char *__malidrv_build_info(void) { return "malidrv: $(VERSION_STRINGS)";}' > $(src)/__malidrv_build_info.c diff --git a/drivers/gpu/arm/utgard/Kconfig b/drivers/gpu/arm/utgard/Kconfig new file mode 100644 index 00000000000000..e174e572d18200 --- /dev/null +++ b/drivers/gpu/arm/utgard/Kconfig @@ -0,0 +1,118 @@ +config MALI400 + tristate "Mali-300/400/450 support" + depends on ARM || ARM64 + select DMA_SHARED_BUFFER + ---help--- + This enables support for the ARM Mali-300, Mali-400, and Mali-450 + GPUs. + + To compile this driver as a module, choose M here: the module will be + called mali. + +config MALI450 + bool "Enable Mali-450 support" + depends on MALI400 + ---help--- + This enables support for Mali-450 specific features. + +config MALI470 + bool "Enable Mali-470 support" + depends on MALI400 + ---help--- + This enables support for Mali-470 specific features. + +config MALI400_DEBUG + bool "Enable debug in Mali driver" + depends on MALI400 + ---help--- + This enabled extra debug checks and messages in the Mali driver. + +config MALI400_PROFILING + bool "Enable Mali profiling" + depends on MALI400 + select TRACEPOINTS + default y + ---help--- + This enables gator profiling of Mali GPU events. + +config MALI400_INTERNAL_PROFILING + bool "Enable internal Mali profiling API" + depends on MALI400_PROFILING + default n + ---help--- + This enables the internal legacy Mali profiling API. + +config MALI400_UMP + bool "Enable UMP support" + depends on MALI400 + ---help--- + This enables support for the UMP memory sharing API in the Mali driver. + +config MALI_DVFS + bool "Enable Mali dynamically frequency change" + depends on MALI400 + default y + ---help--- + This enables support for dynamic change frequency of Mali with the goal of lowering power consumption. + +config MALI_DMA_BUF_MAP_ON_ATTACH + bool "Map dma-buf attachments on attach" + depends on MALI400 && DMA_SHARED_BUFFER + default y + ---help--- + This makes the Mali driver map dma-buf attachments after doing + attach. If this is not set the dma-buf attachments will be mapped for + every time the GPU need to access the buffer. + + Mapping for each access can cause lower performance. + +config MALI_SHARED_INTERRUPTS + bool "Support for shared interrupts" + depends on MALI400 + default n + ---help--- + Adds functionality required to properly support shared interrupts. Without this support, + the device driver will fail during insmod if it detects shared interrupts. This also + works when the GPU is not using shared interrupts, but might have a slight performance + impact. + +config MALI_PMU_PARALLEL_POWER_UP + bool "Power up Mali PMU domains in parallel" + depends on MALI400 + default n + ---help--- + This makes the Mali driver power up all PMU power domains in parallel, instead of + powering up domains one by one, with a slight delay in between. Powering on all power + domains at the same time may cause peak currents higher than what some systems can handle. + These systems must not enable this option. + +config MALI_DT + bool "Using device tree to initialize module" + depends on MALI400 && OF + default n + ---help--- + This enable the Mali driver to choose the device tree path to get platform resoures + and disable the old config method. Mali driver could run on the platform which the + device tree is enabled in kernel and corresponding hardware description is implemented + properly in device DTS file. + +config MALI_PLAT_SPECIFIC_DT + bool "Platform specific Device Tree is being used" + depends on MALI_DT + default n + ---help--- + This is a pragmatic approach for some platforms which make + use of a device tree entry that does not strictly comply to + what the standard Utgard driver expects to find, but have + their platform data implemented the old way. Such platforms + should be converted to using the Device Tree so this + configuration option can be removed. + +config MALI_QUIET + bool "Make Mali driver very quiet" + depends on MALI400 && !MALI400_DEBUG + default n + ---help--- + This forces the Mali driver to never print any messages. + + If unsure, say N. diff --git a/drivers/gpu/arm/utgard/Makefile b/drivers/gpu/arm/utgard/Makefile new file mode 100644 index 00000000000000..44c7bb83981a00 --- /dev/null +++ b/drivers/gpu/arm/utgard/Makefile @@ -0,0 +1,190 @@ +# +# Copyright (C) 2010-2015 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the GNU General Public License version 2 +# as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained from Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# + +USE_UMPV2=0 +USING_PROFILING ?= 1 +USING_INTERNAL_PROFILING ?= 0 +USING_DVFS ?= 1 +MALI_HEATMAPS_ENABLED ?= 0 +MALI_DMA_BUF_MAP_ON_ATTACH ?= 1 +MALI_PMU_PARALLEL_POWER_UP ?= 0 +USING_DT ?= 0 +MALI_MEM_SWAP_TRACKING ?= 0 + +# The Makefile sets up "arch" based on the CONFIG, creates the version info +# string and the __malidrv_build_info.c file, and then call the Linux build +# system to actually build the driver. After that point the Kbuild file takes +# over. + +# set up defaults if not defined by the user +ARCH ?= arm + +OSKOS=linux +FILES_PREFIX= + +check_cc2 = \ + $(shell if $(1) -S -o /dev/null -xc /dev/null > /dev/null 2>&1; \ + then \ + echo "$(2)"; \ + else \ + echo "$(3)"; \ + fi ;) + +# This conditional makefile exports the global definition ARM_INTERNAL_BUILD. Customer releases will not include arm_internal.mak +-include ../../../arm_internal.mak + +# Give warning of old config parameters are used +ifneq ($(CONFIG),) +$(warning "You have specified the CONFIG variable which is no longer in used. Use TARGET_PLATFORM instead.") +endif + +ifneq ($(CPU),) +$(warning "You have specified the CPU variable which is no longer in used. Use TARGET_PLATFORM instead.") +endif + +# Include the mapping between TARGET_PLATFORM and KDIR + MALI_PLATFORM +-include MALI_CONFIGURATION +export KDIR ?= $(KDIR-$(TARGET_PLATFORM)) +export MALI_PLATFORM ?= $(MALI_PLATFORM-$(TARGET_PLATFORM)) + +ifneq ($(TARGET_PLATFORM),) +ifeq ($(MALI_PLATFORM),) +$(error "Invalid TARGET_PLATFORM: $(TARGET_PLATFORM)") +endif +endif + +# validate lookup result +ifeq ($(KDIR),) +$(error No KDIR found for platform $(TARGET_PLATFORM)) +endif + +ifeq ($(USING_GPU_UTILIZATION), 1) + ifeq ($(USING_DVFS), 1) + $(error USING_GPU_UTILIZATION conflict with USING_DVFS you can read the Integration Guide to choose which one do you need) + endif +endif + +ifeq ($(USING_UMP),1) +export CONFIG_MALI400_UMP=y +export EXTRA_DEFINES += -DCONFIG_MALI400_UMP=1 +ifeq ($(USE_UMPV2),1) +UMP_SYMVERS_FILE ?= ../umpv2/Module.symvers +else +UMP_SYMVERS_FILE ?= ../ump/Module.symvers +endif +KBUILD_EXTRA_SYMBOLS = $(realpath $(UMP_SYMVERS_FILE)) +$(warning $(KBUILD_EXTRA_SYMBOLS)) +endif + +# Define host system directory +KDIR-$(shell uname -m):=/lib/modules/$(shell uname -r)/build + +include $(KDIR)/.config + +ifeq ($(ARCH), arm) +# when compiling for ARM we're cross compiling +export CROSS_COMPILE ?= $(call check_cc2, arm-linux-gnueabi-gcc, arm-linux-gnueabi-, arm-none-linux-gnueabi-) +endif + +# report detected/selected settings +ifdef ARM_INTERNAL_BUILD +$(warning TARGET_PLATFORM $(TARGET_PLATFORM)) +$(warning KDIR $(KDIR)) +$(warning MALI_PLATFORM $(MALI_PLATFORM)) +endif + +# Set up build config +export CONFIG_MALI400=m +export CONFIG_MALI450=y +export CONFIG_MALI470=y + +export EXTRA_DEFINES += -DCONFIG_MALI400=1 +export EXTRA_DEFINES += -DCONFIG_MALI450=1 +export EXTRA_DEFINES += -DCONFIG_MALI470=1 + +ifneq ($(MALI_PLATFORM),) +export EXTRA_DEFINES += -DMALI_FAKE_PLATFORM_DEVICE=1 +export MALI_PLATFORM_FILES = $(wildcard platform/$(MALI_PLATFORM)/*.c) +endif + +ifeq ($(USING_PROFILING),1) +ifeq ($(CONFIG_TRACEPOINTS),) +$(warning CONFIG_TRACEPOINTS required for profiling) +else +export CONFIG_MALI400_PROFILING=y +export EXTRA_DEFINES += -DCONFIG_MALI400_PROFILING=1 +ifeq ($(USING_INTERNAL_PROFILING),1) +export CONFIG_MALI400_INTERNAL_PROFILING=y +export EXTRA_DEFINES += -DCONFIG_MALI400_INTERNAL_PROFILING=1 +endif +ifeq ($(MALI_HEATMAPS_ENABLED),1) +export MALI_HEATMAPS_ENABLED=y +export EXTRA_DEFINES += -DCONFIG_MALI400_HEATMAPS_ENABLED +endif +endif +endif + +ifeq ($(MALI_DMA_BUF_MAP_ON_ATTACH),1) +export CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH=y +export EXTRA_DEFINES += -DCONFIG_MALI_DMA_BUF_MAP_ON_ATTACH +endif + +ifeq ($(MALI_SHARED_INTERRUPTS),1) +export CONFIG_MALI_SHARED_INTERRUPTS=y +export EXTRA_DEFINES += -DCONFIG_MALI_SHARED_INTERRUPTS +endif + +ifeq ($(USING_DVFS),1) +export CONFIG_MALI_DVFS=y +export EXTRA_DEFINES += -DCONFIG_MALI_DVFS +endif + +ifeq ($(MALI_PMU_PARALLEL_POWER_UP),1) +export CONFIG_MALI_PMU_PARALLEL_POWER_UP=y +export EXTRA_DEFINES += -DCONFIG_MALI_PMU_PARALLEL_POWER_UP +endif + +ifdef CONFIG_OF +ifeq ($(USING_DT),1) +export CONFIG_MALI_DT=y +export EXTRA_DEFINES += -DCONFIG_MALI_DT +endif +endif + +ifneq ($(BUILD),release) +# Debug +export CONFIG_MALI400_DEBUG=y +else +# Release +ifeq ($(MALI_QUIET),1) +export CONFIG_MALI_QUIET=y +export EXTRA_DEFINES += -DCONFIG_MALI_QUIET +endif +endif + +ifeq ($(MALI_SKIP_JOBS),1) +EXTRA_DEFINES += -DPROFILING_SKIP_PP_JOBS=1 -DPROFILING_SKIP_GP_JOBS=1 +endif + +ifeq ($(MALI_MEM_SWAP_TRACKING),1) +EXTRA_DEFINES += -DMALI_MEM_SWAP_TRACKING=1 +endif + +all: $(UMP_SYMVERS_FILE) + $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) modules + @rm $(FILES_PREFIX)__malidrv_build_info.c $(FILES_PREFIX)__malidrv_build_info.o + +clean: + $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean + +kernelrelease: + $(MAKE) ARCH=$(ARCH) -C $(KDIR) kernelrelease + +export CONFIG KBUILD_EXTRA_SYMBOLS diff --git a/drivers/gpu/arm/utgard/common/mali_broadcast.c b/drivers/gpu/arm/utgard/common/mali_broadcast.c new file mode 100644 index 00000000000000..136db61ace4a4f --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_broadcast.c @@ -0,0 +1,142 @@ +/* + * Copyright (C) 2012-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_broadcast.h" +#include "mali_kernel_common.h" +#include "mali_osk.h" + +#define MALI_BROADCAST_REGISTER_SIZE 0x1000 +#define MALI_BROADCAST_REG_BROADCAST_MASK 0x0 +#define MALI_BROADCAST_REG_INTERRUPT_MASK 0x4 + +struct mali_bcast_unit { + struct mali_hw_core hw_core; + u32 current_mask; +}; + +struct mali_bcast_unit *mali_bcast_unit_create(const _mali_osk_resource_t *resource) +{ + struct mali_bcast_unit *bcast_unit = NULL; + + MALI_DEBUG_ASSERT_POINTER(resource); + MALI_DEBUG_PRINT(2, ("Broadcast: Creating Mali Broadcast unit: %s\n", + resource->description)); + + bcast_unit = _mali_osk_malloc(sizeof(struct mali_bcast_unit)); + if (NULL == bcast_unit) { + MALI_PRINT_ERROR(("Broadcast: Failed to allocate memory for Broadcast unit\n")); + return NULL; + } + + if (_MALI_OSK_ERR_OK == mali_hw_core_create(&bcast_unit->hw_core, + resource, MALI_BROADCAST_REGISTER_SIZE)) { + bcast_unit->current_mask = 0; + mali_bcast_reset(bcast_unit); + + return bcast_unit; + } else { + MALI_PRINT_ERROR(("Broadcast: Failed map broadcast unit\n")); + } + + _mali_osk_free(bcast_unit); + + return NULL; +} + +void mali_bcast_unit_delete(struct mali_bcast_unit *bcast_unit) +{ + MALI_DEBUG_ASSERT_POINTER(bcast_unit); + mali_hw_core_delete(&bcast_unit->hw_core); + _mali_osk_free(bcast_unit); +} + +/* Call this function to add the @group's id into bcast mask + * Note: redundant calling this function with same @group + * doesn't make any difference as calling it once + */ +void mali_bcast_add_group(struct mali_bcast_unit *bcast_unit, + struct mali_group *group) +{ + u32 bcast_id; + u32 broadcast_mask; + + MALI_DEBUG_ASSERT_POINTER(bcast_unit); + MALI_DEBUG_ASSERT_POINTER(group); + + bcast_id = mali_pp_core_get_bcast_id(mali_group_get_pp_core(group)); + + broadcast_mask = bcast_unit->current_mask; + + broadcast_mask |= (bcast_id); /* add PP core to broadcast */ + broadcast_mask |= (bcast_id << 16); /* add MMU to broadcast */ + + /* store mask so we can restore on reset */ + bcast_unit->current_mask = broadcast_mask; +} + +/* Call this function to remove @group's id from bcast mask + * Note: redundant calling this function with same @group + * doesn't make any difference as calling it once + */ +void mali_bcast_remove_group(struct mali_bcast_unit *bcast_unit, + struct mali_group *group) +{ + u32 bcast_id; + u32 broadcast_mask; + + MALI_DEBUG_ASSERT_POINTER(bcast_unit); + MALI_DEBUG_ASSERT_POINTER(group); + + bcast_id = mali_pp_core_get_bcast_id(mali_group_get_pp_core(group)); + + broadcast_mask = bcast_unit->current_mask; + + broadcast_mask &= ~((bcast_id << 16) | bcast_id); + + /* store mask so we can restore on reset */ + bcast_unit->current_mask = broadcast_mask; +} + +void mali_bcast_reset(struct mali_bcast_unit *bcast_unit) +{ + MALI_DEBUG_ASSERT_POINTER(bcast_unit); + + MALI_DEBUG_PRINT(4, + ("Broadcast: setting mask 0x%08X + 0x%08X (reset)\n", + bcast_unit->current_mask, + bcast_unit->current_mask & 0xFF)); + + /* set broadcast mask */ + mali_hw_core_register_write(&bcast_unit->hw_core, + MALI_BROADCAST_REG_BROADCAST_MASK, + bcast_unit->current_mask); + + /* set IRQ override mask */ + mali_hw_core_register_write(&bcast_unit->hw_core, + MALI_BROADCAST_REG_INTERRUPT_MASK, + bcast_unit->current_mask & 0xFF); +} + +void mali_bcast_disable(struct mali_bcast_unit *bcast_unit) +{ + MALI_DEBUG_ASSERT_POINTER(bcast_unit); + + MALI_DEBUG_PRINT(4, ("Broadcast: setting mask 0x0 + 0x0 (disable)\n")); + + /* set broadcast mask */ + mali_hw_core_register_write(&bcast_unit->hw_core, + MALI_BROADCAST_REG_BROADCAST_MASK, + 0x0); + + /* set IRQ override mask */ + mali_hw_core_register_write(&bcast_unit->hw_core, + MALI_BROADCAST_REG_INTERRUPT_MASK, + 0x0); +} diff --git a/drivers/gpu/arm/utgard/common/mali_broadcast.h b/drivers/gpu/arm/utgard/common/mali_broadcast.h new file mode 100644 index 00000000000000..efce44142ee9fa --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_broadcast.h @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2012-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_BROADCAST_H__ +#define __MALI_BROADCAST_H__ + +/* + * Interface for the broadcast unit on Mali-450. + * + * - Represents up to 8 × (MMU + PP) pairs. + * - Supports dynamically changing which (MMU + PP) pairs receive the broadcast by + * setting a mask. + */ + +#include "mali_hw_core.h" +#include "mali_group.h" + +struct mali_bcast_unit; + +struct mali_bcast_unit *mali_bcast_unit_create(const _mali_osk_resource_t *resource); +void mali_bcast_unit_delete(struct mali_bcast_unit *bcast_unit); + +/* Add a group to the list of (MMU + PP) pairs broadcasts go out to. */ +void mali_bcast_add_group(struct mali_bcast_unit *bcast_unit, struct mali_group *group); + +/* Remove a group to the list of (MMU + PP) pairs broadcasts go out to. */ +void mali_bcast_remove_group(struct mali_bcast_unit *bcast_unit, struct mali_group *group); + +/* Re-set cached mask. This needs to be called after having been suspended. */ +void mali_bcast_reset(struct mali_bcast_unit *bcast_unit); + +/** + * Disable broadcast unit + * + * mali_bcast_enable must be called to re-enable the unit. Cores may not be + * added or removed when the unit is disabled. + */ +void mali_bcast_disable(struct mali_bcast_unit *bcast_unit); + +/** + * Re-enable broadcast unit + * + * This resets the masks to include the cores present when mali_bcast_disable was called. + */ +MALI_STATIC_INLINE void mali_bcast_enable(struct mali_bcast_unit *bcast_unit) +{ + mali_bcast_reset(bcast_unit); +} + +#endif /* __MALI_BROADCAST_H__ */ diff --git a/drivers/gpu/arm/utgard/common/mali_control_timer.c b/drivers/gpu/arm/utgard/common/mali_control_timer.c new file mode 100644 index 00000000000000..d0dd95ac1b39d6 --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_control_timer.c @@ -0,0 +1,128 @@ +/* + * Copyright (C) 2010-2012, 2014-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_kernel_utilization.h" +#include "mali_osk.h" +#include "mali_osk_mali.h" +#include "mali_kernel_common.h" +#include "mali_session.h" +#include "mali_dvfs_policy.h" +#include "mali_control_timer.h" + +static u64 period_start_time = 0; + +static _mali_osk_timer_t *mali_control_timer = NULL; +static mali_bool timer_running = MALI_FALSE; + +static u32 mali_control_timeout = 1000; + +void mali_control_timer_add(u32 timeout) +{ + _mali_osk_timer_add(mali_control_timer, _mali_osk_time_mstoticks(timeout)); +} + +static void mali_control_timer_callback(void *arg) +{ + if (mali_utilization_enabled()) { + struct mali_gpu_utilization_data *util_data = NULL; + u64 time_period = 0; + mali_bool need_add_timer = MALI_TRUE; + + /* Calculate gpu utilization */ + util_data = mali_utilization_calculate(&period_start_time, &time_period, &need_add_timer); + + if (util_data) { +#if defined(CONFIG_MALI_DVFS) + mali_dvfs_policy_realize(util_data, time_period); +#else + mali_utilization_platform_realize(util_data); +#endif + + if (MALI_TRUE == need_add_timer) { + mali_control_timer_add(mali_control_timeout); + } + } + } +} + +/* Init a timer (for now it is used for GPU utilization and dvfs) */ +_mali_osk_errcode_t mali_control_timer_init(void) +{ + _mali_osk_device_data data; + + if (_MALI_OSK_ERR_OK == _mali_osk_device_data_get(&data)) { + /* Use device specific settings (if defined) */ + if (0 != data.control_interval) { + mali_control_timeout = data.control_interval; + MALI_DEBUG_PRINT(2, ("Mali GPU Timer: %u\n", mali_control_timeout)); + } + } + + mali_control_timer = _mali_osk_timer_init(); + if (NULL == mali_control_timer) { + return _MALI_OSK_ERR_FAULT; + } + _mali_osk_timer_setcallback(mali_control_timer, mali_control_timer_callback, NULL); + + return _MALI_OSK_ERR_OK; +} + +void mali_control_timer_term(void) +{ + if (NULL != mali_control_timer) { + _mali_osk_timer_del(mali_control_timer); + timer_running = MALI_FALSE; + _mali_osk_timer_term(mali_control_timer); + mali_control_timer = NULL; + } +} + +mali_bool mali_control_timer_resume(u64 time_now) +{ + mali_utilization_data_assert_locked(); + + if (timer_running != MALI_TRUE) { + timer_running = MALI_TRUE; + + period_start_time = time_now; + + mali_utilization_reset(); + + return MALI_TRUE; + } + + return MALI_FALSE; +} + +void mali_control_timer_pause(void) +{ + mali_utilization_data_assert_locked(); + if (timer_running == MALI_TRUE) { + timer_running = MALI_FALSE; + } +} + +void mali_control_timer_suspend(mali_bool suspend) +{ + mali_utilization_data_lock(); + + if (timer_running == MALI_TRUE) { + timer_running = MALI_FALSE; + + mali_utilization_data_unlock(); + + if (suspend == MALI_TRUE) { + _mali_osk_timer_del(mali_control_timer); + mali_utilization_reset(); + } + } else { + mali_utilization_data_unlock(); + } +} diff --git a/drivers/gpu/arm/utgard/common/mali_control_timer.h b/drivers/gpu/arm/utgard/common/mali_control_timer.h new file mode 100644 index 00000000000000..4f919ecfd70a55 --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_control_timer.h @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2010-2012, 2014-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_CONTROL_TIMER_H__ +#define __MALI_CONTROL_TIMER_H__ + +#include "mali_osk.h" + +_mali_osk_errcode_t mali_control_timer_init(void); + +void mali_control_timer_term(void); + +mali_bool mali_control_timer_resume(u64 time_now); + +void mali_control_timer_suspend(mali_bool suspend); +void mali_control_timer_pause(void); + +void mali_control_timer_add(u32 timeout); + +#endif /* __MALI_CONTROL_TIMER_H__ */ + diff --git a/drivers/gpu/arm/utgard/common/mali_dlbu.c b/drivers/gpu/arm/utgard/common/mali_dlbu.c new file mode 100644 index 00000000000000..efe1ab3ab5cc32 --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_dlbu.c @@ -0,0 +1,213 @@ +/* + * Copyright (C) 2012-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_dlbu.h" +#include "mali_memory.h" +#include "mali_pp.h" +#include "mali_group.h" +#include "mali_osk.h" +#include "mali_hw_core.h" + +/** + * Size of DLBU registers in bytes + */ +#define MALI_DLBU_SIZE 0x400 + +mali_dma_addr mali_dlbu_phys_addr = 0; +static mali_io_address mali_dlbu_cpu_addr = NULL; + +/** + * DLBU register numbers + * Used in the register read/write routines. + * See the hardware documentation for more information about each register + */ +typedef enum mali_dlbu_register { + MALI_DLBU_REGISTER_MASTER_TLLIST_PHYS_ADDR = 0x0000, /**< Master tile list physical base address; + 31:12 Physical address to the page used for the DLBU + 0 DLBU enable - set this bit to 1 enables the AXI bus + between PPs and L2s, setting to 0 disables the router and + no further transactions are sent to DLBU */ + MALI_DLBU_REGISTER_MASTER_TLLIST_VADDR = 0x0004, /**< Master tile list virtual base address; + 31:12 Virtual address to the page used for the DLBU */ + MALI_DLBU_REGISTER_TLLIST_VBASEADDR = 0x0008, /**< Tile list virtual base address; + 31:12 Virtual address to the tile list. This address is used when + calculating the call address sent to PP.*/ + MALI_DLBU_REGISTER_FB_DIM = 0x000C, /**< Framebuffer dimension; + 23:16 Number of tiles in Y direction-1 + 7:0 Number of tiles in X direction-1 */ + MALI_DLBU_REGISTER_TLLIST_CONF = 0x0010, /**< Tile list configuration; + 29:28 select the size of each allocated block: 0=128 bytes, 1=256, 2=512, 3=1024 + 21:16 2^n number of tiles to be binned to one tile list in Y direction + 5:0 2^n number of tiles to be binned to one tile list in X direction */ + MALI_DLBU_REGISTER_START_TILE_POS = 0x0014, /**< Start tile positions; + 31:24 start position in Y direction for group 1 + 23:16 start position in X direction for group 1 + 15:8 start position in Y direction for group 0 + 7:0 start position in X direction for group 0 */ + MALI_DLBU_REGISTER_PP_ENABLE_MASK = 0x0018, /**< PP enable mask; + 7 enable PP7 for load balancing + 6 enable PP6 for load balancing + 5 enable PP5 for load balancing + 4 enable PP4 for load balancing + 3 enable PP3 for load balancing + 2 enable PP2 for load balancing + 1 enable PP1 for load balancing + 0 enable PP0 for load balancing */ +} mali_dlbu_register; + +typedef enum { + PP0ENABLE = 0, + PP1ENABLE, + PP2ENABLE, + PP3ENABLE, + PP4ENABLE, + PP5ENABLE, + PP6ENABLE, + PP7ENABLE +} mali_dlbu_pp_enable; + +struct mali_dlbu_core { + struct mali_hw_core hw_core; /**< Common for all HW cores */ + u32 pp_cores_mask; /**< This is a mask for the PP cores whose operation will be controlled by LBU + see MALI_DLBU_REGISTER_PP_ENABLE_MASK register */ +}; + +_mali_osk_errcode_t mali_dlbu_initialize(void) +{ + MALI_DEBUG_PRINT(2, ("Mali DLBU: Initializing\n")); + + if (_MALI_OSK_ERR_OK == + mali_mmu_get_table_page(&mali_dlbu_phys_addr, + &mali_dlbu_cpu_addr)) { + return _MALI_OSK_ERR_OK; + } + + return _MALI_OSK_ERR_FAULT; +} + +void mali_dlbu_terminate(void) +{ + MALI_DEBUG_PRINT(3, ("Mali DLBU: terminating\n")); + + if (0 != mali_dlbu_phys_addr && 0 != mali_dlbu_cpu_addr) { + mali_mmu_release_table_page(mali_dlbu_phys_addr, + mali_dlbu_cpu_addr); + mali_dlbu_phys_addr = 0; + mali_dlbu_cpu_addr = 0; + } +} + +struct mali_dlbu_core *mali_dlbu_create(const _mali_osk_resource_t *resource) +{ + struct mali_dlbu_core *core = NULL; + + MALI_DEBUG_PRINT(2, ("Mali DLBU: Creating Mali dynamic load balancing unit: %s\n", resource->description)); + + core = _mali_osk_malloc(sizeof(struct mali_dlbu_core)); + if (NULL != core) { + if (_MALI_OSK_ERR_OK == mali_hw_core_create(&core->hw_core, resource, MALI_DLBU_SIZE)) { + core->pp_cores_mask = 0; + if (_MALI_OSK_ERR_OK == mali_dlbu_reset(core)) { + return core; + } + MALI_PRINT_ERROR(("Failed to reset DLBU %s\n", core->hw_core.description)); + mali_hw_core_delete(&core->hw_core); + } + + _mali_osk_free(core); + } else { + MALI_PRINT_ERROR(("Mali DLBU: Failed to allocate memory for DLBU core\n")); + } + + return NULL; +} + +void mali_dlbu_delete(struct mali_dlbu_core *dlbu) +{ + MALI_DEBUG_ASSERT_POINTER(dlbu); + mali_hw_core_delete(&dlbu->hw_core); + _mali_osk_free(dlbu); +} + +_mali_osk_errcode_t mali_dlbu_reset(struct mali_dlbu_core *dlbu) +{ + u32 dlbu_registers[7]; + _mali_osk_errcode_t err = _MALI_OSK_ERR_FAULT; + MALI_DEBUG_ASSERT_POINTER(dlbu); + + MALI_DEBUG_PRINT(4, ("Mali DLBU: mali_dlbu_reset: %s\n", dlbu->hw_core.description)); + + dlbu_registers[0] = mali_dlbu_phys_addr | 1; /* bit 0 enables the whole core */ + dlbu_registers[1] = MALI_DLBU_VIRT_ADDR; + dlbu_registers[2] = 0; + dlbu_registers[3] = 0; + dlbu_registers[4] = 0; + dlbu_registers[5] = 0; + dlbu_registers[6] = dlbu->pp_cores_mask; + + /* write reset values to core registers */ + mali_hw_core_register_write_array_relaxed(&dlbu->hw_core, MALI_DLBU_REGISTER_MASTER_TLLIST_PHYS_ADDR, dlbu_registers, 7); + + err = _MALI_OSK_ERR_OK; + + return err; +} + +void mali_dlbu_update_mask(struct mali_dlbu_core *dlbu) +{ + MALI_DEBUG_ASSERT_POINTER(dlbu); + + mali_hw_core_register_write(&dlbu->hw_core, MALI_DLBU_REGISTER_PP_ENABLE_MASK, dlbu->pp_cores_mask); +} + +void mali_dlbu_add_group(struct mali_dlbu_core *dlbu, struct mali_group *group) +{ + struct mali_pp_core *pp_core; + u32 bcast_id; + + MALI_DEBUG_ASSERT_POINTER(dlbu); + MALI_DEBUG_ASSERT_POINTER(group); + + pp_core = mali_group_get_pp_core(group); + bcast_id = mali_pp_core_get_bcast_id(pp_core); + + dlbu->pp_cores_mask |= bcast_id; + MALI_DEBUG_PRINT(3, ("Mali DLBU: Adding core[%d] New mask= 0x%02x\n", bcast_id , dlbu->pp_cores_mask)); +} + +/* Remove a group from the DLBU */ +void mali_dlbu_remove_group(struct mali_dlbu_core *dlbu, struct mali_group *group) +{ + struct mali_pp_core *pp_core; + u32 bcast_id; + + MALI_DEBUG_ASSERT_POINTER(dlbu); + MALI_DEBUG_ASSERT_POINTER(group); + + pp_core = mali_group_get_pp_core(group); + bcast_id = mali_pp_core_get_bcast_id(pp_core); + + dlbu->pp_cores_mask &= ~bcast_id; + MALI_DEBUG_PRINT(3, ("Mali DLBU: Removing core[%d] New mask= 0x%02x\n", bcast_id, dlbu->pp_cores_mask)); +} + +/* Configure the DLBU for \a job. This needs to be done before the job is started on the groups in the DLBU. */ +void mali_dlbu_config_job(struct mali_dlbu_core *dlbu, struct mali_pp_job *job) +{ + u32 *registers; + MALI_DEBUG_ASSERT(job); + registers = mali_pp_job_get_dlbu_registers(job); + MALI_DEBUG_PRINT(4, ("Mali DLBU: Starting job\n")); + + /* Writing 4 registers: + * DLBU registers except the first two (written once at DLBU initialisation / reset) and the PP_ENABLE_MASK register */ + mali_hw_core_register_write_array_relaxed(&dlbu->hw_core, MALI_DLBU_REGISTER_TLLIST_VBASEADDR, registers, 4); + +} diff --git a/drivers/gpu/arm/utgard/common/mali_dlbu.h b/drivers/gpu/arm/utgard/common/mali_dlbu.h new file mode 100644 index 00000000000000..6b068884bd499f --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_dlbu.h @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2012-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_DLBU_H__ +#define __MALI_DLBU_H__ + +#define MALI_DLBU_VIRT_ADDR 0xFFF00000 /* master tile virtual address fixed at this value and mapped into every session */ + +#include "mali_osk.h" + +struct mali_pp_job; +struct mali_group; +struct mali_dlbu_core; + +extern mali_dma_addr mali_dlbu_phys_addr; + +_mali_osk_errcode_t mali_dlbu_initialize(void); +void mali_dlbu_terminate(void); + +struct mali_dlbu_core *mali_dlbu_create(const _mali_osk_resource_t *resource); +void mali_dlbu_delete(struct mali_dlbu_core *dlbu); + +_mali_osk_errcode_t mali_dlbu_reset(struct mali_dlbu_core *dlbu); + +void mali_dlbu_add_group(struct mali_dlbu_core *dlbu, struct mali_group *group); +void mali_dlbu_remove_group(struct mali_dlbu_core *dlbu, struct mali_group *group); + +/** @brief Called to update HW after DLBU state changed + * + * This function must be called after \a mali_dlbu_add_group or \a + * mali_dlbu_remove_group to write the updated mask to hardware, unless the + * same is accomplished by calling \a mali_dlbu_reset. + */ +void mali_dlbu_update_mask(struct mali_dlbu_core *dlbu); + +void mali_dlbu_config_job(struct mali_dlbu_core *dlbu, struct mali_pp_job *job); + +#endif /* __MALI_DLBU_H__ */ diff --git a/drivers/gpu/arm/utgard/common/mali_dvfs_policy.c b/drivers/gpu/arm/utgard/common/mali_dvfs_policy.c new file mode 100644 index 00000000000000..12ba069ec13bdc --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_dvfs_policy.c @@ -0,0 +1,308 @@ +/* + * Copyright (C) 2010-2012, 2014-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include +#include "mali_kernel_common.h" +#include "mali_scheduler.h" +#include "mali_dvfs_policy.h" +#include "mali_osk_mali.h" +#include "mali_osk_profiling.h" + +#define CLOCK_TUNING_TIME_DEBUG 0 + +#define MAX_PERFORMANCE_VALUE 256 +#define MALI_PERCENTAGE_TO_UTILIZATION_FRACTION(percent) ((int) ((percent)*(MAX_PERFORMANCE_VALUE)/100.0 + 0.5)) + +/** The max fps the same as display vsync default 60, can set by module insert parameter */ +int mali_max_system_fps = 60; +/** A lower limit on their desired FPS default 58, can set by module insert parameter */ +int mali_desired_fps = 58; + +static int mali_fps_step1 = 0; +static int mali_fps_step2 = 0; + +static int clock_step = -1; +static int cur_clk_step = -1; +static struct mali_gpu_clock *gpu_clk = NULL; + +/*Function prototype */ +static int (*mali_gpu_set_freq)(int) = NULL; +static int (*mali_gpu_get_freq)(void) = NULL; + +static mali_bool mali_dvfs_enabled = MALI_FALSE; + +#define NUMBER_OF_NANOSECONDS_PER_SECOND 1000000000ULL +static u32 calculate_window_render_fps(u64 time_period) +{ + u32 max_window_number; + u64 tmp; + u64 max = time_period; + u32 leading_zeroes; + u32 shift_val; + u32 time_period_shift; + u32 max_window_number_shift; + u32 ret_val; + + max_window_number = mali_session_max_window_num(); + + /* To avoid float division, extend the dividend to ns unit */ + tmp = (u64)max_window_number * NUMBER_OF_NANOSECONDS_PER_SECOND; + if (tmp > time_period) { + max = tmp; + } + + /* + * We may have 64-bit values, a dividend or a divisor or both + * To avoid dependencies to a 64-bit divider, we shift down the two values + * equally first. + */ + leading_zeroes = _mali_osk_clz((u32)(max >> 32)); + shift_val = 32 - leading_zeroes; + + time_period_shift = (u32)(time_period >> shift_val); + max_window_number_shift = (u32)(tmp >> shift_val); + + ret_val = max_window_number_shift / time_period_shift; + + return ret_val; +} + +static bool mali_pickup_closest_avail_clock(int target_clock_mhz, mali_bool pick_clock_up) +{ + int i = 0; + bool clock_changed = false; + + /* Round up the closest available frequency step for target_clock_hz */ + for (i = 0; i < gpu_clk->num_of_steps; i++) { + /* Find the first item > target_clock_hz */ + if (((int)(gpu_clk->item[i].clock) - target_clock_mhz) > 0) { + break; + } + } + + /* If the target clock greater than the maximum clock just pick the maximum one*/ + if (i == gpu_clk->num_of_steps) { + i = gpu_clk->num_of_steps - 1; + } else { + if ((!pick_clock_up) && (i > 0)) { + i = i - 1; + } + } + + clock_step = i; + if (cur_clk_step != clock_step) { + clock_changed = true; + } + + return clock_changed; +} + +void mali_dvfs_policy_realize(struct mali_gpu_utilization_data *data, u64 time_period) +{ + int under_perform_boundary_value = 0; + int over_perform_boundary_value = 0; + int current_fps = 0; + int current_gpu_util = 0; + bool clock_changed = false; +#if CLOCK_TUNING_TIME_DEBUG + struct timeval start; + struct timeval stop; + unsigned int elapse_time; + do_gettimeofday(&start); +#endif + u32 window_render_fps; + + if (NULL == gpu_clk) { + MALI_DEBUG_PRINT(2, ("Enable DVFS but patform doesn't Support freq change. \n")); + return; + } + + window_render_fps = calculate_window_render_fps(time_period); + + current_fps = window_render_fps; + current_gpu_util = data->utilization_gpu; + + /* Get the specific under_perform_boundary_value and over_perform_boundary_value */ + if ((mali_desired_fps <= current_fps) && (current_fps < mali_max_system_fps)) { + under_perform_boundary_value = MALI_PERCENTAGE_TO_UTILIZATION_FRACTION(90); + over_perform_boundary_value = MALI_PERCENTAGE_TO_UTILIZATION_FRACTION(70); + } else if ((mali_fps_step1 <= current_fps) && (current_fps < mali_desired_fps)) { + under_perform_boundary_value = MALI_PERCENTAGE_TO_UTILIZATION_FRACTION(55); + over_perform_boundary_value = MALI_PERCENTAGE_TO_UTILIZATION_FRACTION(35); + } else if ((mali_fps_step2 <= current_fps) && (current_fps < mali_fps_step1)) { + under_perform_boundary_value = MALI_PERCENTAGE_TO_UTILIZATION_FRACTION(70); + over_perform_boundary_value = MALI_PERCENTAGE_TO_UTILIZATION_FRACTION(50); + } else { + under_perform_boundary_value = MALI_PERCENTAGE_TO_UTILIZATION_FRACTION(55); + over_perform_boundary_value = MALI_PERCENTAGE_TO_UTILIZATION_FRACTION(35); + } + + MALI_DEBUG_PRINT(5, ("Using ARM power policy: gpu util = %d \n", current_gpu_util)); + MALI_DEBUG_PRINT(5, ("Using ARM power policy: under_perform = %d, over_perform = %d \n", under_perform_boundary_value, over_perform_boundary_value)); + MALI_DEBUG_PRINT(5, ("Using ARM power policy: render fps = %d, pressure render fps = %d \n", current_fps, window_render_fps)); + + /* Get current clock value */ + cur_clk_step = mali_gpu_get_freq(); + + /* Consider offscreen */ + if (0 == current_fps) { + /* GP or PP under perform, need to give full power */ + if (current_gpu_util > over_perform_boundary_value) { + if (cur_clk_step != gpu_clk->num_of_steps - 1) { + clock_changed = true; + clock_step = gpu_clk->num_of_steps - 1; + } + } + + /* If GPU is idle, use lowest power */ + if (0 == current_gpu_util) { + if (cur_clk_step != 0) { + clock_changed = true; + clock_step = 0; + } + } + + goto real_setting; + } + + /* 2. Calculate target clock if the GPU clock can be tuned */ + if (-1 != cur_clk_step) { + int target_clk_mhz = -1; + mali_bool pick_clock_up = MALI_TRUE; + + if (current_gpu_util > under_perform_boundary_value) { + /* when under perform, need to consider the fps part */ + target_clk_mhz = gpu_clk->item[cur_clk_step].clock * current_gpu_util * mali_desired_fps / under_perform_boundary_value / current_fps; + pick_clock_up = MALI_TRUE; + } else if (current_gpu_util < over_perform_boundary_value) { + /* when over perform, did't need to consider fps, system didn't want to reach desired fps */ + target_clk_mhz = gpu_clk->item[cur_clk_step].clock * current_gpu_util / under_perform_boundary_value; + pick_clock_up = MALI_FALSE; + } + + if (-1 != target_clk_mhz) { + clock_changed = mali_pickup_closest_avail_clock(target_clk_mhz, pick_clock_up); + } + } + +real_setting: + if (clock_changed) { + mali_gpu_set_freq(clock_step); + + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | + MALI_PROFILING_EVENT_CHANNEL_GPU | + MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE, + gpu_clk->item[clock_step].clock, + gpu_clk->item[clock_step].vol / 1000, + 0, 0, 0); + } + +#if CLOCK_TUNING_TIME_DEBUG + do_gettimeofday(&stop); + + elapse_time = timeval_to_ns(&stop) - timeval_to_ns(&start); + MALI_DEBUG_PRINT(2, ("Using ARM power policy: eclapse time = %d\n", elapse_time)); +#endif +} + +_mali_osk_errcode_t mali_dvfs_policy_init(void) +{ + _mali_osk_device_data data; + _mali_osk_errcode_t err = _MALI_OSK_ERR_OK; + + if (_MALI_OSK_ERR_OK == _mali_osk_device_data_get(&data)) { + if ((NULL != data.get_clock_info) && (NULL != data.set_freq) && (NULL != data.get_freq)) { + MALI_DEBUG_PRINT(2, ("Mali DVFS init: using arm dvfs policy \n")); + + + mali_fps_step1 = mali_max_system_fps / 3; + mali_fps_step2 = mali_max_system_fps / 5; + + data.get_clock_info(&gpu_clk); + + if (gpu_clk != NULL) { +#ifdef DEBUG + int i; + for (i = 0; i < gpu_clk->num_of_steps; i++) { + MALI_DEBUG_PRINT(5, ("mali gpu clock info: step%d clock(%d)Hz,vol(%d) \n", + i, gpu_clk->item[i].clock, gpu_clk->item[i].vol)); + } +#endif + } else { + MALI_DEBUG_PRINT(2, ("Mali DVFS init: platform didn't define enough info for ddk to do DVFS \n")); + } + + mali_gpu_get_freq = data.get_freq; + mali_gpu_set_freq = data.set_freq; + + if ((NULL != gpu_clk) && (gpu_clk->num_of_steps > 0) + && (NULL != mali_gpu_get_freq) && (NULL != mali_gpu_set_freq)) { + mali_dvfs_enabled = MALI_TRUE; + } + } else { + MALI_DEBUG_PRINT(2, ("Mali DVFS init: platform function callback incomplete, need check mali_gpu_device_data in platform .\n")); + } + } else { + err = _MALI_OSK_ERR_FAULT; + MALI_DEBUG_PRINT(2, ("Mali DVFS init: get platform data error .\n")); + } + + return err; +} + +/* + * Always give full power when start a new period, + * if mali dvfs enabled, for performance consideration + */ +void mali_dvfs_policy_new_period(void) +{ + /* Always give full power when start a new period */ + unsigned int cur_clk_step = 0; + + cur_clk_step = mali_gpu_get_freq(); + + if (cur_clk_step != (gpu_clk->num_of_steps - 1)) { + mali_gpu_set_freq(gpu_clk->num_of_steps - 1); + + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | + MALI_PROFILING_EVENT_CHANNEL_GPU | + MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE, gpu_clk->item[gpu_clk->num_of_steps - 1].clock, + gpu_clk->item[gpu_clk->num_of_steps - 1].vol / 1000, 0, 0, 0); + } +} + +mali_bool mali_dvfs_policy_enabled(void) +{ + return mali_dvfs_enabled; +} + +#if defined(CONFIG_MALI400_PROFILING) +void mali_get_current_gpu_clk_item(struct mali_gpu_clk_item *clk_item) +{ + if (mali_platform_device != NULL) { + + struct mali_gpu_device_data *device_data = NULL; + device_data = (struct mali_gpu_device_data *)mali_platform_device->dev.platform_data; + + if ((NULL != device_data->get_clock_info) && (NULL != device_data->get_freq)) { + + int cur_clk_step = device_data->get_freq(); + struct mali_gpu_clock *mali_gpu_clk = NULL; + + device_data->get_clock_info(&mali_gpu_clk); + clk_item->clock = mali_gpu_clk->item[cur_clk_step].clock; + clk_item->vol = mali_gpu_clk->item[cur_clk_step].vol; + } else { + MALI_DEBUG_PRINT(2, ("Mali GPU Utilization: platform function callback incomplete, need check mali_gpu_device_data in platform .\n")); + } + } +} +#endif + diff --git a/drivers/gpu/arm/utgard/common/mali_dvfs_policy.h b/drivers/gpu/arm/utgard/common/mali_dvfs_policy.h new file mode 100644 index 00000000000000..55e4b354c1a948 --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_dvfs_policy.h @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2010-2012, 2014-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_DVFS_POLICY_H__ +#define __MALI_DVFS_POLICY_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +void mali_dvfs_policy_realize(struct mali_gpu_utilization_data *data, u64 time_period); + +_mali_osk_errcode_t mali_dvfs_policy_init(void); + +void mali_dvfs_policy_new_period(void); + +mali_bool mali_dvfs_policy_enabled(void); + +#if defined(CONFIG_MALI400_PROFILING) +void mali_get_current_gpu_clk_item(struct mali_gpu_clk_item *clk_item); +#endif + +#ifdef __cplusplus +} +#endif + +#endif/* __MALI_DVFS_POLICY_H__ */ diff --git a/drivers/gpu/arm/utgard/common/mali_executor.c b/drivers/gpu/arm/utgard/common/mali_executor.c new file mode 100644 index 00000000000000..3c286438679890 --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_executor.c @@ -0,0 +1,2642 @@ +/* + * Copyright (C) 2012-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_executor.h" +#include "mali_scheduler.h" +#include "mali_kernel_common.h" +#include "mali_kernel_core.h" +#include "mali_osk.h" +#include "mali_osk_list.h" +#include "mali_pp.h" +#include "mali_pp_job.h" +#include "mali_group.h" +#include "mali_pm.h" +#include "mali_timeline.h" +#include "mali_osk_profiling.h" +#include "mali_session.h" + +/* + * If dma_buf with map on demand is used, we defer job deletion and job queue + * if in atomic context, since both might sleep. + */ +#if defined(CONFIG_DMA_SHARED_BUFFER) && !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) +#define MALI_EXECUTOR_USE_DEFERRED_PP_JOB_DELETE 1 +#define MALI_EXECUTOR_USE_DEFERRED_PP_JOB_QUEUE 1 +#endif /* !defined(CONFIG_DMA_SHARED_BUFFER) && !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) */ + +/* + * ---------- static type definitions (structs, enums, etc) ---------- + */ + +enum mali_executor_state_t { + EXEC_STATE_NOT_PRESENT, /* Virtual group on Mali-300/400 (do not use) */ + EXEC_STATE_DISABLED, /* Disabled by core scaling (do not use) */ + EXEC_STATE_EMPTY, /* No child groups for virtual group (do not use) */ + EXEC_STATE_INACTIVE, /* Can be used, but must be activate first */ + EXEC_STATE_IDLE, /* Active and ready to be used */ + EXEC_STATE_WORKING, /* Executing a job */ +}; + +/* + * ---------- global variables (exported due to inline functions) ---------- + */ + +/* Lock for this module (protecting all HW access except L2 caches) */ +_mali_osk_spinlock_irq_t *mali_executor_lock_obj = NULL; + +mali_bool mali_executor_hints[MALI_EXECUTOR_HINT_MAX]; + +/* + * ---------- static variables ---------- + */ + +/* Used to defer job scheduling */ +static _mali_osk_wq_work_t *executor_wq_high_pri = NULL; + +/* Store version from GP and PP (user space wants to know this) */ +static u32 pp_version = 0; +static u32 gp_version = 0; + +/* List of physical PP groups which are disabled by some external source */ +static _MALI_OSK_LIST_HEAD_STATIC_INIT(group_list_disabled); +static u32 group_list_disabled_count = 0; + +/* List of groups which can be used, but activate first */ +static _MALI_OSK_LIST_HEAD_STATIC_INIT(group_list_inactive); +static u32 group_list_inactive_count = 0; + +/* List of groups which are active and ready to be used */ +static _MALI_OSK_LIST_HEAD_STATIC_INIT(group_list_idle); +static u32 group_list_idle_count = 0; + +/* List of groups which are executing a job */ +static _MALI_OSK_LIST_HEAD_STATIC_INIT(group_list_working); +static u32 group_list_working_count = 0; + +/* Virtual group (if any) */ +static struct mali_group *virtual_group = NULL; + +/* Virtual group state is tracked with a state variable instead of 4 lists */ +static enum mali_executor_state_t virtual_group_state = EXEC_STATE_NOT_PRESENT; + +/* GP group */ +static struct mali_group *gp_group = NULL; + +/* GP group state is tracked with a state variable instead of 4 lists */ +static enum mali_executor_state_t gp_group_state = EXEC_STATE_NOT_PRESENT; + +static u32 gp_returned_cookie = 0; + +/* Total number of physical PP cores present */ +static u32 num_physical_pp_cores_total = 0; + +/* Number of physical cores which are enabled */ +static u32 num_physical_pp_cores_enabled = 0; + +/* Enable or disable core scaling */ +static mali_bool core_scaling_enabled = MALI_TRUE; + +/* Variables to allow safe pausing of the scheduler */ +static _mali_osk_wait_queue_t *executor_working_wait_queue = NULL; +static u32 pause_count = 0; + +/* PP cores haven't been enabled because of some pp cores haven't been disabled. */ +static int core_scaling_delay_up_mask[MALI_MAX_NUMBER_OF_DOMAINS] = { 0 }; + +/* Variables used to implement notify pp core changes to userspace when core scaling + * is finished in mali_executor_complete_group() function. */ +static _mali_osk_wq_work_t *executor_wq_notify_core_change = NULL; +static _mali_osk_wait_queue_t *executor_notify_core_change_wait_queue = NULL; + +/* + * ---------- Forward declaration of static functions ---------- + */ +static mali_bool mali_executor_is_suspended(void *data); +static mali_bool mali_executor_is_working(void); +static void mali_executor_disable_empty_virtual(void); +static mali_bool mali_executor_physical_rejoin_virtual(struct mali_group *group); +static mali_bool mali_executor_has_virtual_group(void); +static mali_bool mali_executor_virtual_group_is_usable(void); +static void mali_executor_schedule(void); +static void mali_executor_wq_schedule(void *arg); +static void mali_executor_send_gp_oom_to_user(struct mali_gp_job *job, u32 added_size); +static void mali_executor_complete_group(struct mali_group *group, + mali_bool success, + struct mali_gp_job **gp_job_done, + struct mali_pp_job **pp_job_done); +static void mali_executor_change_state_pp_physical(struct mali_group *group, + _mali_osk_list_t *old_list, + u32 *old_count, + _mali_osk_list_t *new_list, + u32 *new_count); +static mali_bool mali_executor_group_is_in_state(struct mali_group *group, + enum mali_executor_state_t state); + +static void mali_executor_group_enable_internal(struct mali_group *group); +static void mali_executor_group_disable_internal(struct mali_group *group); +static void mali_executor_core_scale(unsigned int target_core_nr); +static void mali_executor_core_scale_in_group_complete(struct mali_group *group); +static void mali_executor_notify_core_change(u32 num_cores); +static void mali_executor_wq_notify_core_change(void *arg); +static void mali_executor_change_group_status_disabled(struct mali_group *group); +static mali_bool mali_executor_deactivate_list_idle(mali_bool deactivate_idle_group); +static void mali_executor_set_state_pp_physical(struct mali_group *group, + _mali_osk_list_t *new_list, + u32 *new_count); + +/* + * ---------- Actual implementation ---------- + */ + +_mali_osk_errcode_t mali_executor_initialize(void) +{ + mali_executor_lock_obj = _mali_osk_spinlock_irq_init(_MALI_OSK_LOCKFLAG_ORDERED, _MALI_OSK_LOCK_ORDER_EXECUTOR); + if (NULL == mali_executor_lock_obj) { + mali_executor_terminate(); + return _MALI_OSK_ERR_NOMEM; + } + + executor_wq_high_pri = _mali_osk_wq_create_work_high_pri(mali_executor_wq_schedule, NULL); + if (NULL == executor_wq_high_pri) { + mali_executor_terminate(); + return _MALI_OSK_ERR_NOMEM; + } + + executor_working_wait_queue = _mali_osk_wait_queue_init(); + if (NULL == executor_working_wait_queue) { + mali_executor_terminate(); + return _MALI_OSK_ERR_NOMEM; + } + + executor_wq_notify_core_change = _mali_osk_wq_create_work(mali_executor_wq_notify_core_change, NULL); + if (NULL == executor_wq_notify_core_change) { + mali_executor_terminate(); + return _MALI_OSK_ERR_NOMEM; + } + + executor_notify_core_change_wait_queue = _mali_osk_wait_queue_init(); + if (NULL == executor_notify_core_change_wait_queue) { + mali_executor_terminate(); + return _MALI_OSK_ERR_NOMEM; + } + + return _MALI_OSK_ERR_OK; +} + +void mali_executor_terminate(void) +{ + if (NULL != executor_notify_core_change_wait_queue) { + _mali_osk_wait_queue_term(executor_notify_core_change_wait_queue); + executor_notify_core_change_wait_queue = NULL; + } + + if (NULL != executor_wq_notify_core_change) { + _mali_osk_wq_delete_work(executor_wq_notify_core_change); + executor_wq_notify_core_change = NULL; + } + + if (NULL != executor_working_wait_queue) { + _mali_osk_wait_queue_term(executor_working_wait_queue); + executor_working_wait_queue = NULL; + } + + if (NULL != executor_wq_high_pri) { + _mali_osk_wq_delete_work(executor_wq_high_pri); + executor_wq_high_pri = NULL; + } + + if (NULL != mali_executor_lock_obj) { + _mali_osk_spinlock_irq_term(mali_executor_lock_obj); + mali_executor_lock_obj = NULL; + } +} + +void mali_executor_populate(void) +{ + u32 num_groups; + u32 i; + + num_groups = mali_group_get_glob_num_groups(); + + /* Do we have a virtual group? */ + for (i = 0; i < num_groups; i++) { + struct mali_group *group = mali_group_get_glob_group(i); + + if (mali_group_is_virtual(group)) { + virtual_group = group; + virtual_group_state = EXEC_STATE_INACTIVE; + break; + } + } + + /* Find all the available physical GP and PP cores */ + for (i = 0; i < num_groups; i++) { + struct mali_group *group = mali_group_get_glob_group(i); + + if (NULL != group) { + struct mali_pp_core *pp_core = mali_group_get_pp_core(group); + struct mali_gp_core *gp_core = mali_group_get_gp_core(group); + + if (!mali_group_is_virtual(group)) { + if (NULL != pp_core) { + if (0 == pp_version) { + /* Retrieve PP version from the first available PP core */ + pp_version = mali_pp_core_get_version(pp_core); + } + + if (NULL != virtual_group) { + mali_executor_lock(); + mali_group_add_group(virtual_group, group); + mali_executor_unlock(); + } else { + _mali_osk_list_add(&group->executor_list, &group_list_inactive); + group_list_inactive_count++; + } + + num_physical_pp_cores_total++; + } else { + MALI_DEBUG_ASSERT_POINTER(gp_core); + + if (0 == gp_version) { + /* Retrieve GP version */ + gp_version = mali_gp_core_get_version(gp_core); + } + + gp_group = group; + gp_group_state = EXEC_STATE_INACTIVE; + } + + } + } + } + + num_physical_pp_cores_enabled = num_physical_pp_cores_total; +} + +void mali_executor_depopulate(void) +{ + struct mali_group *group; + struct mali_group *temp; + + MALI_DEBUG_ASSERT(EXEC_STATE_WORKING != gp_group_state); + + if (NULL != gp_group) { + mali_group_delete(gp_group); + gp_group = NULL; + } + + MALI_DEBUG_ASSERT(EXEC_STATE_WORKING != virtual_group_state); + + if (NULL != virtual_group) { + mali_group_delete(virtual_group); + virtual_group = NULL; + } + + MALI_DEBUG_ASSERT(_mali_osk_list_empty(&group_list_working)); + + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_idle, struct mali_group, executor_list) { + mali_group_delete(group); + } + + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_inactive, struct mali_group, executor_list) { + mali_group_delete(group); + } + + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_disabled, struct mali_group, executor_list) { + mali_group_delete(group); + } +} + +void mali_executor_suspend(void) +{ + mali_executor_lock(); + + /* Increment the pause_count so that no more jobs will be scheduled */ + pause_count++; + + mali_executor_unlock(); + + _mali_osk_wait_queue_wait_event(executor_working_wait_queue, + mali_executor_is_suspended, NULL); + + /* + * mali_executor_complete_XX() leaves jobs in idle state. + * deactivate option is used when we are going to power down + * the entire GPU (OS suspend) and want a consistent SW vs HW + * state. + */ + mali_executor_lock(); + + mali_executor_deactivate_list_idle(MALI_TRUE); + + /* + * The following steps are used to deactive all of activated + * (MALI_GROUP_STATE_ACTIVE) and activating (MALI_GROUP + * _STAET_ACTIVATION_PENDING) groups, to make sure the variable + * pd_mask_wanted is equal with 0. */ + if (MALI_GROUP_STATE_INACTIVE != mali_group_get_state(gp_group)) { + gp_group_state = EXEC_STATE_INACTIVE; + mali_group_deactivate(gp_group); + } + + if (mali_executor_has_virtual_group()) { + if (MALI_GROUP_STATE_INACTIVE + != mali_group_get_state(virtual_group)) { + virtual_group_state = EXEC_STATE_INACTIVE; + mali_group_deactivate(virtual_group); + } + } + + if (0 < group_list_inactive_count) { + struct mali_group *group; + struct mali_group *temp; + + _MALI_OSK_LIST_FOREACHENTRY(group, temp, + &group_list_inactive, + struct mali_group, executor_list) { + if (MALI_GROUP_STATE_ACTIVATION_PENDING + == mali_group_get_state(group)) { + mali_group_deactivate(group); + } + + /* + * On mali-450 platform, we may have physical group in the group inactive + * list, and its state is MALI_GROUP_STATE_ACTIVATION_PENDING, so we only + * deactivate it is not enough, we still also need add it back to virtual group. + * And now, virtual group must be in INACTIVE state, so it's safe to add + * physical group to virtual group at this point. + */ + if (NULL != virtual_group) { + _mali_osk_list_delinit(&group->executor_list); + group_list_inactive_count--; + + mali_group_add_group(virtual_group, group); + } + } + } + + mali_executor_unlock(); +} + +void mali_executor_resume(void) +{ + mali_executor_lock(); + + /* Decrement pause_count to allow scheduling again (if it reaches 0) */ + pause_count--; + if (0 == pause_count) { + mali_executor_schedule(); + } + + mali_executor_unlock(); +} + +u32 mali_executor_get_num_cores_total(void) +{ + return num_physical_pp_cores_total; +} + +u32 mali_executor_get_num_cores_enabled(void) +{ + return num_physical_pp_cores_enabled; +} + +struct mali_pp_core *mali_executor_get_virtual_pp(void) +{ + MALI_DEBUG_ASSERT_POINTER(virtual_group); + MALI_DEBUG_ASSERT_POINTER(virtual_group->pp_core); + return virtual_group->pp_core; +} + +struct mali_group *mali_executor_get_virtual_group(void) +{ + return virtual_group; +} + +void mali_executor_zap_all_active(struct mali_session_data *session) +{ + struct mali_group *group; + struct mali_group *temp; + mali_bool ret; + + mali_executor_lock(); + + /* + * This function is a bit complicated because + * mali_group_zap_session() can fail. This only happens because the + * group is in an unhandled page fault status. + * We need to make sure this page fault is handled before we return, + * so that we know every single outstanding MMU transactions have + * completed. This will allow caller to safely remove physical pages + * when we have returned. + */ + + MALI_DEBUG_ASSERT(NULL != gp_group); + ret = mali_group_zap_session(gp_group, session); + if (MALI_FALSE == ret) { + struct mali_gp_job *gp_job = NULL; + + mali_executor_complete_group(gp_group, MALI_FALSE, &gp_job, NULL); + + MALI_DEBUG_ASSERT_POINTER(gp_job); + + /* GP job completed, make sure it is freed */ + mali_scheduler_complete_gp_job(gp_job, MALI_FALSE, + MALI_TRUE, MALI_TRUE); + } + + if (mali_executor_has_virtual_group()) { + ret = mali_group_zap_session(virtual_group, session); + if (MALI_FALSE == ret) { + struct mali_pp_job *pp_job = NULL; + + mali_executor_complete_group(virtual_group, MALI_FALSE, NULL, &pp_job); + + if (NULL != pp_job) { + /* PP job completed, make sure it is freed */ + mali_scheduler_complete_pp_job(pp_job, 0, + MALI_FALSE, MALI_TRUE); + } + } + } + + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_working, + struct mali_group, executor_list) { + ret = mali_group_zap_session(group, session); + if (MALI_FALSE == ret) { + ret = mali_group_zap_session(group, session); + if (MALI_FALSE == ret) { + struct mali_pp_job *pp_job = NULL; + + mali_executor_complete_group(group, MALI_FALSE, NULL, &pp_job); + + if (NULL != pp_job) { + /* PP job completed, free it */ + mali_scheduler_complete_pp_job(pp_job, + 0, MALI_FALSE, + MALI_TRUE); + } + } + } + } + + mali_executor_unlock(); +} + +void mali_executor_schedule_from_mask(mali_scheduler_mask mask, mali_bool deferred_schedule) +{ + if (MALI_SCHEDULER_MASK_EMPTY != mask) { + if (MALI_TRUE == deferred_schedule) { + _mali_osk_wq_schedule_work_high_pri(executor_wq_high_pri); + } else { + /* Schedule from this thread*/ + mali_executor_lock(); + mali_executor_schedule(); + mali_executor_unlock(); + } + } +} + +_mali_osk_errcode_t mali_executor_interrupt_gp(struct mali_group *group, + mali_bool in_upper_half) +{ + enum mali_interrupt_result int_result; + mali_bool time_out = MALI_FALSE; + + MALI_DEBUG_PRINT(4, ("Executor: GP interrupt from %s in %s half\n", + mali_group_core_description(group), + in_upper_half ? "upper" : "bottom")); + + mali_executor_lock(); + if (!mali_group_is_working(group)) { + /* Not working, so nothing to do */ + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + MALI_DEBUG_ASSERT(mali_group_is_working(group)); + + if (mali_group_has_timed_out(group)) { + int_result = MALI_INTERRUPT_RESULT_ERROR; + time_out = MALI_TRUE; + MALI_PRINT(("Executor GP: Job %d Timeout on %s\n", + mali_gp_job_get_id(group->gp_running_job), + mali_group_core_description(group))); + } else { + int_result = mali_group_get_interrupt_result_gp(group); + if (MALI_INTERRUPT_RESULT_NONE == int_result) { + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } + } + +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + if (MALI_INTERRUPT_RESULT_NONE == int_result) { + /* No interrupts signalled, so nothing to do */ + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } +#else + MALI_DEBUG_ASSERT(MALI_INTERRUPT_RESULT_NONE != int_result); +#endif + + mali_group_mask_all_interrupts_gp(group); + + if (MALI_INTERRUPT_RESULT_SUCCESS_VS == int_result) { + if (mali_group_gp_is_active(group)) { + /* Only VS completed so far, while PLBU is still active */ + + /* Enable all but the current interrupt */ + mali_group_enable_interrupts_gp(group, int_result); + + mali_executor_unlock(); + return _MALI_OSK_ERR_OK; + } + } else if (MALI_INTERRUPT_RESULT_SUCCESS_PLBU == int_result) { + if (mali_group_gp_is_active(group)) { + /* Only PLBU completed so far, while VS is still active */ + + /* Enable all but the current interrupt */ + mali_group_enable_interrupts_gp(group, int_result); + + mali_executor_unlock(); + return _MALI_OSK_ERR_OK; + } + } else if (MALI_INTERRUPT_RESULT_OOM == int_result) { + + mali_executor_unlock(); + + mali_group_schedule_oom_work_handler(group); + + return _MALI_OSK_ERR_OK; + } + + /* We should now have a real interrupt to handle */ + + MALI_DEBUG_PRINT(4, ("Executor: Group %s completed with %s\n", + mali_group_core_description(group), + (MALI_INTERRUPT_RESULT_ERROR == int_result) ? + "ERROR" : "success")); + + if (in_upper_half && MALI_INTERRUPT_RESULT_ERROR == int_result) { + /* Don't bother to do processing of errors in upper half */ + mali_executor_unlock(); + + if (MALI_FALSE == time_out) { + mali_group_schedule_bottom_half_gp(group); + } + } else { + struct mali_gp_job *job; + mali_bool success; + + if (MALI_TRUE == time_out) { + mali_group_dump_status(group); + } + + success = (int_result != MALI_INTERRUPT_RESULT_ERROR) ? + MALI_TRUE : MALI_FALSE; + + mali_executor_complete_group(group, success, &job, NULL); + + mali_executor_unlock(); + + /* GP jobs always fully complete */ + MALI_DEBUG_ASSERT(NULL != job); + + /* This will notify user space and close the job object */ + mali_scheduler_complete_gp_job(job, success, + MALI_TRUE, MALI_TRUE); + } + + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t mali_executor_interrupt_pp(struct mali_group *group, + mali_bool in_upper_half) +{ + enum mali_interrupt_result int_result; + mali_bool time_out = MALI_FALSE; + + MALI_DEBUG_PRINT(4, ("Executor: PP interrupt from %s in %s half\n", + mali_group_core_description(group), + in_upper_half ? "upper" : "bottom")); + + mali_executor_lock(); + + if (!mali_group_is_working(group)) { + /* Not working, so nothing to do */ + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } + + if (in_upper_half) { + if (mali_group_is_in_virtual(group)) { + /* Child groups should never handle PP interrupts */ + MALI_DEBUG_ASSERT(!mali_group_has_timed_out(group)); + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } + } + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + MALI_DEBUG_ASSERT(mali_group_is_working(group)); + MALI_DEBUG_ASSERT(!mali_group_is_in_virtual(group)); + + if (mali_group_has_timed_out(group)) { + int_result = MALI_INTERRUPT_RESULT_ERROR; + time_out = MALI_TRUE; + MALI_PRINT(("Executor PP: Job %d Timeout on %s\n", + mali_pp_job_get_id(group->pp_running_job), + mali_group_core_description(group))); + } else { + int_result = mali_group_get_interrupt_result_pp(group); + if (MALI_INTERRUPT_RESULT_NONE == int_result) { + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } + } + +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + if (MALI_INTERRUPT_RESULT_NONE == int_result) { + /* No interrupts signalled, so nothing to do */ + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } else if (MALI_INTERRUPT_RESULT_SUCCESS == int_result) { + if (mali_group_is_virtual(group) && mali_group_pp_is_active(group)) { + /* Some child groups are still working, so nothing to do right now */ + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } + } +#else + MALI_DEBUG_ASSERT(MALI_INTERRUPT_RESULT_NONE != int_result); +#endif + + /* We should now have a real interrupt to handle */ + + MALI_DEBUG_PRINT(4, ("Executor: Group %s completed with %s\n", + mali_group_core_description(group), + (MALI_INTERRUPT_RESULT_ERROR == int_result) ? + "ERROR" : "success")); + + if (in_upper_half && MALI_INTERRUPT_RESULT_ERROR == int_result) { + /* Don't bother to do processing of errors in upper half */ + mali_group_mask_all_interrupts_pp(group); + mali_executor_unlock(); + + if (MALI_FALSE == time_out) { + mali_group_schedule_bottom_half_pp(group); + } + } else { + struct mali_pp_job *job = NULL; + mali_bool success; + + if (MALI_TRUE == time_out) { + mali_group_dump_status(group); + } + + success = (int_result == MALI_INTERRUPT_RESULT_SUCCESS) ? + MALI_TRUE : MALI_FALSE; + + mali_executor_complete_group(group, success, NULL, &job); + + mali_executor_unlock(); + + if (NULL != job) { + /* Notify user space and close the job object */ + mali_scheduler_complete_pp_job(job, + num_physical_pp_cores_total, + MALI_TRUE, MALI_TRUE); + } + } + + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t mali_executor_interrupt_mmu(struct mali_group *group, + mali_bool in_upper_half) +{ + enum mali_interrupt_result int_result; + + MALI_DEBUG_PRINT(4, ("Executor: MMU interrupt from %s in %s half\n", + mali_group_core_description(group), + in_upper_half ? "upper" : "bottom")); + + mali_executor_lock(); + if (!mali_group_is_working(group)) { + /* Not working, so nothing to do */ + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + MALI_DEBUG_ASSERT(mali_group_is_working(group)); + + int_result = mali_group_get_interrupt_result_mmu(group); + if (MALI_INTERRUPT_RESULT_NONE == int_result) { + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } + +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + if (MALI_INTERRUPT_RESULT_NONE == int_result) { + /* No interrupts signalled, so nothing to do */ + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } +#else + MALI_DEBUG_ASSERT(MALI_INTERRUPT_RESULT_ERROR == int_result); +#endif + + /* We should now have a real interrupt to handle */ + + if (in_upper_half) { + /* Don't bother to do processing of errors in upper half */ + + struct mali_group *parent = group->parent_group; + + mali_mmu_mask_all_interrupts(group->mmu); + + mali_executor_unlock(); + + if (NULL == parent) { + mali_group_schedule_bottom_half_mmu(group); + } else { + mali_group_schedule_bottom_half_mmu(parent); + } + + } else { + struct mali_gp_job *gp_job = NULL; + struct mali_pp_job *pp_job = NULL; + +#ifdef DEBUG + + u32 fault_address = mali_mmu_get_page_fault_addr(group->mmu); + u32 status = mali_mmu_get_status(group->mmu); + MALI_DEBUG_PRINT(2, ("Executor: Mali page fault detected at 0x%x from bus id %d of type %s on %s\n", + (void *)(uintptr_t)fault_address, + (status >> 6) & 0x1F, + (status & 32) ? "write" : "read", + group->mmu->hw_core.description)); + MALI_DEBUG_PRINT(3, ("Executor: MMU rawstat = 0x%08X, MMU status = 0x%08X\n", + mali_mmu_get_rawstat(group->mmu), status)); + mali_mmu_pagedir_diag(mali_session_get_page_directory(group->session), fault_address); +#endif + + mali_executor_complete_group(group, MALI_FALSE, &gp_job, &pp_job); + + mali_executor_unlock(); + + if (NULL != gp_job) { + MALI_DEBUG_ASSERT(NULL == pp_job); + + /* Notify user space and close the job object */ + mali_scheduler_complete_gp_job(gp_job, MALI_FALSE, + MALI_TRUE, MALI_TRUE); + } else if (NULL != pp_job) { + MALI_DEBUG_ASSERT(NULL == gp_job); + + /* Notify user space and close the job object */ + mali_scheduler_complete_pp_job(pp_job, + num_physical_pp_cores_total, + MALI_TRUE, MALI_TRUE); + } + } + + return _MALI_OSK_ERR_OK; +} + +void mali_executor_group_oom(struct mali_group *group) +{ + struct mali_gp_job *job = NULL; + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->gp_core); + MALI_DEBUG_ASSERT_POINTER(group->mmu); + + mali_executor_lock(); + + job = mali_group_get_running_gp_job(group); + + MALI_DEBUG_ASSERT_POINTER(job); + +#if defined(CONFIG_MALI400_PROFILING) + /* Give group a chance to generate a SUSPEND event */ + mali_group_oom(group); +#endif + + mali_gp_job_set_current_heap_addr(job, mali_gp_read_plbu_alloc_start_addr(group->gp_core)); + + mali_executor_unlock(); + + if (_MALI_OSK_ERR_OK == mali_mem_add_mem_size(job->session, job->heap_base_addr, job->heap_grow_size)) { + _mali_osk_notification_t *new_notification = NULL; + + new_notification = _mali_osk_notification_create( + _MALI_NOTIFICATION_GP_STALLED, + sizeof(_mali_uk_gp_job_suspended_s)); + + /* resume job with new heap, + * This will also re-enable interrupts + */ + mali_executor_lock(); + + mali_executor_send_gp_oom_to_user(job, job->heap_grow_size); + + if (NULL != new_notification) { + + mali_gp_job_set_oom_notification(job, new_notification); + + mali_group_resume_gp_with_new_heap(group, mali_gp_job_get_id(job), + job->heap_current_addr, + job->heap_current_addr + job->heap_grow_size); + } + mali_executor_unlock(); + } else { + mali_executor_lock(); + mali_executor_send_gp_oom_to_user(job, 0); + mali_executor_unlock(); + } + +} + +void mali_executor_group_power_up(struct mali_group *groups[], u32 num_groups) +{ + u32 i; + mali_bool child_groups_activated = MALI_FALSE; + mali_bool do_schedule = MALI_FALSE; +#if defined(DEBUG) + u32 num_activated = 0; +#endif + + MALI_DEBUG_ASSERT_POINTER(groups); + MALI_DEBUG_ASSERT(0 < num_groups); + + mali_executor_lock(); + + MALI_DEBUG_PRINT(3, ("Executor: powering up %u groups\n", num_groups)); + + for (i = 0; i < num_groups; i++) { + MALI_DEBUG_PRINT(3, ("Executor: powering up group %s\n", + mali_group_core_description(groups[i]))); + + mali_group_power_up(groups[i]); + + if ((MALI_GROUP_STATE_ACTIVATION_PENDING != mali_group_get_state(groups[i]) || + (MALI_TRUE != mali_executor_group_is_in_state(groups[i], EXEC_STATE_INACTIVE)))) { + /* nothing more to do for this group */ + continue; + } + + MALI_DEBUG_PRINT(3, ("Executor: activating group %s\n", + mali_group_core_description(groups[i]))); + +#if defined(DEBUG) + num_activated++; +#endif + + if (mali_group_is_in_virtual(groups[i])) { + /* + * At least one child group of virtual group is powered on. + */ + child_groups_activated = MALI_TRUE; + } else if (MALI_FALSE == mali_group_is_virtual(groups[i])) { + /* Set gp and pp not in virtual to active. */ + mali_group_set_active(groups[i]); + } + + /* Move group from inactive to idle list */ + if (groups[i] == gp_group) { + MALI_DEBUG_ASSERT(EXEC_STATE_INACTIVE == + gp_group_state); + gp_group_state = EXEC_STATE_IDLE; + } else if (MALI_FALSE == mali_group_is_in_virtual(groups[i]) + && MALI_FALSE == mali_group_is_virtual(groups[i])) { + MALI_DEBUG_ASSERT(MALI_TRUE == mali_executor_group_is_in_state(groups[i], + EXEC_STATE_INACTIVE)); + + mali_executor_change_state_pp_physical(groups[i], + &group_list_inactive, + &group_list_inactive_count, + &group_list_idle, + &group_list_idle_count); + } + + do_schedule = MALI_TRUE; + } + + if (mali_executor_has_virtual_group() && + MALI_TRUE == child_groups_activated && + MALI_GROUP_STATE_ACTIVATION_PENDING == + mali_group_get_state(virtual_group)) { + /* + * Try to active virtual group while it may be not sucessful every time, + * because there is one situation that not all of child groups are powered on + * in one time and virtual group is in activation pending state. + */ + if (mali_group_set_active(virtual_group)) { + /* Move group from inactive to idle */ + MALI_DEBUG_ASSERT(EXEC_STATE_INACTIVE == + virtual_group_state); + virtual_group_state = EXEC_STATE_IDLE; + + MALI_DEBUG_PRINT(3, ("Executor: powering up %u groups completed, %u physical activated, 1 virtual activated.\n", num_groups, num_activated)); + } else { + MALI_DEBUG_PRINT(3, ("Executor: powering up %u groups completed, %u physical activated\n", num_groups, num_activated)); + } + } else { + MALI_DEBUG_PRINT(3, ("Executor: powering up %u groups completed, %u physical activated\n", num_groups, num_activated)); + } + + if (MALI_TRUE == do_schedule) { + /* Trigger a schedule */ + mali_executor_schedule(); + } + + mali_executor_unlock(); +} + +void mali_executor_group_power_down(struct mali_group *groups[], + u32 num_groups) +{ + u32 i; + + MALI_DEBUG_ASSERT_POINTER(groups); + MALI_DEBUG_ASSERT(0 < num_groups); + + mali_executor_lock(); + + MALI_DEBUG_PRINT(3, ("Executor: powering down %u groups\n", num_groups)); + + for (i = 0; i < num_groups; i++) { + /* Groups must be either disabled or inactive. while for virtual group, + * it maybe in empty state, because when we meet pm_runtime_suspend, + * virtual group could be powered off, and before we acquire mali_executor_lock, + * we must release mali_pm_state_lock, if there is a new physical job was queued, + * all of physical groups in virtual group could be pulled out, so we only can + * powered down an empty virtual group. Those physical groups will be powered + * up in following pm_runtime_resume callback function. + */ + MALI_DEBUG_ASSERT(mali_executor_group_is_in_state(groups[i], + EXEC_STATE_DISABLED) || + mali_executor_group_is_in_state(groups[i], + EXEC_STATE_INACTIVE) || + mali_executor_group_is_in_state(groups[i], + EXEC_STATE_EMPTY)); + + MALI_DEBUG_PRINT(3, ("Executor: powering down group %s\n", + mali_group_core_description(groups[i]))); + + mali_group_power_down(groups[i]); + } + + MALI_DEBUG_PRINT(3, ("Executor: powering down %u groups completed\n", num_groups)); + + mali_executor_unlock(); +} + +void mali_executor_abort_session(struct mali_session_data *session) +{ + struct mali_group *group; + struct mali_group *tmp_group; + + MALI_DEBUG_ASSERT_POINTER(session); + MALI_DEBUG_ASSERT(session->is_aborting); + + MALI_DEBUG_PRINT(3, + ("Executor: Aborting all jobs from session 0x%08X.\n", + session)); + + mali_executor_lock(); + + if (mali_group_get_session(gp_group) == session) { + if (EXEC_STATE_WORKING == gp_group_state) { + struct mali_gp_job *gp_job = NULL; + + mali_executor_complete_group(gp_group, MALI_FALSE, &gp_job, NULL); + + MALI_DEBUG_ASSERT_POINTER(gp_job); + + /* GP job completed, make sure it is freed */ + mali_scheduler_complete_gp_job(gp_job, MALI_FALSE, + MALI_FALSE, MALI_TRUE); + } else { + /* Same session, but not working, so just clear it */ + mali_group_clear_session(gp_group); + } + } + + if (mali_executor_has_virtual_group()) { + if (EXEC_STATE_WORKING == virtual_group_state + && mali_group_get_session(virtual_group) == session) { + struct mali_pp_job *pp_job = NULL; + + mali_executor_complete_group(virtual_group, MALI_FALSE, NULL, &pp_job); + + if (NULL != pp_job) { + /* PP job completed, make sure it is freed */ + mali_scheduler_complete_pp_job(pp_job, 0, + MALI_FALSE, MALI_TRUE); + } + } + } + + _MALI_OSK_LIST_FOREACHENTRY(group, tmp_group, &group_list_working, + struct mali_group, executor_list) { + if (mali_group_get_session(group) == session) { + struct mali_pp_job *pp_job = NULL; + + mali_executor_complete_group(group, MALI_FALSE, NULL, &pp_job); + + if (NULL != pp_job) { + /* PP job completed, make sure it is freed */ + mali_scheduler_complete_pp_job(pp_job, 0, + MALI_FALSE, MALI_TRUE); + } + } + } + + _MALI_OSK_LIST_FOREACHENTRY(group, tmp_group, &group_list_idle, struct mali_group, executor_list) { + mali_group_clear_session(group); + } + + _MALI_OSK_LIST_FOREACHENTRY(group, tmp_group, &group_list_inactive, struct mali_group, executor_list) { + mali_group_clear_session(group); + } + + _MALI_OSK_LIST_FOREACHENTRY(group, tmp_group, &group_list_disabled, struct mali_group, executor_list) { + mali_group_clear_session(group); + } + + mali_executor_unlock(); +} + + +void mali_executor_core_scaling_enable(void) +{ + /* PS: Core scaling is by default enabled */ + core_scaling_enabled = MALI_TRUE; +} + +void mali_executor_core_scaling_disable(void) +{ + core_scaling_enabled = MALI_FALSE; +} + +mali_bool mali_executor_core_scaling_is_enabled(void) +{ + return core_scaling_enabled; +} + +void mali_executor_group_enable(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + + mali_executor_lock(); + + if ((NULL != mali_group_get_gp_core(group) || NULL != mali_group_get_pp_core(group)) + && (mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED))) { + mali_executor_group_enable_internal(group); + } + + mali_executor_schedule(); + mali_executor_unlock(); + + _mali_osk_wq_schedule_work(executor_wq_notify_core_change); +} + +/* + * If a physical group is inactive or idle, we should disable it immediately, + * if group is in virtual, and virtual group is idle, disable given physical group in it. + */ +void mali_executor_group_disable(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + + mali_executor_lock(); + + if ((NULL != mali_group_get_gp_core(group) || NULL != mali_group_get_pp_core(group)) + && (!mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED))) { + mali_executor_group_disable_internal(group); + } + + mali_executor_schedule(); + mali_executor_unlock(); + + _mali_osk_wq_schedule_work(executor_wq_notify_core_change); +} + +mali_bool mali_executor_group_is_disabled(struct mali_group *group) +{ + /* NB: This function is not optimized for time critical usage */ + + mali_bool ret; + + MALI_DEBUG_ASSERT_POINTER(group); + + mali_executor_lock(); + ret = mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED); + mali_executor_unlock(); + + return ret; +} + +int mali_executor_set_perf_level(unsigned int target_core_nr, mali_bool override) +{ + if (target_core_nr == num_physical_pp_cores_enabled) return 0; + if (MALI_FALSE == core_scaling_enabled && MALI_FALSE == override) return -EPERM; + if (target_core_nr > num_physical_pp_cores_total) return -EINVAL; + if (0 == target_core_nr) return -EINVAL; + + mali_executor_core_scale(target_core_nr); + + _mali_osk_wq_schedule_work(executor_wq_notify_core_change); + + return 0; +} + +#if MALI_STATE_TRACKING +u32 mali_executor_dump_state(char *buf, u32 size) +{ + int n = 0; + struct mali_group *group; + struct mali_group *temp; + + mali_executor_lock(); + + switch (gp_group_state) { + case EXEC_STATE_INACTIVE: + n += _mali_osk_snprintf(buf + n, size - n, + "GP group is in state INACTIVE\n"); + break; + case EXEC_STATE_IDLE: + n += _mali_osk_snprintf(buf + n, size - n, + "GP group is in state IDLE\n"); + break; + case EXEC_STATE_WORKING: + n += _mali_osk_snprintf(buf + n, size - n, + "GP group is in state WORKING\n"); + break; + default: + n += _mali_osk_snprintf(buf + n, size - n, + "GP group is in unknown/illegal state %u\n", + gp_group_state); + break; + } + + n += mali_group_dump_state(gp_group, buf + n, size - n); + + n += _mali_osk_snprintf(buf + n, size - n, + "Physical PP groups in WORKING state (count = %u):\n", + group_list_working_count); + + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_working, struct mali_group, executor_list) { + n += mali_group_dump_state(group, buf + n, size - n); + } + + n += _mali_osk_snprintf(buf + n, size - n, + "Physical PP groups in IDLE state (count = %u):\n", + group_list_idle_count); + + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_idle, struct mali_group, executor_list) { + n += mali_group_dump_state(group, buf + n, size - n); + } + + n += _mali_osk_snprintf(buf + n, size - n, + "Physical PP groups in INACTIVE state (count = %u):\n", + group_list_inactive_count); + + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_inactive, struct mali_group, executor_list) { + n += mali_group_dump_state(group, buf + n, size - n); + } + + n += _mali_osk_snprintf(buf + n, size - n, + "Physical PP groups in DISABLED state (count = %u):\n", + group_list_disabled_count); + + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_disabled, struct mali_group, executor_list) { + n += mali_group_dump_state(group, buf + n, size - n); + } + + if (mali_executor_has_virtual_group()) { + switch (virtual_group_state) { + case EXEC_STATE_EMPTY: + n += _mali_osk_snprintf(buf + n, size - n, + "Virtual PP group is in state EMPTY\n"); + break; + case EXEC_STATE_INACTIVE: + n += _mali_osk_snprintf(buf + n, size - n, + "Virtual PP group is in state INACTIVE\n"); + break; + case EXEC_STATE_IDLE: + n += _mali_osk_snprintf(buf + n, size - n, + "Virtual PP group is in state IDLE\n"); + break; + case EXEC_STATE_WORKING: + n += _mali_osk_snprintf(buf + n, size - n, + "Virtual PP group is in state WORKING\n"); + break; + default: + n += _mali_osk_snprintf(buf + n, size - n, + "Virtual PP group is in unknown/illegal state %u\n", + virtual_group_state); + break; + } + + n += mali_group_dump_state(virtual_group, buf + n, size - n); + } + + mali_executor_unlock(); + + n += _mali_osk_snprintf(buf + n, size - n, "\n"); + + return n; +} +#endif + +_mali_osk_errcode_t _mali_ukk_get_pp_number_of_cores(_mali_uk_get_pp_number_of_cores_s *args) +{ + MALI_DEBUG_ASSERT_POINTER(args); + MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); + args->number_of_total_cores = num_physical_pp_cores_total; + args->number_of_enabled_cores = num_physical_pp_cores_enabled; + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t _mali_ukk_get_pp_core_version(_mali_uk_get_pp_core_version_s *args) +{ + MALI_DEBUG_ASSERT_POINTER(args); + MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); + args->version = pp_version; + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t _mali_ukk_get_gp_number_of_cores(_mali_uk_get_gp_number_of_cores_s *args) +{ + MALI_DEBUG_ASSERT_POINTER(args); + MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); + args->number_of_cores = 1; + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t _mali_ukk_get_gp_core_version(_mali_uk_get_gp_core_version_s *args) +{ + MALI_DEBUG_ASSERT_POINTER(args); + MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); + args->version = gp_version; + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t _mali_ukk_gp_suspend_response(_mali_uk_gp_suspend_response_s *args) +{ + struct mali_session_data *session; + struct mali_gp_job *job; + + MALI_DEBUG_ASSERT_POINTER(args); + MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); + + session = (struct mali_session_data *)(uintptr_t)args->ctx; + + if (_MALIGP_JOB_RESUME_WITH_NEW_HEAP == args->code) { + _mali_osk_notification_t *new_notification = NULL; + + new_notification = _mali_osk_notification_create( + _MALI_NOTIFICATION_GP_STALLED, + sizeof(_mali_uk_gp_job_suspended_s)); + + if (NULL != new_notification) { + MALI_DEBUG_PRINT(3, ("Executor: Resuming job %u with new heap; 0x%08X - 0x%08X\n", + args->cookie, args->arguments[0], args->arguments[1])); + + mali_executor_lock(); + + /* Resume the job in question if it is still running */ + job = mali_group_get_running_gp_job(gp_group); + if (NULL != job && + args->cookie == mali_gp_job_get_id(job) && + session == mali_gp_job_get_session(job)) { + /* + * Correct job is running, resume with new heap + */ + + mali_gp_job_set_oom_notification(job, + new_notification); + + /* This will also re-enable interrupts */ + mali_group_resume_gp_with_new_heap(gp_group, + args->cookie, + args->arguments[0], + args->arguments[1]); + + job->heap_base_addr = args->arguments[0]; + job->heap_current_addr = args->arguments[0]; + + mali_executor_unlock(); + return _MALI_OSK_ERR_OK; + } else { + MALI_PRINT_ERROR(("Executor: Unable to resume, GP job no longer running.\n")); + + _mali_osk_notification_delete(new_notification); + + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } + } else { + MALI_PRINT_ERROR(("Executor: Failed to allocate notification object. Will abort GP job.\n")); + } + } else { + MALI_DEBUG_PRINT(2, ("Executor: Aborting job %u, no new heap provided\n", args->cookie)); + } + + mali_executor_lock(); + + /* Abort the job in question if it is still running */ + job = mali_group_get_running_gp_job(gp_group); + if (NULL != job && + args->cookie == mali_gp_job_get_id(job) && + session == mali_gp_job_get_session(job)) { + /* Correct job is still running */ + struct mali_gp_job *job_done = NULL; + + mali_executor_complete_group(gp_group, MALI_FALSE, &job_done, NULL); + + /* The same job should have completed */ + MALI_DEBUG_ASSERT(job_done == job); + + /* GP job completed, make sure it is freed */ + mali_scheduler_complete_gp_job(job_done, MALI_FALSE, + MALI_TRUE, MALI_TRUE); + } + + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; +} + + +/* + * ---------- Implementation of static functions ---------- + */ + +void mali_executor_lock(void) +{ + _mali_osk_spinlock_irq_lock(mali_executor_lock_obj); + MALI_DEBUG_PRINT(5, ("Executor: lock taken\n")); +} + +void mali_executor_unlock(void) +{ + MALI_DEBUG_PRINT(5, ("Executor: Releasing lock\n")); + _mali_osk_spinlock_irq_unlock(mali_executor_lock_obj); +} + +static mali_bool mali_executor_is_suspended(void *data) +{ + mali_bool ret; + + /* This callback does not use the data pointer. */ + MALI_IGNORE(data); + + mali_executor_lock(); + + ret = pause_count > 0 && !mali_executor_is_working(); + + mali_executor_unlock(); + + return ret; +} + +static mali_bool mali_executor_is_working() +{ + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + return (0 != group_list_working_count || + EXEC_STATE_WORKING == gp_group_state || + EXEC_STATE_WORKING == virtual_group_state); +} + +static void mali_executor_disable_empty_virtual(void) +{ + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + MALI_DEBUG_ASSERT(virtual_group_state != EXEC_STATE_EMPTY); + MALI_DEBUG_ASSERT(virtual_group_state != EXEC_STATE_WORKING); + + if (mali_group_is_empty(virtual_group)) { + virtual_group_state = EXEC_STATE_EMPTY; + } +} + +static mali_bool mali_executor_physical_rejoin_virtual(struct mali_group *group) +{ + mali_bool trigger_pm_update = MALI_FALSE; + + MALI_DEBUG_ASSERT_POINTER(group); + /* Only rejoining after job has completed (still active) */ + MALI_DEBUG_ASSERT(MALI_GROUP_STATE_ACTIVE == + mali_group_get_state(group)); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + MALI_DEBUG_ASSERT(MALI_TRUE == mali_executor_has_virtual_group()); + MALI_DEBUG_ASSERT(MALI_FALSE == mali_group_is_virtual(group)); + + /* Make sure group and virtual group have same status */ + + if (MALI_GROUP_STATE_INACTIVE == mali_group_get_state(virtual_group)) { + if (mali_group_deactivate(group)) { + trigger_pm_update = MALI_TRUE; + } + + if (virtual_group_state == EXEC_STATE_EMPTY) { + virtual_group_state = EXEC_STATE_INACTIVE; + } + } else if (MALI_GROUP_STATE_ACTIVATION_PENDING == + mali_group_get_state(virtual_group)) { + /* + * Activation is pending for virtual group, leave + * this child group as active. + */ + if (virtual_group_state == EXEC_STATE_EMPTY) { + virtual_group_state = EXEC_STATE_INACTIVE; + } + } else { + MALI_DEBUG_ASSERT(MALI_GROUP_STATE_ACTIVE == + mali_group_get_state(virtual_group)); + + if (virtual_group_state == EXEC_STATE_EMPTY) { + virtual_group_state = EXEC_STATE_IDLE; + } + } + + /* Remove group from idle list */ + MALI_DEBUG_ASSERT(mali_executor_group_is_in_state(group, + EXEC_STATE_IDLE)); + _mali_osk_list_delinit(&group->executor_list); + group_list_idle_count--; + + /* + * And finally rejoin the virtual group + * group will start working on same job as virtual_group, + * if virtual_group is working on a job + */ + mali_group_add_group(virtual_group, group); + + return trigger_pm_update; +} + +static mali_bool mali_executor_has_virtual_group(void) +{ +#if (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) + return (NULL != virtual_group) ? MALI_TRUE : MALI_FALSE; +#else + return MALI_FALSE; +#endif /* (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) */ +} + +static mali_bool mali_executor_virtual_group_is_usable(void) +{ +#if (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + return ((EXEC_STATE_INACTIVE == virtual_group_state || + EXEC_STATE_IDLE == virtual_group_state) && (virtual_group->state != MALI_GROUP_STATE_ACTIVATION_PENDING)) ? + MALI_TRUE : MALI_FALSE; +#else + return MALI_FALSE; +#endif /* (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) */ +} + +static mali_bool mali_executor_tackle_gp_bound(void) +{ + struct mali_pp_job *job; + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + job = mali_scheduler_job_pp_physical_peek(); + + if (NULL != job && MALI_TRUE == mali_is_mali400()) { + if (0 < group_list_working_count && + mali_pp_job_is_large_and_unstarted(job)) { + return MALI_TRUE; + } + } + + return MALI_FALSE; +} + +/* + * This is where jobs are actually started. + */ +static void mali_executor_schedule(void) +{ + u32 i; + u32 num_physical_needed = 0; + u32 num_physical_to_process = 0; + mali_bool trigger_pm_update = MALI_FALSE; + mali_bool deactivate_idle_group = MALI_TRUE; + + /* Physical groups + jobs to start in this function */ + struct mali_group *groups_to_start[MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS]; + struct mali_pp_job *jobs_to_start[MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS]; + u32 sub_jobs_to_start[MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS]; + int num_jobs_to_start = 0; + + /* Virtual job to start in this function */ + struct mali_pp_job *virtual_job_to_start = NULL; + + /* GP job to start in this function */ + struct mali_gp_job *gp_job_to_start = NULL; + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + if (pause_count > 0) { + /* Execution is suspended, don't schedule any jobs. */ + return; + } + + /* Lock needed in order to safely handle the job queues */ + mali_scheduler_lock(); + + /* 1. Activate gp firstly if have gp job queued. */ + if (EXEC_STATE_INACTIVE == gp_group_state && + 0 < mali_scheduler_job_gp_count()) { + + enum mali_group_state state = + mali_group_activate(gp_group); + if (MALI_GROUP_STATE_ACTIVE == state) { + /* Set GP group state to idle */ + gp_group_state = EXEC_STATE_IDLE; + } else { + trigger_pm_update = MALI_TRUE; + } + } + + /* 2. Prepare as many physical groups as needed/possible */ + + num_physical_needed = mali_scheduler_job_physical_head_count(); + + /* On mali-450 platform, we don't need to enter in this block frequently. */ + if (0 < num_physical_needed) { + + if (num_physical_needed <= group_list_idle_count) { + /* We have enough groups on idle list already */ + num_physical_to_process = num_physical_needed; + num_physical_needed = 0; + } else { + /* We need to get a hold of some more groups */ + num_physical_to_process = group_list_idle_count; + num_physical_needed -= group_list_idle_count; + } + + if (0 < num_physical_needed) { + + /* 2.1. Activate groups which are inactive */ + + struct mali_group *group; + struct mali_group *temp; + + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_inactive, + struct mali_group, executor_list) { + enum mali_group_state state = + mali_group_activate(group); + if (MALI_GROUP_STATE_ACTIVE == state) { + /* Move from inactive to idle */ + mali_executor_change_state_pp_physical(group, + &group_list_inactive, + &group_list_inactive_count, + &group_list_idle, + &group_list_idle_count); + num_physical_to_process++; + } else { + trigger_pm_update = MALI_TRUE; + } + + num_physical_needed--; + if (0 == num_physical_needed) { + /* We have activated all the groups we need */ + break; + } + } + } + + if (mali_executor_virtual_group_is_usable()) { + + /* + * 2.2. And finally, steal and activate groups + * from virtual group if we need even more + */ + while (0 < num_physical_needed) { + struct mali_group *group; + + group = mali_group_acquire_group(virtual_group); + if (NULL != group) { + enum mali_group_state state; + + mali_executor_disable_empty_virtual(); + + state = mali_group_activate(group); + if (MALI_GROUP_STATE_ACTIVE == state) { + /* Group is ready, add to idle list */ + _mali_osk_list_add( + &group->executor_list, + &group_list_idle); + group_list_idle_count++; + num_physical_to_process++; + } else { + /* + * Group is not ready yet, + * add to inactive list + */ + _mali_osk_list_add( + &group->executor_list, + &group_list_inactive); + group_list_inactive_count++; + + trigger_pm_update = MALI_TRUE; + } + num_physical_needed--; + } else { + /* + * We could not get enough groups + * from the virtual group. + */ + break; + } + } + } + + /* 2.3. Assign physical jobs to groups */ + + if (0 < num_physical_to_process) { + struct mali_group *group; + struct mali_group *temp; + + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_idle, + struct mali_group, executor_list) { + struct mali_pp_job *job = NULL; + u32 sub_job = MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS; + + MALI_DEBUG_ASSERT(num_jobs_to_start < + MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS); + + MALI_DEBUG_ASSERT(0 < + mali_scheduler_job_physical_head_count()); + + if (mali_executor_hint_is_enabled( + MALI_EXECUTOR_HINT_GP_BOUND)) { + if (MALI_TRUE == mali_executor_tackle_gp_bound()) { + /* + * We're gp bound, + * don't start this right now. + */ + deactivate_idle_group = MALI_FALSE; + num_physical_to_process = 0; + break; + } + } + + job = mali_scheduler_job_pp_physical_get( + &sub_job); + + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT(sub_job <= MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS); + + /* Put job + group on list of jobs to start later on */ + + groups_to_start[num_jobs_to_start] = group; + jobs_to_start[num_jobs_to_start] = job; + sub_jobs_to_start[num_jobs_to_start] = sub_job; + num_jobs_to_start++; + + /* Move group from idle to working */ + mali_executor_change_state_pp_physical(group, + &group_list_idle, + &group_list_idle_count, + &group_list_working, + &group_list_working_count); + + num_physical_to_process--; + if (0 == num_physical_to_process) { + /* Got all we needed */ + break; + } + } + } + } + + + /* 3. Deactivate idle pp group , must put deactive here before active vitual group + * for cover case first only has physical job in normal queue but group inactive, + * so delay the job start go to active group, when group activated, + * call scheduler again, but now if we get high queue virtual job, + * we will do nothing in schedule cause executor schedule stop + */ + + if (MALI_TRUE == mali_executor_deactivate_list_idle(deactivate_idle_group + && (!mali_timeline_has_physical_pp_job()))) { + trigger_pm_update = MALI_TRUE; + } + + /* 4. Activate virtual group, if needed */ + + if (EXEC_STATE_INACTIVE == virtual_group_state && + 0 < mali_scheduler_job_next_is_virtual()) { + enum mali_group_state state = + mali_group_activate(virtual_group); + if (MALI_GROUP_STATE_ACTIVE == state) { + /* Set virtual group state to idle */ + virtual_group_state = EXEC_STATE_IDLE; + } else { + trigger_pm_update = MALI_TRUE; + } + } + + /* 5. To power up group asap, we trigger pm update here. */ + + if (MALI_TRUE == trigger_pm_update) { + trigger_pm_update = MALI_FALSE; + mali_pm_update_async(); + } + + /* 6. Assign jobs to idle virtual group (or deactivate if no job) */ + + if (EXEC_STATE_IDLE == virtual_group_state) { + if (0 < mali_scheduler_job_next_is_virtual()) { + virtual_job_to_start = + mali_scheduler_job_pp_virtual_get(); + virtual_group_state = EXEC_STATE_WORKING; + } else if (!mali_timeline_has_virtual_pp_job()) { + virtual_group_state = EXEC_STATE_INACTIVE; + + if (mali_group_deactivate(virtual_group)) { + trigger_pm_update = MALI_TRUE; + } + } + } + + /* 7. Assign job to idle GP group (or deactivate if no job) */ + + if (EXEC_STATE_IDLE == gp_group_state) { + if (0 < mali_scheduler_job_gp_count()) { + gp_job_to_start = mali_scheduler_job_gp_get(); + gp_group_state = EXEC_STATE_WORKING; + } else if (!mali_timeline_has_gp_job()) { + gp_group_state = EXEC_STATE_INACTIVE; + if (mali_group_deactivate(gp_group)) { + trigger_pm_update = MALI_TRUE; + } + } + } + + /* 8. We no longer need the schedule/queue lock */ + + mali_scheduler_unlock(); + + /* 9. start jobs */ + + if (NULL != virtual_job_to_start) { + MALI_DEBUG_ASSERT(!mali_group_pp_is_active(virtual_group)); + mali_group_start_pp_job(virtual_group, + virtual_job_to_start, 0); + } + + for (i = 0; i < num_jobs_to_start; i++) { + MALI_DEBUG_ASSERT(!mali_group_pp_is_active( + groups_to_start[i])); + mali_group_start_pp_job(groups_to_start[i], + jobs_to_start[i], + sub_jobs_to_start[i]); + } + + MALI_DEBUG_ASSERT_POINTER(gp_group); + + if (NULL != gp_job_to_start) { + MALI_DEBUG_ASSERT(!mali_group_gp_is_active(gp_group)); + mali_group_start_gp_job(gp_group, gp_job_to_start); + } + + /* 10. Trigger any pending PM updates */ + if (MALI_TRUE == trigger_pm_update) { + mali_pm_update_async(); + } +} + +/* Handler for deferred schedule requests */ +static void mali_executor_wq_schedule(void *arg) +{ + MALI_IGNORE(arg); + mali_executor_lock(); + mali_executor_schedule(); + mali_executor_unlock(); +} + +static void mali_executor_send_gp_oom_to_user(struct mali_gp_job *job, u32 added_size) +{ + _mali_uk_gp_job_suspended_s *jobres; + _mali_osk_notification_t *notification; + + notification = mali_gp_job_get_oom_notification(job); + + /* + * Remember the id we send to user space, so we have something to + * verify when we get a response + */ + gp_returned_cookie = mali_gp_job_get_id(job); + + jobres = (_mali_uk_gp_job_suspended_s *)notification->result_buffer; + jobres->user_job_ptr = mali_gp_job_get_user_id(job); + jobres->cookie = gp_returned_cookie; + jobres->heap_added_size = added_size; + mali_session_send_notification(mali_gp_job_get_session(job), + notification); +} +static struct mali_gp_job *mali_executor_complete_gp(struct mali_group *group, + mali_bool success) +{ + struct mali_gp_job *job; + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + /* Extracts the needed HW status from core and reset */ + job = mali_group_complete_gp(group, success); + + MALI_DEBUG_ASSERT_POINTER(job); + + /* Core is now ready to go into idle list */ + gp_group_state = EXEC_STATE_IDLE; + + /* This will potentially queue more GP and PP jobs */ + mali_timeline_tracker_release(&job->tracker); + + /* Signal PP job */ + mali_gp_job_signal_pp_tracker(job, success); + + return job; +} + +static struct mali_pp_job *mali_executor_complete_pp(struct mali_group *group, + mali_bool success) +{ + struct mali_pp_job *job; + u32 sub_job; + mali_bool job_is_done; + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + /* Extracts the needed HW status from core and reset */ + job = mali_group_complete_pp(group, success, &sub_job); + + MALI_DEBUG_ASSERT_POINTER(job); + + /* Core is now ready to go into idle list */ + if (mali_group_is_virtual(group)) { + virtual_group_state = EXEC_STATE_IDLE; + } else { + /* Move from working to idle state */ + mali_executor_change_state_pp_physical(group, + &group_list_working, + &group_list_working_count, + &group_list_idle, + &group_list_idle_count); + } + + /* It is the executor module which owns the jobs themselves by now */ + mali_pp_job_mark_sub_job_completed(job, success); + job_is_done = mali_pp_job_is_complete(job); + + if (job_is_done) { + /* This will potentially queue more GP and PP jobs */ + mali_timeline_tracker_release(&job->tracker); + } + + return job; +} + +static void mali_executor_complete_group(struct mali_group *group, + mali_bool success, + struct mali_gp_job **gp_job_done, + struct mali_pp_job **pp_job_done) +{ + struct mali_gp_core *gp_core = mali_group_get_gp_core(group); + struct mali_pp_core *pp_core = mali_group_get_pp_core(group); + struct mali_gp_job *gp_job = NULL; + struct mali_pp_job *pp_job = NULL; + mali_bool pp_job_is_done = MALI_TRUE; + + if (NULL != gp_core) { + gp_job = mali_executor_complete_gp(group, success); + } else { + MALI_DEBUG_ASSERT_POINTER(pp_core); + MALI_IGNORE(pp_core); + pp_job = mali_executor_complete_pp(group, success); + + pp_job_is_done = mali_pp_job_is_complete(pp_job); + } + + if (pause_count > 0) { + /* Execution has been suspended */ + + if (!mali_executor_is_working()) { + /* Last job completed, wake up sleepers */ + _mali_osk_wait_queue_wake_up( + executor_working_wait_queue); + } + } else if (MALI_TRUE == mali_group_disable_requested(group)) { + mali_executor_core_scale_in_group_complete(group); + + mali_executor_schedule(); + } else { + /* try to schedule new jobs */ + mali_executor_schedule(); + } + + if (NULL != gp_job) { + MALI_DEBUG_ASSERT_POINTER(gp_job_done); + *gp_job_done = gp_job; + } else if (pp_job_is_done) { + MALI_DEBUG_ASSERT_POINTER(pp_job); + MALI_DEBUG_ASSERT_POINTER(pp_job_done); + *pp_job_done = pp_job; + } +} + +static void mali_executor_change_state_pp_physical(struct mali_group *group, + _mali_osk_list_t *old_list, + u32 *old_count, + _mali_osk_list_t *new_list, + u32 *new_count) +{ + /* + * It's a bit more complicated to change the state for the physical PP + * groups since their state is determined by the list they are on. + */ +#if defined(DEBUG) + mali_bool found = MALI_FALSE; + struct mali_group *group_iter; + struct mali_group *temp; + u32 old_counted = 0; + u32 new_counted = 0; + + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(old_list); + MALI_DEBUG_ASSERT_POINTER(old_count); + MALI_DEBUG_ASSERT_POINTER(new_list); + MALI_DEBUG_ASSERT_POINTER(new_count); + + /* + * Verify that group is present on old list, + * and that the count is correct + */ + + _MALI_OSK_LIST_FOREACHENTRY(group_iter, temp, old_list, + struct mali_group, executor_list) { + old_counted++; + if (group == group_iter) { + found = MALI_TRUE; + } + } + + _MALI_OSK_LIST_FOREACHENTRY(group_iter, temp, new_list, + struct mali_group, executor_list) { + new_counted++; + } + + if (MALI_FALSE == found) { + if (old_list == &group_list_idle) { + MALI_DEBUG_PRINT(1, (" old Group list is idle,")); + } else if (old_list == &group_list_inactive) { + MALI_DEBUG_PRINT(1, (" old Group list is inactive,")); + } else if (old_list == &group_list_working) { + MALI_DEBUG_PRINT(1, (" old Group list is working,")); + } else if (old_list == &group_list_disabled) { + MALI_DEBUG_PRINT(1, (" old Group list is disable,")); + } + + if (MALI_TRUE == mali_executor_group_is_in_state(group, EXEC_STATE_WORKING)) { + MALI_DEBUG_PRINT(1, (" group in working \n")); + } else if (MALI_TRUE == mali_executor_group_is_in_state(group, EXEC_STATE_INACTIVE)) { + MALI_DEBUG_PRINT(1, (" group in inactive \n")); + } else if (MALI_TRUE == mali_executor_group_is_in_state(group, EXEC_STATE_IDLE)) { + MALI_DEBUG_PRINT(1, (" group in idle \n")); + } else if (MALI_TRUE == mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED)) { + MALI_DEBUG_PRINT(1, (" but group in disabled \n")); + } + } + + MALI_DEBUG_ASSERT(MALI_TRUE == found); + MALI_DEBUG_ASSERT(0 < (*old_count)); + MALI_DEBUG_ASSERT((*old_count) == old_counted); + MALI_DEBUG_ASSERT((*new_count) == new_counted); +#endif + + _mali_osk_list_move(&group->executor_list, new_list); + (*old_count)--; + (*new_count)++; +} + +static void mali_executor_set_state_pp_physical(struct mali_group *group, + _mali_osk_list_t *new_list, + u32 *new_count) +{ + _mali_osk_list_add(&group->executor_list, new_list); + (*new_count)++; +} + +static mali_bool mali_executor_group_is_in_state(struct mali_group *group, + enum mali_executor_state_t state) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + if (gp_group == group) { + if (gp_group_state == state) { + return MALI_TRUE; + } + } else if (virtual_group == group || mali_group_is_in_virtual(group)) { + if (virtual_group_state == state) { + return MALI_TRUE; + } + } else { + /* Physical PP group */ + struct mali_group *group_iter; + struct mali_group *temp; + _mali_osk_list_t *list; + + if (EXEC_STATE_DISABLED == state) { + list = &group_list_disabled; + } else if (EXEC_STATE_INACTIVE == state) { + list = &group_list_inactive; + } else if (EXEC_STATE_IDLE == state) { + list = &group_list_idle; + } else { + MALI_DEBUG_ASSERT(EXEC_STATE_WORKING == state); + list = &group_list_working; + } + + _MALI_OSK_LIST_FOREACHENTRY(group_iter, temp, list, + struct mali_group, executor_list) { + if (group_iter == group) { + return MALI_TRUE; + } + } + } + + /* group not in correct state */ + return MALI_FALSE; +} + +static void mali_executor_group_enable_internal(struct mali_group *group) +{ + MALI_DEBUG_ASSERT(group); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + MALI_DEBUG_ASSERT(mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED)); + + /* Put into inactive state (== "lowest" enabled state) */ + if (group == gp_group) { + MALI_DEBUG_ASSERT(EXEC_STATE_DISABLED == gp_group_state); + gp_group_state = EXEC_STATE_INACTIVE; + } else { + mali_executor_change_state_pp_physical(group, + &group_list_disabled, + &group_list_disabled_count, + &group_list_inactive, + &group_list_inactive_count); + + ++num_physical_pp_cores_enabled; + MALI_DEBUG_PRINT(4, ("Enabling group id %d \n", group->pp_core->core_id)); + } + + if (MALI_GROUP_STATE_ACTIVE == mali_group_activate(group)) { + MALI_DEBUG_ASSERT(MALI_TRUE == mali_group_power_is_on(group)); + + /* Move from inactive to idle */ + if (group == gp_group) { + gp_group_state = EXEC_STATE_IDLE; + } else { + mali_executor_change_state_pp_physical(group, + &group_list_inactive, + &group_list_inactive_count, + &group_list_idle, + &group_list_idle_count); + + if (mali_executor_has_virtual_group()) { + if (mali_executor_physical_rejoin_virtual(group)) { + mali_pm_update_async(); + } + } + } + } else { + mali_pm_update_async(); + } +} + +static void mali_executor_group_disable_internal(struct mali_group *group) +{ + mali_bool working; + + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + MALI_DEBUG_ASSERT(!mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED)); + + working = mali_executor_group_is_in_state(group, EXEC_STATE_WORKING); + if (MALI_TRUE == working) { + /** Group to be disabled once it completes current work, + * when virtual group completes, also check child groups for this flag */ + mali_group_set_disable_request(group, MALI_TRUE); + return; + } + + /* Put into disabled state */ + if (group == gp_group) { + /* GP group */ + MALI_DEBUG_ASSERT(EXEC_STATE_WORKING != gp_group_state); + gp_group_state = EXEC_STATE_DISABLED; + } else { + if (mali_group_is_in_virtual(group)) { + /* A child group of virtual group. move the specific group from virtual group */ + MALI_DEBUG_ASSERT(EXEC_STATE_WORKING != virtual_group_state); + + mali_executor_set_state_pp_physical(group, + &group_list_disabled, + &group_list_disabled_count); + + mali_group_remove_group(virtual_group, group); + mali_executor_disable_empty_virtual(); + } else { + mali_executor_change_group_status_disabled(group); + } + + --num_physical_pp_cores_enabled; + MALI_DEBUG_PRINT(4, ("Disabling group id %d \n", group->pp_core->core_id)); + } + + if (MALI_GROUP_STATE_INACTIVE != group->state) { + if (MALI_TRUE == mali_group_deactivate(group)) { + mali_pm_update_async(); + } + } +} + +static void mali_executor_notify_core_change(u32 num_cores) +{ + mali_bool done = MALI_FALSE; + + if (mali_is_mali450() || mali_is_mali470()) { + return; + } + + /* + * This function gets a bit complicated because we can't hold the session lock while + * allocating notification objects. + */ + while (!done) { + u32 i; + u32 num_sessions_alloc; + u32 num_sessions_with_lock; + u32 used_notification_objects = 0; + _mali_osk_notification_t **notobjs; + + /* Pre allocate the number of notifications objects we need right now (might change after lock has been taken) */ + num_sessions_alloc = mali_session_get_count(); + if (0 == num_sessions_alloc) { + /* No sessions to report to */ + return; + } + + notobjs = (_mali_osk_notification_t **)_mali_osk_malloc(sizeof(_mali_osk_notification_t *) * num_sessions_alloc); + if (NULL == notobjs) { + MALI_PRINT_ERROR(("Failed to notify user space session about num PP core change (alloc failure)\n")); + /* there is probably no point in trying again, system must be really low on memory and probably unusable now anyway */ + return; + } + + for (i = 0; i < num_sessions_alloc; i++) { + notobjs[i] = _mali_osk_notification_create(_MALI_NOTIFICATION_PP_NUM_CORE_CHANGE, sizeof(_mali_uk_pp_num_cores_changed_s)); + if (NULL != notobjs[i]) { + _mali_uk_pp_num_cores_changed_s *data = notobjs[i]->result_buffer; + data->number_of_enabled_cores = num_cores; + } else { + MALI_PRINT_ERROR(("Failed to notify user space session about num PP core change (alloc failure %u)\n", i)); + } + } + + mali_session_lock(); + + /* number of sessions will not change while we hold the lock */ + num_sessions_with_lock = mali_session_get_count(); + + if (num_sessions_alloc >= num_sessions_with_lock) { + /* We have allocated enough notification objects for all the sessions atm */ + struct mali_session_data *session, *tmp; + MALI_SESSION_FOREACH(session, tmp, link) { + MALI_DEBUG_ASSERT(used_notification_objects < num_sessions_alloc); + if (NULL != notobjs[used_notification_objects]) { + mali_session_send_notification(session, notobjs[used_notification_objects]); + notobjs[used_notification_objects] = NULL; /* Don't track this notification object any more */ + } + used_notification_objects++; + } + done = MALI_TRUE; + } + + mali_session_unlock(); + + /* Delete any remaining/unused notification objects */ + for (; used_notification_objects < num_sessions_alloc; used_notification_objects++) { + if (NULL != notobjs[used_notification_objects]) { + _mali_osk_notification_delete(notobjs[used_notification_objects]); + } + } + + _mali_osk_free(notobjs); + } +} + +static mali_bool mali_executor_core_scaling_is_done(void *data) +{ + u32 i; + u32 num_groups; + mali_bool ret = MALI_TRUE; + + MALI_IGNORE(data); + + mali_executor_lock(); + + num_groups = mali_group_get_glob_num_groups(); + + for (i = 0; i < num_groups; i++) { + struct mali_group *group = mali_group_get_glob_group(i); + + if (NULL != group) { + if (MALI_TRUE == group->disable_requested && NULL != mali_group_get_pp_core(group)) { + ret = MALI_FALSE; + break; + } + } + } + mali_executor_unlock(); + + return ret; +} + +static void mali_executor_wq_notify_core_change(void *arg) +{ + MALI_IGNORE(arg); + + if (mali_is_mali450() || mali_is_mali470()) { + return; + } + + _mali_osk_wait_queue_wait_event(executor_notify_core_change_wait_queue, + mali_executor_core_scaling_is_done, NULL); + + mali_executor_notify_core_change(num_physical_pp_cores_enabled); +} + +/** + * Clear all disable request from the _last_ core scaling behavior. + */ +static void mali_executor_core_scaling_reset(void) +{ + u32 i; + u32 num_groups; + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + num_groups = mali_group_get_glob_num_groups(); + + for (i = 0; i < num_groups; i++) { + struct mali_group *group = mali_group_get_glob_group(i); + + if (NULL != group) { + group->disable_requested = MALI_FALSE; + } + } + + for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) { + core_scaling_delay_up_mask[i] = 0; + } +} + +static void mali_executor_core_scale(unsigned int target_core_nr) +{ + int current_core_scaling_mask[MALI_MAX_NUMBER_OF_DOMAINS] = { 0 }; + int target_core_scaling_mask[MALI_MAX_NUMBER_OF_DOMAINS] = { 0 }; + int i; + + MALI_DEBUG_ASSERT(0 < target_core_nr); + MALI_DEBUG_ASSERT(num_physical_pp_cores_total >= target_core_nr); + + mali_executor_lock(); + + if (target_core_nr < num_physical_pp_cores_enabled) { + MALI_DEBUG_PRINT(2, ("Requesting %d cores: disabling %d cores\n", target_core_nr, num_physical_pp_cores_enabled - target_core_nr)); + } else { + MALI_DEBUG_PRINT(2, ("Requesting %d cores: enabling %d cores\n", target_core_nr, target_core_nr - num_physical_pp_cores_enabled)); + } + + /* When a new core scaling request is comming, we should remove the un-doing + * part of the last core scaling request. It's safe because we have only one + * lock(executor lock) protection. */ + mali_executor_core_scaling_reset(); + + mali_pm_get_best_power_cost_mask(num_physical_pp_cores_enabled, current_core_scaling_mask); + mali_pm_get_best_power_cost_mask(target_core_nr, target_core_scaling_mask); + + for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) { + target_core_scaling_mask[i] = target_core_scaling_mask[i] - current_core_scaling_mask[i]; + MALI_DEBUG_PRINT(5, ("target_core_scaling_mask[%d] = %d\n", i, target_core_scaling_mask[i])); + } + + for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) { + if (0 > target_core_scaling_mask[i]) { + struct mali_pm_domain *domain; + + domain = mali_pm_domain_get_from_index(i); + + /* Domain is valid and has pp cores */ + if ((NULL != domain) && !(_mali_osk_list_empty(&domain->group_list))) { + struct mali_group *group; + struct mali_group *temp; + + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &domain->group_list, struct mali_group, pm_domain_list) { + if (NULL != mali_group_get_pp_core(group) && (!mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED)) + && (!mali_group_is_virtual(group))) { + mali_executor_group_disable_internal(group); + target_core_scaling_mask[i]++; + if ((0 == target_core_scaling_mask[i])) { + break; + } + + } + } + } + } + } + + for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) { + /** + * Target_core_scaling_mask[i] is bigger than 0, + * means we need to enable some pp cores in + * this domain whose domain index is i. + */ + if (0 < target_core_scaling_mask[i]) { + struct mali_pm_domain *domain; + + if (num_physical_pp_cores_enabled >= target_core_nr) { + break; + } + + domain = mali_pm_domain_get_from_index(i); + + /* Domain is valid and has pp cores */ + if ((NULL != domain) && !(_mali_osk_list_empty(&domain->group_list))) { + struct mali_group *group; + struct mali_group *temp; + + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &domain->group_list, struct mali_group, pm_domain_list) { + if (NULL != mali_group_get_pp_core(group) && mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED) + && (!mali_group_is_virtual(group))) { + mali_executor_group_enable_internal(group); + target_core_scaling_mask[i]--; + + if ((0 == target_core_scaling_mask[i]) || num_physical_pp_cores_enabled == target_core_nr) { + break; + } + } + } + } + } + } + + /** + * Here, we may still have some pp cores not been enabled because of some + * pp cores need to be disabled are still in working state. + */ + for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) { + if (0 < target_core_scaling_mask[i]) { + core_scaling_delay_up_mask[i] = target_core_scaling_mask[i]; + } + } + + mali_executor_schedule(); + mali_executor_unlock(); +} + +static void mali_executor_core_scale_in_group_complete(struct mali_group *group) +{ + int num_pp_cores_disabled = 0; + int num_pp_cores_to_enable = 0; + int i; + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + MALI_DEBUG_ASSERT(MALI_TRUE == mali_group_disable_requested(group)); + + /* Disable child group of virtual group */ + if (mali_group_is_virtual(group)) { + struct mali_group *child; + struct mali_group *temp; + + _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list) { + if (MALI_TRUE == mali_group_disable_requested(child)) { + mali_group_set_disable_request(child, MALI_FALSE); + mali_executor_group_disable_internal(child); + num_pp_cores_disabled++; + } + } + mali_group_set_disable_request(group, MALI_FALSE); + } else { + mali_executor_group_disable_internal(group); + mali_group_set_disable_request(group, MALI_FALSE); + if (NULL != mali_group_get_pp_core(group)) { + num_pp_cores_disabled++; + } + } + + num_pp_cores_to_enable = num_pp_cores_disabled; + + for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) { + if (0 < core_scaling_delay_up_mask[i]) { + struct mali_pm_domain *domain; + + if (0 == num_pp_cores_to_enable) { + break; + } + + domain = mali_pm_domain_get_from_index(i); + + /* Domain is valid and has pp cores */ + if ((NULL != domain) && !(_mali_osk_list_empty(&domain->group_list))) { + struct mali_group *disabled_group; + struct mali_group *temp; + + _MALI_OSK_LIST_FOREACHENTRY(disabled_group, temp, &domain->group_list, struct mali_group, pm_domain_list) { + if (NULL != mali_group_get_pp_core(disabled_group) && mali_executor_group_is_in_state(disabled_group, EXEC_STATE_DISABLED)) { + mali_executor_group_enable_internal(disabled_group); + core_scaling_delay_up_mask[i]--; + num_pp_cores_to_enable--; + + if ((0 == core_scaling_delay_up_mask[i]) || 0 == num_pp_cores_to_enable) { + break; + } + } + } + } + } + } + + _mali_osk_wait_queue_wake_up(executor_notify_core_change_wait_queue); +} + +static void mali_executor_change_group_status_disabled(struct mali_group *group) +{ + /* Physical PP group */ + mali_bool idle; + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + idle = mali_executor_group_is_in_state(group, EXEC_STATE_IDLE); + if (MALI_TRUE == idle) { + mali_executor_change_state_pp_physical(group, + &group_list_idle, + &group_list_idle_count, + &group_list_disabled, + &group_list_disabled_count); + } else { + mali_executor_change_state_pp_physical(group, + &group_list_inactive, + &group_list_inactive_count, + &group_list_disabled, + &group_list_disabled_count); + } +} + +static mali_bool mali_executor_deactivate_list_idle(mali_bool deactivate_idle_group) +{ + mali_bool trigger_pm_update = MALI_FALSE; + + if (group_list_idle_count > 0) { + if (mali_executor_has_virtual_group()) { + + /* Rejoin virtual group on Mali-450 */ + + struct mali_group *group; + struct mali_group *temp; + + _MALI_OSK_LIST_FOREACHENTRY(group, temp, + &group_list_idle, + struct mali_group, executor_list) { + if (mali_executor_physical_rejoin_virtual( + group)) { + trigger_pm_update = MALI_TRUE; + } + } + } else if (deactivate_idle_group) { + struct mali_group *group; + struct mali_group *temp; + + /* Deactivate group on Mali-300/400 */ + + _MALI_OSK_LIST_FOREACHENTRY(group, temp, + &group_list_idle, + struct mali_group, executor_list) { + if (mali_group_deactivate(group)) { + trigger_pm_update = MALI_TRUE; + } + + /* Move from idle to inactive */ + mali_executor_change_state_pp_physical(group, + &group_list_idle, + &group_list_idle_count, + &group_list_inactive, + &group_list_inactive_count); + } + } + } + + return trigger_pm_update; +} + +void mali_executor_running_status_print(void) +{ + struct mali_group *group = NULL; + struct mali_group *temp = NULL; + + MALI_PRINT(("GP running job: %p\n", gp_group->gp_running_job)); + if ((gp_group->gp_core) && (gp_group->is_working)) { + mali_group_dump_status(gp_group); + } + MALI_PRINT(("Physical PP groups in WORKING state (count = %u):\n", group_list_working_count)); + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_working, struct mali_group, executor_list) { + MALI_PRINT(("PP running job: %p, subjob %d \n", group->pp_running_job, group->pp_running_sub_job)); + mali_group_dump_status(group); + } + MALI_PRINT(("Physical PP groups in INACTIVE state (count = %u):\n", group_list_inactive_count)); + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_inactive, struct mali_group, executor_list) { + MALI_PRINT(("\tPP status %d, SW power: %s\n", group->state, group->power_is_on ? "On" : "Off")); + MALI_PRINT(("\tPP #%d: %s\n", group->pp_core->core_id, group->pp_core->hw_core.description)); + } + MALI_PRINT(("Physical PP groups in IDLE state (count = %u):\n", group_list_idle_count)); + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_idle, struct mali_group, executor_list) { + MALI_PRINT(("\tPP status %d, SW power: %s\n", group->state, group->power_is_on ? "On" : "Off")); + MALI_PRINT(("\tPP #%d: %s\n", group->pp_core->core_id, group->pp_core->hw_core.description)); + } + MALI_PRINT(("Physical PP groups in DISABLED state (count = %u):\n", group_list_disabled_count)); + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_disabled, struct mali_group, executor_list) { + MALI_PRINT(("\tPP status %d, SW power: %s\n", group->state, group->power_is_on ? "On" : "Off")); + MALI_PRINT(("\tPP #%d: %s\n", group->pp_core->core_id, group->pp_core->hw_core.description)); + } + + if (mali_executor_has_virtual_group()) { + MALI_PRINT(("Virtual group running job: %p\n", virtual_group->pp_running_job)); + MALI_PRINT(("Virtual group status: %d\n", virtual_group_state)); + MALI_PRINT(("Virtual group->status: %d\n", virtual_group->state)); + MALI_PRINT(("\tSW power: %s\n", virtual_group->power_is_on ? "On" : "Off")); + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &virtual_group->group_list, + struct mali_group, group_list) { + int i = 0; + MALI_PRINT(("\tchild group(%s) running job: %p\n", group->pp_core->hw_core.description, group->pp_running_job)); + MALI_PRINT(("\tchild group(%s)->status: %d\n", group->pp_core->hw_core.description, group->state)); + MALI_PRINT(("\tchild group(%s) SW power: %s\n", group->pp_core->hw_core.description, group->power_is_on ? "On" : "Off")); + if (group->pm_domain) { + MALI_PRINT(("\tPower domain: id %u\n", mali_pm_domain_get_id(group->pm_domain))); + MALI_PRINT(("\tMask:0x%04x \n", mali_pm_domain_get_mask(group->pm_domain))); + MALI_PRINT(("\tUse-count:%u \n", mali_pm_domain_get_use_count(group->pm_domain))); + MALI_PRINT(("\tCurrent power status:%s \n", (mali_pm_domain_get_mask(group->pm_domain)& mali_pm_get_current_mask()) ? "On" : "Off")); + MALI_PRINT(("\tWanted power status:%s \n", (mali_pm_domain_get_mask(group->pm_domain)& mali_pm_get_wanted_mask()) ? "On" : "Off")); + } + + for (i = 0; i < 2; i++) { + if (NULL != group->l2_cache_core[i]) { + struct mali_pm_domain *domain; + domain = mali_l2_cache_get_pm_domain(group->l2_cache_core[i]); + MALI_PRINT(("\t L2(index %d) group SW power: %s\n", i, group->l2_cache_core[i]->power_is_on ? "On" : "Off")); + if (domain) { + MALI_PRINT(("\tL2 Power domain: id %u\n", mali_pm_domain_get_id(domain))); + MALI_PRINT(("\tL2 Mask:0x%04x \n", mali_pm_domain_get_mask(domain))); + MALI_PRINT(("\tL2 Use-count:%u \n", mali_pm_domain_get_use_count(domain))); + MALI_PRINT(("\tL2 Current power status:%s \n", (mali_pm_domain_get_mask(domain) & mali_pm_get_current_mask()) ? "On" : "Off")); + MALI_PRINT(("\tL2 Wanted power status:%s \n", (mali_pm_domain_get_mask(domain) & mali_pm_get_wanted_mask()) ? "On" : "Off")); + } + } + } + } + if (EXEC_STATE_WORKING == virtual_group_state) { + mali_group_dump_status(virtual_group); + } + } +} + +void mali_executor_status_dump(void) +{ + mali_executor_lock(); + mali_scheduler_lock(); + + /* print schedule queue status */ + mali_scheduler_gp_pp_job_queue_print(); + + mali_scheduler_unlock(); + mali_executor_unlock(); +} diff --git a/drivers/gpu/arm/utgard/common/mali_executor.h b/drivers/gpu/arm/utgard/common/mali_executor.h new file mode 100644 index 00000000000000..a756d3f40c2abd --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_executor.h @@ -0,0 +1,104 @@ +/* + * Copyright (C) 2012, 2014-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_EXECUTOR_H__ +#define __MALI_EXECUTOR_H__ + +#include "mali_osk.h" +#include "mali_scheduler_types.h" +#include "mali_kernel_common.h" + +typedef enum { + MALI_EXECUTOR_HINT_GP_BOUND = 0 +#define MALI_EXECUTOR_HINT_MAX 1 +} mali_executor_hint; + +extern mali_bool mali_executor_hints[MALI_EXECUTOR_HINT_MAX]; + +/* forward declare struct instead of using include */ +struct mali_session_data; +struct mali_group; +struct mali_pp_core; + +extern _mali_osk_spinlock_irq_t *mali_executor_lock_obj; + +#define MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD() MALI_DEBUG_ASSERT_LOCK_HELD(mali_executor_lock_obj); + +_mali_osk_errcode_t mali_executor_initialize(void); +void mali_executor_terminate(void); + +void mali_executor_populate(void); +void mali_executor_depopulate(void); + +void mali_executor_suspend(void); +void mali_executor_resume(void); + +u32 mali_executor_get_num_cores_total(void); +u32 mali_executor_get_num_cores_enabled(void); +struct mali_pp_core *mali_executor_get_virtual_pp(void); +struct mali_group *mali_executor_get_virtual_group(void); + +void mali_executor_zap_all_active(struct mali_session_data *session); + +/** + * Schedule GP and PP according to bitmask. + * + * @param mask A scheduling bitmask. + * @param deferred_schedule MALI_TRUE if schedule should be deferred, MALI_FALSE if not. + */ +void mali_executor_schedule_from_mask(mali_scheduler_mask mask, mali_bool deferred_schedule); + +_mali_osk_errcode_t mali_executor_interrupt_gp(struct mali_group *group, mali_bool in_upper_half); +_mali_osk_errcode_t mali_executor_interrupt_pp(struct mali_group *group, mali_bool in_upper_half); +_mali_osk_errcode_t mali_executor_interrupt_mmu(struct mali_group *group, mali_bool in_upper_half); + +void mali_executor_group_oom(struct mali_group *group); +void mali_executor_group_power_up(struct mali_group *groups[], u32 num_groups); +void mali_executor_group_power_down(struct mali_group *groups[], u32 num_groups); + +void mali_executor_abort_session(struct mali_session_data *session); + +void mali_executor_core_scaling_enable(void); +void mali_executor_core_scaling_disable(void); +mali_bool mali_executor_core_scaling_is_enabled(void); + +void mali_executor_group_enable(struct mali_group *group); +void mali_executor_group_disable(struct mali_group *group); +mali_bool mali_executor_group_is_disabled(struct mali_group *group); + +int mali_executor_set_perf_level(unsigned int target_core_nr, mali_bool override); + +#if MALI_STATE_TRACKING +u32 mali_executor_dump_state(char *buf, u32 size); +#endif + +MALI_STATIC_INLINE void mali_executor_hint_enable(mali_executor_hint hint) +{ + MALI_DEBUG_ASSERT(hint < MALI_EXECUTOR_HINT_MAX); + mali_executor_hints[hint] = MALI_TRUE; +} + +MALI_STATIC_INLINE void mali_executor_hint_disable(mali_executor_hint hint) +{ + MALI_DEBUG_ASSERT(hint < MALI_EXECUTOR_HINT_MAX); + mali_executor_hints[hint] = MALI_FALSE; +} + +MALI_STATIC_INLINE mali_bool mali_executor_hint_is_enabled(mali_executor_hint hint) +{ + MALI_DEBUG_ASSERT(hint < MALI_EXECUTOR_HINT_MAX); + return mali_executor_hints[hint]; +} + +void mali_executor_running_status_print(void); +void mali_executor_status_dump(void); +void mali_executor_lock(void); +void mali_executor_unlock(void); +#endif /* __MALI_EXECUTOR_H__ */ diff --git a/drivers/gpu/arm/utgard/common/mali_gp.c b/drivers/gpu/arm/utgard/common/mali_gp.c new file mode 100644 index 00000000000000..a690781837e98f --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_gp.c @@ -0,0 +1,357 @@ +/* + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_gp.h" +#include "mali_hw_core.h" +#include "mali_group.h" +#include "mali_osk.h" +#include "regs/mali_gp_regs.h" +#include "mali_kernel_common.h" +#include "mali_kernel_core.h" +#if defined(CONFIG_MALI400_PROFILING) +#include "mali_osk_profiling.h" +#endif + +static struct mali_gp_core *mali_global_gp_core = NULL; + +/* Interrupt handlers */ +static void mali_gp_irq_probe_trigger(void *data); +static _mali_osk_errcode_t mali_gp_irq_probe_ack(void *data); + +struct mali_gp_core *mali_gp_create(const _mali_osk_resource_t *resource, struct mali_group *group) +{ + struct mali_gp_core *core = NULL; + + MALI_DEBUG_ASSERT(NULL == mali_global_gp_core); + MALI_DEBUG_PRINT(2, ("Mali GP: Creating Mali GP core: %s\n", resource->description)); + + core = _mali_osk_malloc(sizeof(struct mali_gp_core)); + if (NULL != core) { + if (_MALI_OSK_ERR_OK == mali_hw_core_create(&core->hw_core, resource, MALIGP2_REGISTER_ADDRESS_SPACE_SIZE)) { + _mali_osk_errcode_t ret; + + ret = mali_gp_reset(core); + + if (_MALI_OSK_ERR_OK == ret) { + ret = mali_group_add_gp_core(group, core); + if (_MALI_OSK_ERR_OK == ret) { + /* Setup IRQ handlers (which will do IRQ probing if needed) */ + core->irq = _mali_osk_irq_init(resource->irq, + mali_group_upper_half_gp, + group, + mali_gp_irq_probe_trigger, + mali_gp_irq_probe_ack, + core, + resource->description); + if (NULL != core->irq) { + MALI_DEBUG_PRINT(4, ("Mali GP: set global gp core from 0x%08X to 0x%08X\n", mali_global_gp_core, core)); + mali_global_gp_core = core; + + return core; + } else { + MALI_PRINT_ERROR(("Mali GP: Failed to setup interrupt handlers for GP core %s\n", core->hw_core.description)); + } + mali_group_remove_gp_core(group); + } else { + MALI_PRINT_ERROR(("Mali GP: Failed to add core %s to group\n", core->hw_core.description)); + } + } + mali_hw_core_delete(&core->hw_core); + } + + _mali_osk_free(core); + } else { + MALI_PRINT_ERROR(("Failed to allocate memory for GP core\n")); + } + + return NULL; +} + +void mali_gp_delete(struct mali_gp_core *core) +{ + MALI_DEBUG_ASSERT_POINTER(core); + + _mali_osk_irq_term(core->irq); + mali_hw_core_delete(&core->hw_core); + mali_global_gp_core = NULL; + _mali_osk_free(core); +} + +void mali_gp_stop_bus(struct mali_gp_core *core) +{ + MALI_DEBUG_ASSERT_POINTER(core); + + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_CMD, MALIGP2_REG_VAL_CMD_STOP_BUS); +} + +_mali_osk_errcode_t mali_gp_stop_bus_wait(struct mali_gp_core *core) +{ + int i; + + MALI_DEBUG_ASSERT_POINTER(core); + + /* Send the stop bus command. */ + mali_gp_stop_bus(core); + + /* Wait for bus to be stopped */ + for (i = 0; i < MALI_REG_POLL_COUNT_SLOW; i++) { + if (mali_hw_core_register_read(&core->hw_core, MALIGP2_REG_ADDR_MGMT_STATUS) & MALIGP2_REG_VAL_STATUS_BUS_STOPPED) { + break; + } + } + + if (MALI_REG_POLL_COUNT_SLOW == i) { + MALI_PRINT_ERROR(("Mali GP: Failed to stop bus on %s\n", core->hw_core.description)); + return _MALI_OSK_ERR_FAULT; + } + return _MALI_OSK_ERR_OK; +} + +void mali_gp_hard_reset(struct mali_gp_core *core) +{ + const u32 reset_wait_target_register = MALIGP2_REG_ADDR_MGMT_PERF_CNT_0_LIMIT; + const u32 reset_invalid_value = 0xC0FFE000; + const u32 reset_check_value = 0xC01A0000; + const u32 reset_default_value = 0; + int i; + + MALI_DEBUG_ASSERT_POINTER(core); + MALI_DEBUG_PRINT(4, ("Mali GP: Hard reset of core %s\n", core->hw_core.description)); + + mali_hw_core_register_write(&core->hw_core, reset_wait_target_register, reset_invalid_value); + + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_CMD, MALIGP2_REG_VAL_CMD_RESET); + + for (i = 0; i < MALI_REG_POLL_COUNT_FAST; i++) { + mali_hw_core_register_write(&core->hw_core, reset_wait_target_register, reset_check_value); + if (reset_check_value == mali_hw_core_register_read(&core->hw_core, reset_wait_target_register)) { + break; + } + } + + if (MALI_REG_POLL_COUNT_FAST == i) { + MALI_PRINT_ERROR(("Mali GP: The hard reset loop didn't work, unable to recover\n")); + } + + mali_hw_core_register_write(&core->hw_core, reset_wait_target_register, reset_default_value); /* set it back to the default */ + /* Re-enable interrupts */ + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_CLEAR, MALIGP2_REG_VAL_IRQ_MASK_ALL); + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_MASK, MALIGP2_REG_VAL_IRQ_MASK_USED); + +} + +void mali_gp_reset_async(struct mali_gp_core *core) +{ + MALI_DEBUG_ASSERT_POINTER(core); + + MALI_DEBUG_PRINT(4, ("Mali GP: Reset of core %s\n", core->hw_core.description)); + + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_MASK, 0); /* disable the IRQs */ + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_CLEAR, MALI400GP_REG_VAL_IRQ_RESET_COMPLETED); + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_CMD, MALI400GP_REG_VAL_CMD_SOFT_RESET); + +} + +_mali_osk_errcode_t mali_gp_reset_wait(struct mali_gp_core *core) +{ + int i; + u32 rawstat = 0; + + MALI_DEBUG_ASSERT_POINTER(core); + + for (i = 0; i < MALI_REG_POLL_COUNT_FAST; i++) { + rawstat = mali_hw_core_register_read(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_RAWSTAT); + if (rawstat & MALI400GP_REG_VAL_IRQ_RESET_COMPLETED) { + break; + } + } + + if (i == MALI_REG_POLL_COUNT_FAST) { + MALI_PRINT_ERROR(("Mali GP: Failed to reset core %s, rawstat: 0x%08x\n", + core->hw_core.description, rawstat)); + return _MALI_OSK_ERR_FAULT; + } + + /* Re-enable interrupts */ + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_CLEAR, MALIGP2_REG_VAL_IRQ_MASK_ALL); + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_MASK, MALIGP2_REG_VAL_IRQ_MASK_USED); + + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t mali_gp_reset(struct mali_gp_core *core) +{ + mali_gp_reset_async(core); + return mali_gp_reset_wait(core); +} + +void mali_gp_job_start(struct mali_gp_core *core, struct mali_gp_job *job) +{ + u32 startcmd = 0; + u32 *frame_registers = mali_gp_job_get_frame_registers(job); + u32 counter_src0 = mali_gp_job_get_perf_counter_src0(job); + u32 counter_src1 = mali_gp_job_get_perf_counter_src1(job); + + MALI_DEBUG_ASSERT_POINTER(core); + + if (mali_gp_job_has_vs_job(job)) { + startcmd |= (u32) MALIGP2_REG_VAL_CMD_START_VS; + } + + if (mali_gp_job_has_plbu_job(job)) { + startcmd |= (u32) MALIGP2_REG_VAL_CMD_START_PLBU; + } + + MALI_DEBUG_ASSERT(0 != startcmd); + + mali_hw_core_register_write_array_relaxed(&core->hw_core, MALIGP2_REG_ADDR_MGMT_VSCL_START_ADDR, frame_registers, MALIGP2_NUM_REGS_FRAME); + + if (MALI_HW_CORE_NO_COUNTER != counter_src0) { + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_PERF_CNT_0_SRC, counter_src0); + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_PERF_CNT_0_ENABLE, MALIGP2_REG_VAL_PERF_CNT_ENABLE); + } + if (MALI_HW_CORE_NO_COUNTER != counter_src1) { + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_PERF_CNT_1_SRC, counter_src1); + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_PERF_CNT_1_ENABLE, MALIGP2_REG_VAL_PERF_CNT_ENABLE); + } + + MALI_DEBUG_PRINT(3, ("Mali GP: Starting job (0x%08x) on core %s with command 0x%08X\n", job, core->hw_core.description, startcmd)); + + mali_hw_core_register_write_relaxed(&core->hw_core, MALIGP2_REG_ADDR_MGMT_CMD, MALIGP2_REG_VAL_CMD_UPDATE_PLBU_ALLOC); + + /* Barrier to make sure the previous register write is finished */ + _mali_osk_write_mem_barrier(); + + /* This is the command that starts the core. + * + * Don't actually run the job if PROFILING_SKIP_PP_JOBS are set, just + * force core to assert the completion interrupt. + */ +#if !defined(PROFILING_SKIP_GP_JOBS) + mali_hw_core_register_write_relaxed(&core->hw_core, MALIGP2_REG_ADDR_MGMT_CMD, startcmd); +#else + { + u32 bits = 0; + + if (mali_gp_job_has_vs_job(job)) + bits = MALIGP2_REG_VAL_IRQ_VS_END_CMD_LST; + if (mali_gp_job_has_plbu_job(job)) + bits |= MALIGP2_REG_VAL_IRQ_PLBU_END_CMD_LST; + + mali_hw_core_register_write_relaxed(&core->hw_core, + MALIGP2_REG_ADDR_MGMT_INT_RAWSTAT, bits); + } +#endif + + /* Barrier to make sure the previous register write is finished */ + _mali_osk_write_mem_barrier(); +} + +void mali_gp_resume_with_new_heap(struct mali_gp_core *core, u32 start_addr, u32 end_addr) +{ + u32 irq_readout; + + MALI_DEBUG_ASSERT_POINTER(core); + + irq_readout = mali_hw_core_register_read(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_RAWSTAT); + + if (irq_readout & MALIGP2_REG_VAL_IRQ_PLBU_OUT_OF_MEM) { + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_CLEAR, (MALIGP2_REG_VAL_IRQ_PLBU_OUT_OF_MEM | MALIGP2_REG_VAL_IRQ_HANG)); + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_MASK, MALIGP2_REG_VAL_IRQ_MASK_USED); /* re-enable interrupts */ + mali_hw_core_register_write_relaxed(&core->hw_core, MALIGP2_REG_ADDR_MGMT_PLBU_ALLOC_START_ADDR, start_addr); + mali_hw_core_register_write_relaxed(&core->hw_core, MALIGP2_REG_ADDR_MGMT_PLBU_ALLOC_END_ADDR, end_addr); + + MALI_DEBUG_PRINT(3, ("Mali GP: Resuming job\n")); + + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_CMD, MALIGP2_REG_VAL_CMD_UPDATE_PLBU_ALLOC); + _mali_osk_write_mem_barrier(); + } + /* + * else: core has been reset between PLBU_OUT_OF_MEM interrupt and this new heap response. + * A timeout or a page fault on Mali-200 PP core can cause this behaviour. + */ +} + +u32 mali_gp_core_get_version(struct mali_gp_core *core) +{ + MALI_DEBUG_ASSERT_POINTER(core); + return mali_hw_core_register_read(&core->hw_core, MALIGP2_REG_ADDR_MGMT_VERSION); +} + +struct mali_gp_core *mali_gp_get_global_gp_core(void) +{ + return mali_global_gp_core; +} + +/* ------------- interrupt handling below ------------------ */ +static void mali_gp_irq_probe_trigger(void *data) +{ + struct mali_gp_core *core = (struct mali_gp_core *)data; + + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_MASK, MALIGP2_REG_VAL_IRQ_MASK_USED); + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_RAWSTAT, MALIGP2_REG_VAL_IRQ_AXI_BUS_ERROR); + _mali_osk_mem_barrier(); +} + +static _mali_osk_errcode_t mali_gp_irq_probe_ack(void *data) +{ + struct mali_gp_core *core = (struct mali_gp_core *)data; + u32 irq_readout; + + irq_readout = mali_hw_core_register_read(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_STAT); + if (MALIGP2_REG_VAL_IRQ_AXI_BUS_ERROR & irq_readout) { + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_CLEAR, MALIGP2_REG_VAL_IRQ_AXI_BUS_ERROR); + _mali_osk_mem_barrier(); + return _MALI_OSK_ERR_OK; + } + + return _MALI_OSK_ERR_FAULT; +} + +/* ------ local helper functions below --------- */ +#if MALI_STATE_TRACKING +u32 mali_gp_dump_state(struct mali_gp_core *core, char *buf, u32 size) +{ + int n = 0; + + n += _mali_osk_snprintf(buf + n, size - n, "\tGP: %s\n", core->hw_core.description); + + return n; +} +#endif + +void mali_gp_update_performance_counters(struct mali_gp_core *core, struct mali_gp_job *job) +{ + u32 val0 = 0; + u32 val1 = 0; + u32 counter_src0 = mali_gp_job_get_perf_counter_src0(job); + u32 counter_src1 = mali_gp_job_get_perf_counter_src1(job); + + if (MALI_HW_CORE_NO_COUNTER != counter_src0) { + val0 = mali_hw_core_register_read(&core->hw_core, MALIGP2_REG_ADDR_MGMT_PERF_CNT_0_VALUE); + mali_gp_job_set_perf_counter_value0(job, val0); + +#if defined(CONFIG_MALI400_PROFILING) + _mali_osk_profiling_report_hw_counter(COUNTER_VP_0_C0, val0); + _mali_osk_profiling_record_global_counters(COUNTER_VP_0_C0, val0); +#endif + + } + + if (MALI_HW_CORE_NO_COUNTER != counter_src1) { + val1 = mali_hw_core_register_read(&core->hw_core, MALIGP2_REG_ADDR_MGMT_PERF_CNT_1_VALUE); + mali_gp_job_set_perf_counter_value1(job, val1); + +#if defined(CONFIG_MALI400_PROFILING) + _mali_osk_profiling_report_hw_counter(COUNTER_VP_0_C1, val1); + _mali_osk_profiling_record_global_counters(COUNTER_VP_0_C1, val1); +#endif + } +} diff --git a/drivers/gpu/arm/utgard/common/mali_gp.h b/drivers/gpu/arm/utgard/common/mali_gp.h new file mode 100644 index 00000000000000..8d5f69c23229ce --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_gp.h @@ -0,0 +1,127 @@ +/* + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_GP_H__ +#define __MALI_GP_H__ + +#include "mali_osk.h" +#include "mali_gp_job.h" +#include "mali_hw_core.h" +#include "regs/mali_gp_regs.h" + +struct mali_group; + +/** + * Definition of the GP core struct + * Used to track a GP core in the system. + */ +struct mali_gp_core { + struct mali_hw_core hw_core; /**< Common for all HW cores */ + _mali_osk_irq_t *irq; /**< IRQ handler */ +}; + +_mali_osk_errcode_t mali_gp_initialize(void); +void mali_gp_terminate(void); + +struct mali_gp_core *mali_gp_create(const _mali_osk_resource_t *resource, struct mali_group *group); +void mali_gp_delete(struct mali_gp_core *core); + +void mali_gp_stop_bus(struct mali_gp_core *core); +_mali_osk_errcode_t mali_gp_stop_bus_wait(struct mali_gp_core *core); +void mali_gp_reset_async(struct mali_gp_core *core); +_mali_osk_errcode_t mali_gp_reset_wait(struct mali_gp_core *core); +void mali_gp_hard_reset(struct mali_gp_core *core); +_mali_osk_errcode_t mali_gp_reset(struct mali_gp_core *core); + +void mali_gp_job_start(struct mali_gp_core *core, struct mali_gp_job *job); +void mali_gp_resume_with_new_heap(struct mali_gp_core *core, u32 start_addr, u32 end_addr); + +u32 mali_gp_core_get_version(struct mali_gp_core *core); + +struct mali_gp_core *mali_gp_get_global_gp_core(void); + +#if MALI_STATE_TRACKING +u32 mali_gp_dump_state(struct mali_gp_core *core, char *buf, u32 size); +#endif + +void mali_gp_update_performance_counters(struct mali_gp_core *core, struct mali_gp_job *job); + +MALI_STATIC_INLINE const char *mali_gp_core_description(struct mali_gp_core *core) +{ + return core->hw_core.description; +} + +MALI_STATIC_INLINE enum mali_interrupt_result mali_gp_get_interrupt_result(struct mali_gp_core *core) +{ + u32 stat_used = mali_hw_core_register_read(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_STAT) & + MALIGP2_REG_VAL_IRQ_MASK_USED; + + if (0 == stat_used) { + return MALI_INTERRUPT_RESULT_NONE; + } else if ((MALIGP2_REG_VAL_IRQ_VS_END_CMD_LST | + MALIGP2_REG_VAL_IRQ_PLBU_END_CMD_LST) == stat_used) { + return MALI_INTERRUPT_RESULT_SUCCESS; + } else if (MALIGP2_REG_VAL_IRQ_VS_END_CMD_LST == stat_used) { + return MALI_INTERRUPT_RESULT_SUCCESS_VS; + } else if (MALIGP2_REG_VAL_IRQ_PLBU_END_CMD_LST == stat_used) { + return MALI_INTERRUPT_RESULT_SUCCESS_PLBU; + } else if (MALIGP2_REG_VAL_IRQ_PLBU_OUT_OF_MEM & stat_used) { + return MALI_INTERRUPT_RESULT_OOM; + } + + return MALI_INTERRUPT_RESULT_ERROR; +} + +MALI_STATIC_INLINE u32 mali_gp_get_rawstat(struct mali_gp_core *core) +{ + MALI_DEBUG_ASSERT_POINTER(core); + return mali_hw_core_register_read(&core->hw_core, + MALIGP2_REG_ADDR_MGMT_INT_RAWSTAT); +} + +MALI_STATIC_INLINE u32 mali_gp_is_active(struct mali_gp_core *core) +{ + u32 status = mali_hw_core_register_read(&core->hw_core, MALIGP2_REG_ADDR_MGMT_STATUS); + return (status & MALIGP2_REG_VAL_STATUS_MASK_ACTIVE) ? MALI_TRUE : MALI_FALSE; +} + +MALI_STATIC_INLINE void mali_gp_mask_all_interrupts(struct mali_gp_core *core) +{ + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_MASK, MALIGP2_REG_VAL_IRQ_MASK_NONE); +} + +MALI_STATIC_INLINE void mali_gp_enable_interrupts(struct mali_gp_core *core, enum mali_interrupt_result exceptions) +{ + /* Enable all interrupts, except those specified in exceptions */ + u32 value; + + if (MALI_INTERRUPT_RESULT_SUCCESS_VS == exceptions) { + /* Enable all used except VS complete */ + value = MALIGP2_REG_VAL_IRQ_MASK_USED & + ~MALIGP2_REG_VAL_IRQ_VS_END_CMD_LST; + } else { + MALI_DEBUG_ASSERT(MALI_INTERRUPT_RESULT_SUCCESS_PLBU == + exceptions); + /* Enable all used except PLBU complete */ + value = MALIGP2_REG_VAL_IRQ_MASK_USED & + ~MALIGP2_REG_VAL_IRQ_PLBU_END_CMD_LST; + } + + mali_hw_core_register_write(&core->hw_core, + MALIGP2_REG_ADDR_MGMT_INT_MASK, + value); +} + +MALI_STATIC_INLINE u32 mali_gp_read_plbu_alloc_start_addr(struct mali_gp_core *core) +{ + return mali_hw_core_register_read(&core->hw_core, MALIGP2_REG_ADDR_MGMT_PLBU_ALLOC_START_ADDR); +} + +#endif /* __MALI_GP_H__ */ diff --git a/drivers/gpu/arm/utgard/common/mali_gp_job.c b/drivers/gpu/arm/utgard/common/mali_gp_job.c new file mode 100644 index 00000000000000..0305d98ba2cc8b --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_gp_job.c @@ -0,0 +1,301 @@ +/* + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_gp_job.h" +#include "mali_osk.h" +#include "mali_osk_list.h" +#include "mali_uk_types.h" +#include "mali_memory_virtual.h" +#include "mali_memory_defer_bind.h" + +static u32 gp_counter_src0 = MALI_HW_CORE_NO_COUNTER; /**< Performance counter 0, MALI_HW_CORE_NO_COUNTER for disabled */ +static u32 gp_counter_src1 = MALI_HW_CORE_NO_COUNTER; /**< Performance counter 1, MALI_HW_CORE_NO_COUNTER for disabled */ +static void _mali_gp_del_varying_allocations(struct mali_gp_job *job); + + +static int _mali_gp_add_varying_allocations(struct mali_session_data *session, + struct mali_gp_job *job, + u32 *alloc, + u32 num) +{ + int i = 0; + struct mali_gp_allocation_node *alloc_node; + mali_mem_allocation *mali_alloc = NULL; + struct mali_vma_node *mali_vma_node = NULL; + + for (i = 0 ; i < num ; i++) { + MALI_DEBUG_ASSERT(alloc[i]); + alloc_node = _mali_osk_calloc(1, sizeof(struct mali_gp_allocation_node)); + if (alloc_node) { + INIT_LIST_HEAD(&alloc_node->node); + /* find mali allocation structure by vaddress*/ + mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, alloc[i], 0); + + if (likely(mali_vma_node)) { + mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); + MALI_DEBUG_ASSERT(alloc[i] == mali_vma_node->vm_node.start); + } else { + MALI_DEBUG_PRINT(1, ("ERROE!_mali_gp_add_varying_allocations,can't find allocation %d by address =0x%x, num=%d\n", i, alloc[i], num)); + MALI_DEBUG_ASSERT(0); + } + alloc_node->alloc = mali_alloc; + /* add to gp job varying alloc list*/ + list_move(&alloc_node->node, &job->varying_alloc); + } else + goto fail; + } + + return 0; +fail: + MALI_DEBUG_PRINT(1, ("ERROE!_mali_gp_add_varying_allocations,failed to alloc memory!\n")); + _mali_gp_del_varying_allocations(job); + return -1; +} + + +static void _mali_gp_del_varying_allocations(struct mali_gp_job *job) +{ + struct mali_gp_allocation_node *alloc_node, *tmp_node; + + list_for_each_entry_safe(alloc_node, tmp_node, &job->varying_alloc, node) { + list_del(&alloc_node->node); + kfree(alloc_node); + } + INIT_LIST_HEAD(&job->varying_alloc); +} + +struct mali_gp_job *mali_gp_job_create(struct mali_session_data *session, _mali_uk_gp_start_job_s *uargs, u32 id, struct mali_timeline_tracker *pp_tracker) +{ + struct mali_gp_job *job; + u32 perf_counter_flag; + u32 __user *memory_list = NULL; + struct mali_gp_allocation_node *alloc_node, *tmp_node; + + job = _mali_osk_calloc(1, sizeof(struct mali_gp_job)); + if (NULL != job) { + job->finished_notification = _mali_osk_notification_create(_MALI_NOTIFICATION_GP_FINISHED, sizeof(_mali_uk_gp_job_finished_s)); + if (NULL == job->finished_notification) { + goto fail3; + } + + job->oom_notification = _mali_osk_notification_create(_MALI_NOTIFICATION_GP_STALLED, sizeof(_mali_uk_gp_job_suspended_s)); + if (NULL == job->oom_notification) { + goto fail2; + } + + if (0 != _mali_osk_copy_from_user(&job->uargs, uargs, sizeof(_mali_uk_gp_start_job_s))) { + goto fail1; + } + + perf_counter_flag = mali_gp_job_get_perf_counter_flag(job); + + /* case when no counters came from user space + * so pass the debugfs / DS-5 provided global ones to the job object */ + if (!((perf_counter_flag & _MALI_PERFORMANCE_COUNTER_FLAG_SRC0_ENABLE) || + (perf_counter_flag & _MALI_PERFORMANCE_COUNTER_FLAG_SRC1_ENABLE))) { + mali_gp_job_set_perf_counter_src0(job, mali_gp_job_get_gp_counter_src0()); + mali_gp_job_set_perf_counter_src1(job, mali_gp_job_get_gp_counter_src1()); + } + + _mali_osk_list_init(&job->list); + job->session = session; + job->id = id; + job->heap_base_addr = job->uargs.frame_registers[4]; + job->heap_current_addr = job->uargs.frame_registers[4]; + job->heap_grow_size = job->uargs.heap_grow_size; + job->perf_counter_value0 = 0; + job->perf_counter_value1 = 0; + job->pid = _mali_osk_get_pid(); + job->tid = _mali_osk_get_tid(); + + + INIT_LIST_HEAD(&job->varying_alloc); + INIT_LIST_HEAD(&job->vary_todo); + job->dmem = NULL; + /* add varying allocation list*/ + if (job->uargs.varying_alloc_num) { + /* copy varying list from user space*/ + job->varying_list = _mali_osk_calloc(1, sizeof(u32) * job->uargs.varying_alloc_num); + if (!job->varying_list) { + MALI_PRINT_ERROR(("Mali GP job: allocate varying_list failed varying_alloc_num = %d !\n", uargs->varying_alloc_num)); + goto fail1; + } + + memory_list = (u32 __user *)(uintptr_t)job->uargs.varying_alloc_list; + + if (0 != _mali_osk_copy_from_user(job->varying_list, memory_list, sizeof(u32)*job->uargs.varying_alloc_num)) { + MALI_PRINT_ERROR(("Mali GP job: Failed to copy varying list from user space!\n")); + goto fail; + } + + if (unlikely(_mali_gp_add_varying_allocations(session, job, job->varying_list, + job->uargs.varying_alloc_num))) { + MALI_PRINT_ERROR(("Mali GP job: _mali_gp_add_varying_allocations failed!\n")); + goto fail; + } + + /* do preparetion for each allocation */ + list_for_each_entry_safe(alloc_node, tmp_node, &job->varying_alloc, node) { + if (unlikely(_MALI_OSK_ERR_OK != mali_mem_defer_bind_allocation_prepare(alloc_node->alloc, &job->vary_todo))) { + MALI_PRINT_ERROR(("Mali GP job: mali_mem_defer_bind_allocation_prepare failed!\n")); + goto fail; + } + } + + _mali_gp_del_varying_allocations(job); + + /* bind varying here, to avoid memory latency issue. */ + { + struct mali_defer_mem_block dmem_block; + + INIT_LIST_HEAD(&dmem_block.free_pages); + atomic_set(&dmem_block.num_free_pages, 0); + + if (mali_mem_prepare_mem_for_job(job, &dmem_block)) { + MALI_PRINT_ERROR(("Mali GP job: mali_mem_prepare_mem_for_job failed!\n")); + goto fail; + } + if (_MALI_OSK_ERR_OK != mali_mem_defer_bind(job->uargs.varying_memsize / _MALI_OSK_MALI_PAGE_SIZE, job, &dmem_block)) { + MALI_PRINT_ERROR(("gp job create, mali_mem_defer_bind failed! GP %x fail!", job)); + goto fail; + } + } + + if (job->uargs.varying_memsize > MALI_UK_BIG_VARYING_SIZE) { + job->big_job = 1; + } + } + job->pp_tracker = pp_tracker; + if (NULL != job->pp_tracker) { + /* Take a reference on PP job's tracker that will be released when the GP + job is done. */ + mali_timeline_system_tracker_get(session->timeline_system, pp_tracker); + } + + mali_timeline_tracker_init(&job->tracker, MALI_TIMELINE_TRACKER_GP, NULL, job); + mali_timeline_fence_copy_uk_fence(&(job->tracker.fence), &(job->uargs.fence)); + + return job; + } else { + MALI_PRINT_ERROR(("Mali GP job: _mali_osk_calloc failed!\n")); + return NULL; + } + + +fail: + _mali_osk_free(job->varying_list); + /* Handle allocate fail here, free all varying node */ + { + struct mali_backend_bind_list *bkn, *bkn_tmp; + list_for_each_entry_safe(bkn, bkn_tmp , &job->vary_todo, node) { + list_del(&bkn->node); + _mali_osk_free(bkn); + } + } +fail1: + _mali_osk_notification_delete(job->oom_notification); +fail2: + _mali_osk_notification_delete(job->finished_notification); +fail3: + _mali_osk_free(job); + return NULL; +} + +void mali_gp_job_delete(struct mali_gp_job *job) +{ + struct mali_backend_bind_list *bkn, *bkn_tmp; + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT(NULL == job->pp_tracker); + MALI_DEBUG_ASSERT(_mali_osk_list_empty(&job->list)); + _mali_osk_free(job->varying_list); + + /* Handle allocate fail here, free all varying node */ + list_for_each_entry_safe(bkn, bkn_tmp , &job->vary_todo, node) { + list_del(&bkn->node); + _mali_osk_free(bkn); + } + + if (!list_empty(&job->vary_todo)) { + MALI_DEBUG_ASSERT(0); + } + + mali_mem_defer_dmem_free(job); + + /* de-allocate the pre-allocated oom notifications */ + if (NULL != job->oom_notification) { + _mali_osk_notification_delete(job->oom_notification); + job->oom_notification = NULL; + } + if (NULL != job->finished_notification) { + _mali_osk_notification_delete(job->finished_notification); + job->finished_notification = NULL; + } + + _mali_osk_free(job); +} + +void mali_gp_job_list_add(struct mali_gp_job *job, _mali_osk_list_t *list) +{ + struct mali_gp_job *iter; + struct mali_gp_job *tmp; + + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + + /* Find position in list/queue where job should be added. */ + _MALI_OSK_LIST_FOREACHENTRY_REVERSE(iter, tmp, list, + struct mali_gp_job, list) { + + /* A span is used to handle job ID wrapping. */ + bool job_is_after = (mali_gp_job_get_id(job) - + mali_gp_job_get_id(iter)) < + MALI_SCHEDULER_JOB_ID_SPAN; + + if (job_is_after) { + break; + } + } + + _mali_osk_list_add(&job->list, &iter->list); +} + +u32 mali_gp_job_get_gp_counter_src0(void) +{ + return gp_counter_src0; +} + +void mali_gp_job_set_gp_counter_src0(u32 counter) +{ + gp_counter_src0 = counter; +} + +u32 mali_gp_job_get_gp_counter_src1(void) +{ + return gp_counter_src1; +} + +void mali_gp_job_set_gp_counter_src1(u32 counter) +{ + gp_counter_src1 = counter; +} + +mali_scheduler_mask mali_gp_job_signal_pp_tracker(struct mali_gp_job *job, mali_bool success) +{ + mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; + + MALI_DEBUG_ASSERT_POINTER(job); + + if (NULL != job->pp_tracker) { + schedule_mask |= mali_timeline_system_tracker_put(job->session->timeline_system, job->pp_tracker, MALI_FALSE == success); + job->pp_tracker = NULL; + } + + return schedule_mask; +} diff --git a/drivers/gpu/arm/utgard/common/mali_gp_job.h b/drivers/gpu/arm/utgard/common/mali_gp_job.h new file mode 100644 index 00000000000000..f249439c71552d --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_gp_job.h @@ -0,0 +1,325 @@ +/* + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_GP_JOB_H__ +#define __MALI_GP_JOB_H__ + +#include "mali_osk.h" +#include "mali_osk_list.h" +#include "mali_uk_types.h" +#include "mali_session.h" +#include "mali_timeline.h" +#include "mali_scheduler_types.h" +#include "mali_scheduler.h" +#include "mali_executor.h" +#include "mali_timeline.h" + +struct mali_defer_mem; +/** + * This structure represents a GP job + * + * The GP job object itself is not protected by any single lock, + * but relies on other locks instead (scheduler, executor and timeline lock). + * Think of the job object as moving between these sub systems through-out + * its lifetime. Different part of the GP job struct is used by different + * subsystems. Accessor functions ensure that correct lock is taken. + * Do NOT access any data members directly from outside this module! + */ +struct mali_gp_job { + /* + * These members are typically only set at creation, + * and only read later on. + * They do not require any lock protection. + */ + _mali_uk_gp_start_job_s uargs; /**< Arguments from user space */ + struct mali_session_data *session; /**< Session which submitted this job */ + u32 pid; /**< Process ID of submitting process */ + u32 tid; /**< Thread ID of submitting thread */ + u32 id; /**< Identifier for this job in kernel space (sequential numbering) */ + u32 cache_order; /**< Cache order used for L2 cache flushing (sequential numbering) */ + struct mali_timeline_tracker tracker; /**< Timeline tracker for this job */ + struct mali_timeline_tracker *pp_tracker; /**< Pointer to Timeline tracker for PP job that depends on this job. */ + _mali_osk_notification_t *finished_notification; /**< Notification sent back to userspace on job complete */ + + /* + * These members are used by the scheduler, + * protected by scheduler lock + */ + _mali_osk_list_t list; /**< Used to link jobs together in the scheduler queue */ + + /* + * These members are used by the executor and/or group, + * protected by executor lock + */ + _mali_osk_notification_t *oom_notification; /**< Notification sent back to userspace on OOM */ + + /* + * Set by executor/group on job completion, read by scheduler when + * returning job to user. Hold executor lock when setting, + * no lock needed when reading + */ + u32 heap_base_addr; /** < Holds the base mali addr of mem handle which is used for new heap*/ + u32 heap_current_addr; /**< Holds the current HEAP address when the job has completed */ + u32 heap_grow_size; /** < Holds the HEAP grow size when HEAP oom */ + u32 perf_counter_value0; /**< Value of performance counter 0 (to be returned to user space) */ + u32 perf_counter_value1; /**< Value of performance counter 1 (to be returned to user space) */ + struct mali_defer_mem *dmem; /** < used for defer bind to store dmem info */ + struct list_head varying_alloc; /**< hold the list of varying allocations */ + u32 bind_flag; /** < flag for deferbind*/ + u32 *varying_list; /**< varying memory list need to to defer bind*/ + struct list_head vary_todo; /**< list of backend list need to do defer bind*/ + u32 big_job; /** < if the gp job have large varying output and may take long time*/ +}; + +#define MALI_DEFER_BIND_MEMORY_PREPARED (0x1 << 0) +#define MALI_DEFER_BIND_MEMORY_BINDED (0x1 << 2) + +struct mali_gp_allocation_node { + struct list_head node; + mali_mem_allocation *alloc; +}; + +struct mali_gp_job *mali_gp_job_create(struct mali_session_data *session, _mali_uk_gp_start_job_s *uargs, u32 id, struct mali_timeline_tracker *pp_tracker); +void mali_gp_job_delete(struct mali_gp_job *job); + +u32 mali_gp_job_get_gp_counter_src0(void); +void mali_gp_job_set_gp_counter_src0(u32 counter); +u32 mali_gp_job_get_gp_counter_src1(void); +void mali_gp_job_set_gp_counter_src1(u32 counter); + +MALI_STATIC_INLINE u32 mali_gp_job_get_id(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return (NULL == job) ? 0 : job->id; +} + +MALI_STATIC_INLINE void mali_gp_job_set_cache_order(struct mali_gp_job *job, + u32 cache_order) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + job->cache_order = cache_order; +} + +MALI_STATIC_INLINE u32 mali_gp_job_get_cache_order(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return (NULL == job) ? 0 : job->cache_order; +} + +MALI_STATIC_INLINE u64 mali_gp_job_get_user_id(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.user_job_ptr; +} + +MALI_STATIC_INLINE u32 mali_gp_job_get_frame_builder_id(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.frame_builder_id; +} + +MALI_STATIC_INLINE u32 mali_gp_job_get_flush_id(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.flush_id; +} + +MALI_STATIC_INLINE u32 mali_gp_job_get_pid(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->pid; +} + +MALI_STATIC_INLINE u32 mali_gp_job_get_tid(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->tid; +} + +MALI_STATIC_INLINE u32 *mali_gp_job_get_frame_registers(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.frame_registers; +} + +MALI_STATIC_INLINE struct mali_session_data *mali_gp_job_get_session(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->session; +} + +MALI_STATIC_INLINE mali_bool mali_gp_job_has_vs_job(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return (job->uargs.frame_registers[0] != job->uargs.frame_registers[1]) ? MALI_TRUE : MALI_FALSE; +} + +MALI_STATIC_INLINE mali_bool mali_gp_job_has_plbu_job(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return (job->uargs.frame_registers[2] != job->uargs.frame_registers[3]) ? MALI_TRUE : MALI_FALSE; +} + +MALI_STATIC_INLINE u32 mali_gp_job_get_current_heap_addr(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->heap_current_addr; +} + +MALI_STATIC_INLINE void mali_gp_job_set_current_heap_addr(struct mali_gp_job *job, u32 heap_addr) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + job->heap_current_addr = heap_addr; +} + +MALI_STATIC_INLINE u32 mali_gp_job_get_perf_counter_flag(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.perf_counter_flag; +} + +MALI_STATIC_INLINE u32 mali_gp_job_get_perf_counter_src0(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.perf_counter_src0; +} + +MALI_STATIC_INLINE u32 mali_gp_job_get_perf_counter_src1(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.perf_counter_src1; +} + +MALI_STATIC_INLINE u32 mali_gp_job_get_perf_counter_value0(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->perf_counter_value0; +} + +MALI_STATIC_INLINE u32 mali_gp_job_get_perf_counter_value1(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->perf_counter_value1; +} + +MALI_STATIC_INLINE void mali_gp_job_set_perf_counter_src0(struct mali_gp_job *job, u32 src) +{ + MALI_DEBUG_ASSERT_POINTER(job); + job->uargs.perf_counter_src0 = src; +} + +MALI_STATIC_INLINE void mali_gp_job_set_perf_counter_src1(struct mali_gp_job *job, u32 src) +{ + MALI_DEBUG_ASSERT_POINTER(job); + job->uargs.perf_counter_src1 = src; +} + +MALI_STATIC_INLINE void mali_gp_job_set_perf_counter_value0(struct mali_gp_job *job, u32 value) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + job->perf_counter_value0 = value; +} + +MALI_STATIC_INLINE void mali_gp_job_set_perf_counter_value1(struct mali_gp_job *job, u32 value) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + job->perf_counter_value1 = value; +} + +void mali_gp_job_list_add(struct mali_gp_job *job, _mali_osk_list_t *list); + +MALI_STATIC_INLINE void mali_gp_job_list_move(struct mali_gp_job *job, + _mali_osk_list_t *list) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + MALI_DEBUG_ASSERT(!_mali_osk_list_empty(&job->list)); + _mali_osk_list_move(&job->list, list); +} + +MALI_STATIC_INLINE void mali_gp_job_list_remove(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + _mali_osk_list_delinit(&job->list); +} + +MALI_STATIC_INLINE _mali_osk_notification_t * +mali_gp_job_get_finished_notification(struct mali_gp_job *job) +{ + _mali_osk_notification_t *notification; + + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_POINTER(job->finished_notification); + + notification = job->finished_notification; + job->finished_notification = NULL; + + return notification; +} + +MALI_STATIC_INLINE _mali_osk_notification_t *mali_gp_job_get_oom_notification( + struct mali_gp_job *job) +{ + _mali_osk_notification_t *notification; + + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + MALI_DEBUG_ASSERT_POINTER(job->oom_notification); + + notification = job->oom_notification; + job->oom_notification = NULL; + + return notification; +} + +MALI_STATIC_INLINE void mali_gp_job_set_oom_notification( + struct mali_gp_job *job, + _mali_osk_notification_t *notification) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + MALI_DEBUG_ASSERT(NULL == job->oom_notification); + job->oom_notification = notification; +} + +MALI_STATIC_INLINE struct mali_timeline_tracker *mali_gp_job_get_tracker( + struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return &(job->tracker); +} + + +MALI_STATIC_INLINE u32 *mali_gp_job_get_timeline_point_ptr( + struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return (u32 __user *)(uintptr_t)job->uargs.timeline_point_ptr; +} + + +/** + * Release reference on tracker for PP job that depends on this GP job. + * + * @note If GP job has a reference on tracker, this function MUST be called before the GP job is + * deleted. + * + * @param job GP job that is done. + * @param success MALI_TRUE if job completed successfully, MALI_FALSE if not. + * @return A scheduling bitmask indicating whether scheduling needs to be done. + */ +mali_scheduler_mask mali_gp_job_signal_pp_tracker(struct mali_gp_job *job, mali_bool success); + +#endif /* __MALI_GP_JOB_H__ */ diff --git a/drivers/gpu/arm/utgard/common/mali_group.c b/drivers/gpu/arm/utgard/common/mali_group.c new file mode 100644 index 00000000000000..b4cd3a1fcd8cc2 --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_group.c @@ -0,0 +1,1816 @@ +/* + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#include "mali_kernel_common.h" +#include "mali_group.h" +#include "mali_osk.h" +#include "mali_l2_cache.h" +#include "mali_gp.h" +#include "mali_pp.h" +#include "mali_mmu.h" +#include "mali_dlbu.h" +#include "mali_broadcast.h" +#include "mali_scheduler.h" +#include "mali_osk_profiling.h" +#include "mali_pm_domain.h" +#include "mali_pm.h" +#include "mali_executor.h" + +#if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS) +#include +#include +#endif + +#define MALI_MAX_NUM_DOMAIN_REFS (MALI_MAX_NUMBER_OF_GROUPS * 2) + +#if defined(CONFIG_MALI400_PROFILING) +static void mali_group_report_l2_cache_counters_per_core(struct mali_group *group, u32 core_num); +#endif /* #if defined(CONFIG_MALI400_PROFILING) */ + +static struct mali_group *mali_global_groups[MALI_MAX_NUMBER_OF_GROUPS] = { NULL, }; +static u32 mali_global_num_groups = 0; + +/* SW timer for job execution */ +int mali_max_job_runtime = MALI_MAX_JOB_RUNTIME_DEFAULT; + +/* local helper functions */ +static void mali_group_bottom_half_mmu(void *data); +static void mali_group_bottom_half_gp(void *data); +static void mali_group_bottom_half_pp(void *data); +static void mali_group_timeout(void *data); +static void mali_group_out_of_memory(void *data); + +static void mali_group_reset_pp(struct mali_group *group); +static void mali_group_reset_mmu(struct mali_group *group); + +static void mali_group_activate_page_directory(struct mali_group *group, struct mali_session_data *session); +static void mali_group_recovery_reset(struct mali_group *group); + +struct mali_group *mali_group_create(struct mali_l2_cache_core *core, + struct mali_dlbu_core *dlbu, + struct mali_bcast_unit *bcast, + u32 domain_index) +{ + struct mali_group *group = NULL; + + if (mali_global_num_groups >= MALI_MAX_NUMBER_OF_GROUPS) { + MALI_PRINT_ERROR(("Mali group: Too many group objects created\n")); + return NULL; + } + + group = _mali_osk_calloc(1, sizeof(struct mali_group)); + if (NULL != group) { + group->timeout_timer = _mali_osk_timer_init(); + if (NULL != group->timeout_timer) { + _mali_osk_timer_setcallback(group->timeout_timer, mali_group_timeout, (void *)group); + + group->l2_cache_core[0] = core; + _mali_osk_list_init(&group->group_list); + _mali_osk_list_init(&group->executor_list); + _mali_osk_list_init(&group->pm_domain_list); + group->bcast_core = bcast; + group->dlbu_core = dlbu; + + /* register this object as a part of the correct power domain */ + if ((NULL != core) || (NULL != dlbu) || (NULL != bcast)) + group->pm_domain = mali_pm_register_group(domain_index, group); + + mali_global_groups[mali_global_num_groups] = group; + mali_global_num_groups++; + + return group; + } + _mali_osk_free(group); + } + + return NULL; +} + +void mali_group_delete(struct mali_group *group) +{ + u32 i; + + MALI_DEBUG_PRINT(4, ("Deleting group %s\n", + mali_group_core_description(group))); + + MALI_DEBUG_ASSERT(NULL == group->parent_group); + MALI_DEBUG_ASSERT((MALI_GROUP_STATE_INACTIVE == group->state) || ((MALI_GROUP_STATE_ACTIVATION_PENDING == group->state))); + + /* Delete the resources that this group owns */ + if (NULL != group->gp_core) { + mali_gp_delete(group->gp_core); + } + + if (NULL != group->pp_core) { + mali_pp_delete(group->pp_core); + } + + if (NULL != group->mmu) { + mali_mmu_delete(group->mmu); + } + + if (mali_group_is_virtual(group)) { + /* Remove all groups from virtual group */ + struct mali_group *child; + struct mali_group *temp; + + _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list) { + child->parent_group = NULL; + mali_group_delete(child); + } + + mali_dlbu_delete(group->dlbu_core); + + if (NULL != group->bcast_core) { + mali_bcast_unit_delete(group->bcast_core); + } + } + + for (i = 0; i < mali_global_num_groups; i++) { + if (mali_global_groups[i] == group) { + mali_global_groups[i] = NULL; + mali_global_num_groups--; + + if (i != mali_global_num_groups) { + /* We removed a group from the middle of the array -- move the last + * group to the current position to close the gap */ + mali_global_groups[i] = mali_global_groups[mali_global_num_groups]; + mali_global_groups[mali_global_num_groups] = NULL; + } + + break; + } + } + + if (NULL != group->timeout_timer) { + _mali_osk_timer_del(group->timeout_timer); + _mali_osk_timer_term(group->timeout_timer); + } + + if (NULL != group->bottom_half_work_mmu) { + _mali_osk_wq_delete_work(group->bottom_half_work_mmu); + } + + if (NULL != group->bottom_half_work_gp) { + _mali_osk_wq_delete_work(group->bottom_half_work_gp); + } + + if (NULL != group->bottom_half_work_pp) { + _mali_osk_wq_delete_work(group->bottom_half_work_pp); + } + + _mali_osk_free(group); +} + +_mali_osk_errcode_t mali_group_add_mmu_core(struct mali_group *group, struct mali_mmu_core *mmu_core) +{ + /* This group object now owns the MMU core object */ + group->mmu = mmu_core; + group->bottom_half_work_mmu = _mali_osk_wq_create_work(mali_group_bottom_half_mmu, group); + if (NULL == group->bottom_half_work_mmu) { + return _MALI_OSK_ERR_FAULT; + } + return _MALI_OSK_ERR_OK; +} + +void mali_group_remove_mmu_core(struct mali_group *group) +{ + /* This group object no longer owns the MMU core object */ + group->mmu = NULL; + if (NULL != group->bottom_half_work_mmu) { + _mali_osk_wq_delete_work(group->bottom_half_work_mmu); + } +} + +_mali_osk_errcode_t mali_group_add_gp_core(struct mali_group *group, struct mali_gp_core *gp_core) +{ + /* This group object now owns the GP core object */ + group->gp_core = gp_core; + group->bottom_half_work_gp = _mali_osk_wq_create_work(mali_group_bottom_half_gp, group); + if (NULL == group->bottom_half_work_gp) { + return _MALI_OSK_ERR_FAULT; + } + + group->oom_work_handler = _mali_osk_wq_create_work(mali_group_out_of_memory, group); + if (NULL == group->oom_work_handler) { + _mali_osk_wq_delete_work(group->bottom_half_work_gp); + } + return _MALI_OSK_ERR_OK; +} + +void mali_group_remove_gp_core(struct mali_group *group) +{ + /* This group object no longer owns the GP core object */ + group->gp_core = NULL; + if (NULL != group->bottom_half_work_gp) { + _mali_osk_wq_delete_work(group->bottom_half_work_gp); + } + + if (NULL != group->oom_work_handler) { + _mali_osk_wq_delete_work(group->oom_work_handler); + } +} + +_mali_osk_errcode_t mali_group_add_pp_core(struct mali_group *group, struct mali_pp_core *pp_core) +{ + /* This group object now owns the PP core object */ + group->pp_core = pp_core; + group->bottom_half_work_pp = _mali_osk_wq_create_work(mali_group_bottom_half_pp, group); + if (NULL == group->bottom_half_work_pp) { + return _MALI_OSK_ERR_FAULT; + } + return _MALI_OSK_ERR_OK; +} + +void mali_group_remove_pp_core(struct mali_group *group) +{ + /* This group object no longer owns the PP core object */ + group->pp_core = NULL; + if (NULL != group->bottom_half_work_pp) { + _mali_osk_wq_delete_work(group->bottom_half_work_pp); + } +} + +enum mali_group_state mali_group_activate(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + MALI_DEBUG_PRINT(4, ("Group: Activating group %s\n", + mali_group_core_description(group))); + + if (MALI_GROUP_STATE_INACTIVE == group->state) { + /* Group is inactive, get PM refs in order to power up */ + + /* + * We'll take a maximum of 2 power domain references pr group, + * one for the group itself, and one for it's L2 cache. + */ + struct mali_pm_domain *domains[MALI_MAX_NUM_DOMAIN_REFS]; + struct mali_group *groups[MALI_MAX_NUM_DOMAIN_REFS]; + u32 num_domains = 0; + mali_bool all_groups_on; + + /* Deal with child groups first */ + if (mali_group_is_virtual(group)) { + /* + * The virtual group might have 0, 1 or 2 L2s in + * its l2_cache_core array, but we ignore these and + * let the child groups take the needed L2 cache ref + * on behalf of the virtual group. + * In other words; The L2 refs are taken in pair with + * the physical group which the L2 is attached to. + */ + struct mali_group *child; + struct mali_group *temp; + + /* + * Child group is inactive, get PM + * refs in order to power up. + */ + _MALI_OSK_LIST_FOREACHENTRY(child, temp, + &group->group_list, + struct mali_group, group_list) { + MALI_DEBUG_ASSERT(MALI_GROUP_STATE_INACTIVE + == child->state); + + child->state = MALI_GROUP_STATE_ACTIVATION_PENDING; + + MALI_DEBUG_ASSERT_POINTER( + child->pm_domain); + domains[num_domains] = child->pm_domain; + groups[num_domains] = child; + num_domains++; + + /* + * Take L2 domain ref for child group. + */ + MALI_DEBUG_ASSERT(MALI_MAX_NUM_DOMAIN_REFS + > num_domains); + domains[num_domains] = mali_l2_cache_get_pm_domain( + child->l2_cache_core[0]); + groups[num_domains] = NULL; + MALI_DEBUG_ASSERT(NULL == + child->l2_cache_core[1]); + num_domains++; + } + } else { + /* Take L2 domain ref for physical groups. */ + MALI_DEBUG_ASSERT(MALI_MAX_NUM_DOMAIN_REFS > + num_domains); + + domains[num_domains] = mali_l2_cache_get_pm_domain( + group->l2_cache_core[0]); + groups[num_domains] = NULL; + MALI_DEBUG_ASSERT(NULL == group->l2_cache_core[1]); + num_domains++; + } + + /* Do the group itself last (it's dependencies first) */ + + group->state = MALI_GROUP_STATE_ACTIVATION_PENDING; + + MALI_DEBUG_ASSERT_POINTER(group->pm_domain); + domains[num_domains] = group->pm_domain; + groups[num_domains] = group; + num_domains++; + + all_groups_on = mali_pm_get_domain_refs(domains, groups, + num_domains); + + /* + * Complete activation for group, include + * virtual group or physical group. + */ + if (MALI_TRUE == all_groups_on) { + + mali_group_set_active(group); + } + } else if (MALI_GROUP_STATE_ACTIVE == group->state) { + /* Already active */ + MALI_DEBUG_ASSERT(MALI_TRUE == group->power_is_on); + } else { + /* + * Activation already pending, group->power_is_on could + * be both true or false. We need to wait for power up + * notification anyway. + */ + MALI_DEBUG_ASSERT(MALI_GROUP_STATE_ACTIVATION_PENDING + == group->state); + } + + MALI_DEBUG_PRINT(4, ("Group: group %s activation result: %s\n", + mali_group_core_description(group), + MALI_GROUP_STATE_ACTIVE == group->state ? + "ACTIVE" : "PENDING")); + + return group->state; +} + +mali_bool mali_group_set_active(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + MALI_DEBUG_ASSERT(MALI_GROUP_STATE_ACTIVATION_PENDING == group->state); + MALI_DEBUG_ASSERT(MALI_TRUE == group->power_is_on); + + MALI_DEBUG_PRINT(4, ("Group: Activation completed for %s\n", + mali_group_core_description(group))); + + if (mali_group_is_virtual(group)) { + struct mali_group *child; + struct mali_group *temp; + + _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, + struct mali_group, group_list) { + if (MALI_TRUE != child->power_is_on) { + return MALI_FALSE; + } + + child->state = MALI_GROUP_STATE_ACTIVE; + } + + mali_group_reset(group); + } + + /* Go to ACTIVE state */ + group->state = MALI_GROUP_STATE_ACTIVE; + + return MALI_TRUE; +} + +mali_bool mali_group_deactivate(struct mali_group *group) +{ + struct mali_pm_domain *domains[MALI_MAX_NUM_DOMAIN_REFS]; + u32 num_domains = 0; + mali_bool power_down = MALI_FALSE; + + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + MALI_DEBUG_ASSERT(MALI_GROUP_STATE_INACTIVE != group->state); + + MALI_DEBUG_PRINT(3, ("Group: Deactivating group %s\n", + mali_group_core_description(group))); + + group->state = MALI_GROUP_STATE_INACTIVE; + + MALI_DEBUG_ASSERT_POINTER(group->pm_domain); + domains[num_domains] = group->pm_domain; + num_domains++; + + if (mali_group_is_virtual(group)) { + /* Release refs for all child groups */ + struct mali_group *child; + struct mali_group *temp; + + _MALI_OSK_LIST_FOREACHENTRY(child, temp, + &group->group_list, + struct mali_group, group_list) { + child->state = MALI_GROUP_STATE_INACTIVE; + + MALI_DEBUG_ASSERT_POINTER(child->pm_domain); + domains[num_domains] = child->pm_domain; + num_domains++; + + /* Release L2 cache domain for child groups */ + MALI_DEBUG_ASSERT(MALI_MAX_NUM_DOMAIN_REFS > + num_domains); + domains[num_domains] = mali_l2_cache_get_pm_domain( + child->l2_cache_core[0]); + MALI_DEBUG_ASSERT(NULL == child->l2_cache_core[1]); + num_domains++; + } + + /* + * Must do mali_group_power_down() steps right here for + * virtual group, because virtual group itself is likely to + * stay powered on, however child groups are now very likely + * to be powered off (and thus lose their state). + */ + + mali_group_clear_session(group); + /* + * Disable the broadcast unit (clear it's mask). + * This is needed in case the GPU isn't actually + * powered down at this point and groups are + * removed from an inactive virtual group. + * If not, then the broadcast unit will intercept + * their interrupts! + */ + mali_bcast_disable(group->bcast_core); + } else { + /* Release L2 cache domain for physical groups */ + MALI_DEBUG_ASSERT(MALI_MAX_NUM_DOMAIN_REFS > + num_domains); + domains[num_domains] = mali_l2_cache_get_pm_domain( + group->l2_cache_core[0]); + MALI_DEBUG_ASSERT(NULL == group->l2_cache_core[1]); + num_domains++; + } + + power_down = mali_pm_put_domain_refs(domains, num_domains); + + return power_down; +} + +void mali_group_power_up(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + MALI_DEBUG_PRINT(3, ("Group: Power up for %s\n", + mali_group_core_description(group))); + + group->power_is_on = MALI_TRUE; + + if (MALI_FALSE == mali_group_is_virtual(group) + && MALI_FALSE == mali_group_is_in_virtual(group)) { + mali_group_reset(group); + } + + /* + * When we just acquire only one physical group form virt group, + * we should remove the bcast&dlbu mask from virt group and + * reset bcast and dlbu core, although part of pp cores in virt + * group maybe not be powered on. + */ + if (MALI_TRUE == mali_group_is_virtual(group)) { + mali_bcast_reset(group->bcast_core); + mali_dlbu_update_mask(group->dlbu_core); + } +} + +void mali_group_power_down(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT(MALI_TRUE == group->power_is_on); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + MALI_DEBUG_PRINT(3, ("Group: Power down for %s\n", + mali_group_core_description(group))); + + group->power_is_on = MALI_FALSE; + + if (mali_group_is_virtual(group)) { + /* + * What we do for physical jobs in this function should + * already have been done in mali_group_deactivate() + * for virtual group. + */ + MALI_DEBUG_ASSERT(NULL == group->session); + } else { + mali_group_clear_session(group); + } +} + +MALI_DEBUG_CODE(static void mali_group_print_virtual(struct mali_group *vgroup) +{ + u32 i; + struct mali_group *group; + struct mali_group *temp; + + MALI_DEBUG_PRINT(4, ("Virtual group %s (%p)\n", + mali_group_core_description(vgroup), + vgroup)); + MALI_DEBUG_PRINT(4, ("l2_cache_core[0] = %p, ref = %d\n", vgroup->l2_cache_core[0], vgroup->l2_cache_core_ref_count[0])); + MALI_DEBUG_PRINT(4, ("l2_cache_core[1] = %p, ref = %d\n", vgroup->l2_cache_core[1], vgroup->l2_cache_core_ref_count[1])); + + i = 0; + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &vgroup->group_list, struct mali_group, group_list) { + MALI_DEBUG_PRINT(4, ("[%d] %s (%p), l2_cache_core[0] = %p\n", + i, mali_group_core_description(group), + group, group->l2_cache_core[0])); + i++; + } +}) + +static void mali_group_dump_core_status(struct mali_group *group) +{ + u32 i; + + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT(NULL != group->gp_core || (NULL != group->pp_core && !mali_group_is_virtual(group))); + + if (NULL != group->gp_core) { + MALI_PRINT(("Dump Group %s\n", group->gp_core->hw_core.description)); + + for (i = 0; i < 0xA8; i += 0x10) { + MALI_PRINT(("0x%04x: 0x%08x 0x%08x 0x%08x 0x%08x\n", i, mali_hw_core_register_read(&group->gp_core->hw_core, i), + mali_hw_core_register_read(&group->gp_core->hw_core, i + 4), + mali_hw_core_register_read(&group->gp_core->hw_core, i + 8), + mali_hw_core_register_read(&group->gp_core->hw_core, i + 12))); + } + + + } else { + MALI_PRINT(("Dump Group %s\n", group->pp_core->hw_core.description)); + + for (i = 0; i < 0x5c; i += 0x10) { + MALI_PRINT(("0x%04x: 0x%08x 0x%08x 0x%08x 0x%08x\n", i, mali_hw_core_register_read(&group->pp_core->hw_core, i), + mali_hw_core_register_read(&group->pp_core->hw_core, i + 4), + mali_hw_core_register_read(&group->pp_core->hw_core, i + 8), + mali_hw_core_register_read(&group->pp_core->hw_core, i + 12))); + } + + /* Ignore some minor registers */ + for (i = 0x1000; i < 0x1068; i += 0x10) { + MALI_PRINT(("0x%04x: 0x%08x 0x%08x 0x%08x 0x%08x\n", i, mali_hw_core_register_read(&group->pp_core->hw_core, i), + mali_hw_core_register_read(&group->pp_core->hw_core, i + 4), + mali_hw_core_register_read(&group->pp_core->hw_core, i + 8), + mali_hw_core_register_read(&group->pp_core->hw_core, i + 12))); + } + } + + MALI_PRINT(("Dump Group MMU\n")); + for (i = 0; i < 0x24; i += 0x10) { + MALI_PRINT(("0x%04x: 0x%08x 0x%08x 0x%08x 0x%08x\n", i, mali_hw_core_register_read(&group->mmu->hw_core, i), + mali_hw_core_register_read(&group->mmu->hw_core, i + 4), + mali_hw_core_register_read(&group->mmu->hw_core, i + 8), + mali_hw_core_register_read(&group->mmu->hw_core, i + 12))); + } +} + + +/** + * @Dump group status + */ +void mali_group_dump_status(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + + if (mali_group_is_virtual(group)) { + struct mali_group *group_c; + struct mali_group *temp; + _MALI_OSK_LIST_FOREACHENTRY(group_c, temp, &group->group_list, struct mali_group, group_list) { + mali_group_dump_core_status(group_c); + } + } else { + mali_group_dump_core_status(group); + } +} + +/** + * @brief Add child group to virtual group parent + */ +void mali_group_add_group(struct mali_group *parent, struct mali_group *child) +{ + mali_bool found; + u32 i; + + MALI_DEBUG_PRINT(3, ("Adding group %s to virtual group %s\n", + mali_group_core_description(child), + mali_group_core_description(parent))); + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + MALI_DEBUG_ASSERT(mali_group_is_virtual(parent)); + MALI_DEBUG_ASSERT(!mali_group_is_virtual(child)); + MALI_DEBUG_ASSERT(NULL == child->parent_group); + + _mali_osk_list_addtail(&child->group_list, &parent->group_list); + + child->parent_group = parent; + + MALI_DEBUG_ASSERT_POINTER(child->l2_cache_core[0]); + + MALI_DEBUG_PRINT(4, ("parent->l2_cache_core: [0] = %p, [1] = %p\n", parent->l2_cache_core[0], parent->l2_cache_core[1])); + MALI_DEBUG_PRINT(4, ("child->l2_cache_core: [0] = %p, [1] = %p\n", child->l2_cache_core[0], child->l2_cache_core[1])); + + /* Keep track of the L2 cache cores of child groups */ + found = MALI_FALSE; + for (i = 0; i < 2; i++) { + if (parent->l2_cache_core[i] == child->l2_cache_core[0]) { + MALI_DEBUG_ASSERT(parent->l2_cache_core_ref_count[i] > 0); + parent->l2_cache_core_ref_count[i]++; + found = MALI_TRUE; + } + } + + if (!found) { + /* First time we see this L2 cache, add it to our list */ + i = (NULL == parent->l2_cache_core[0]) ? 0 : 1; + + MALI_DEBUG_PRINT(4, ("First time we see l2_cache %p. Adding to [%d] = %p\n", child->l2_cache_core[0], i, parent->l2_cache_core[i])); + + MALI_DEBUG_ASSERT(NULL == parent->l2_cache_core[i]); + + parent->l2_cache_core[i] = child->l2_cache_core[0]; + parent->l2_cache_core_ref_count[i]++; + } + + /* Update Broadcast Unit and DLBU */ + mali_bcast_add_group(parent->bcast_core, child); + mali_dlbu_add_group(parent->dlbu_core, child); + + if (MALI_TRUE == parent->power_is_on) { + mali_bcast_reset(parent->bcast_core); + mali_dlbu_update_mask(parent->dlbu_core); + } + + if (MALI_TRUE == child->power_is_on) { + if (NULL == parent->session) { + if (NULL != child->session) { + /* + * Parent has no session, so clear + * child session as well. + */ + mali_mmu_activate_empty_page_directory(child->mmu); + } + } else { + if (parent->session == child->session) { + /* We already have same session as parent, + * so a simple zap should be enough. + */ + mali_mmu_zap_tlb(child->mmu); + } else { + /* + * Parent has a different session, so we must + * switch to that sessions page table + */ + mali_mmu_activate_page_directory(child->mmu, mali_session_get_page_directory(parent->session)); + } + + /* It is the parent which keeps the session from now on */ + child->session = NULL; + } + } else { + /* should have been cleared when child was powered down */ + MALI_DEBUG_ASSERT(NULL == child->session); + } + + /* Start job on child when parent is active */ + if (NULL != parent->pp_running_job) { + struct mali_pp_job *job = parent->pp_running_job; + + MALI_DEBUG_PRINT(3, ("Group %x joining running job %d on virtual group %x\n", + child, mali_pp_job_get_id(job), parent)); + + /* Only allowed to add active child to an active parent */ + MALI_DEBUG_ASSERT(MALI_GROUP_STATE_ACTIVE == parent->state); + MALI_DEBUG_ASSERT(MALI_GROUP_STATE_ACTIVE == child->state); + + mali_pp_job_start(child->pp_core, job, mali_pp_core_get_id(child->pp_core), MALI_TRUE); + + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | + MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(child->pp_core)) | + MALI_PROFILING_EVENT_REASON_SINGLE_HW_FLUSH, + mali_pp_job_get_frame_builder_id(job), mali_pp_job_get_flush_id(job), 0, 0, 0); + + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | + MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(child->pp_core)) | + MALI_PROFILING_EVENT_REASON_START_STOP_HW_VIRTUAL, + mali_pp_job_get_pid(job), mali_pp_job_get_tid(job), 0, 0, 0); +#if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS) + trace_gpu_sched_switch( + mali_pp_core_description(group->pp_core), + sched_clock(), mali_pp_job_get_tid(job), + 0, mali_pp_job_get_id(job)); +#endif + +#if defined(CONFIG_MALI400_PROFILING) + trace_mali_core_active(mali_pp_job_get_pid(job), 1 /* active */, 0 /* PP */, mali_pp_core_get_id(child->pp_core), + mali_pp_job_get_frame_builder_id(job), mali_pp_job_get_flush_id(job)); +#endif + } + + MALI_DEBUG_CODE(mali_group_print_virtual(parent);) +} + +/** + * @brief Remove child group from virtual group parent + */ +void mali_group_remove_group(struct mali_group *parent, struct mali_group *child) +{ + u32 i; + + MALI_DEBUG_PRINT(3, ("Removing group %s from virtual group %s\n", + mali_group_core_description(child), + mali_group_core_description(parent))); + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + MALI_DEBUG_ASSERT(mali_group_is_virtual(parent)); + MALI_DEBUG_ASSERT(!mali_group_is_virtual(child)); + MALI_DEBUG_ASSERT(parent == child->parent_group); + + /* Update Broadcast Unit and DLBU */ + mali_bcast_remove_group(parent->bcast_core, child); + mali_dlbu_remove_group(parent->dlbu_core, child); + + if (MALI_TRUE == parent->power_is_on) { + mali_bcast_reset(parent->bcast_core); + mali_dlbu_update_mask(parent->dlbu_core); + } + + child->session = parent->session; + child->parent_group = NULL; + + _mali_osk_list_delinit(&child->group_list); + if (_mali_osk_list_empty(&parent->group_list)) { + parent->session = NULL; + } + + /* Keep track of the L2 cache cores of child groups */ + i = (child->l2_cache_core[0] == parent->l2_cache_core[0]) ? 0 : 1; + + MALI_DEBUG_ASSERT(child->l2_cache_core[0] == parent->l2_cache_core[i]); + + parent->l2_cache_core_ref_count[i]--; + if (parent->l2_cache_core_ref_count[i] == 0) { + parent->l2_cache_core[i] = NULL; + } + + MALI_DEBUG_CODE(mali_group_print_virtual(parent)); +} + +struct mali_group *mali_group_acquire_group(struct mali_group *parent) +{ + struct mali_group *child = NULL; + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + MALI_DEBUG_ASSERT(mali_group_is_virtual(parent)); + + if (!_mali_osk_list_empty(&parent->group_list)) { + child = _MALI_OSK_LIST_ENTRY(parent->group_list.prev, struct mali_group, group_list); + mali_group_remove_group(parent, child); + } + + if (NULL != child) { + if (MALI_GROUP_STATE_ACTIVE != parent->state + && MALI_TRUE == child->power_is_on) { + mali_group_reset(child); + } + } + + return child; +} + +void mali_group_reset(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + MALI_DEBUG_ASSERT(NULL == group->gp_running_job); + MALI_DEBUG_ASSERT(NULL == group->pp_running_job); + MALI_DEBUG_ASSERT(NULL == group->session); + + MALI_DEBUG_PRINT(3, ("Group: reset of %s\n", + mali_group_core_description(group))); + + if (NULL != group->dlbu_core) { + mali_dlbu_reset(group->dlbu_core); + } + + if (NULL != group->bcast_core) { + mali_bcast_reset(group->bcast_core); + } + + MALI_DEBUG_ASSERT(NULL != group->mmu); + mali_group_reset_mmu(group); + + if (NULL != group->gp_core) { + MALI_DEBUG_ASSERT(NULL == group->pp_core); + mali_gp_reset(group->gp_core); + } else { + MALI_DEBUG_ASSERT(NULL != group->pp_core); + mali_group_reset_pp(group); + } +} + +void mali_group_start_gp_job(struct mali_group *group, struct mali_gp_job *job) +{ + struct mali_session_data *session; + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + MALI_DEBUG_PRINT(3, ("Group: Starting GP job 0x%08X on group %s\n", + job, + mali_group_core_description(group))); + + session = mali_gp_job_get_session(job); + + MALI_DEBUG_ASSERT_POINTER(group->l2_cache_core[0]); + mali_l2_cache_invalidate_conditional(group->l2_cache_core[0], mali_gp_job_get_cache_order(job)); + + mali_group_activate_page_directory(group, session); + + mali_gp_job_start(group->gp_core, job); + + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | + MALI_PROFILING_MAKE_EVENT_CHANNEL_GP(0) | + MALI_PROFILING_EVENT_REASON_SINGLE_HW_FLUSH, + mali_gp_job_get_frame_builder_id(job), mali_gp_job_get_flush_id(job), 0, 0, 0); + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | + MALI_PROFILING_MAKE_EVENT_CHANNEL_GP(0), + mali_gp_job_get_pid(job), mali_gp_job_get_tid(job), 0, 0, 0); + +#if defined(CONFIG_MALI400_PROFILING) + trace_mali_core_active(mali_gp_job_get_pid(job), 1 /* active */, 1 /* GP */, 0 /* core */, + mali_gp_job_get_frame_builder_id(job), mali_gp_job_get_flush_id(job)); +#endif + +#if defined(CONFIG_MALI400_PROFILING) + if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[0])) && + (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[0]))) { + mali_group_report_l2_cache_counters_per_core(group, 0); + } +#endif /* #if defined(CONFIG_MALI400_PROFILING) */ + +#if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS) + trace_gpu_sched_switch(mali_gp_core_description(group->gp_core), + sched_clock(), mali_gp_job_get_tid(job), + 0, mali_gp_job_get_id(job)); +#endif + + group->gp_running_job = job; + group->is_working = MALI_TRUE; + + /* Setup SW timer and record start time */ + group->start_time = _mali_osk_time_tickcount(); + _mali_osk_timer_mod(group->timeout_timer, _mali_osk_time_mstoticks(mali_max_job_runtime)); + + MALI_DEBUG_PRINT(4, ("Group: Started GP job 0x%08X on group %s at %u\n", + job, + mali_group_core_description(group), + group->start_time)); +} + +/* Used to set all the registers except frame renderer list address and fragment shader stack address + * It means the caller must set these two registers properly before calling this function + */ +void mali_group_start_pp_job(struct mali_group *group, struct mali_pp_job *job, u32 sub_job) +{ + struct mali_session_data *session; + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + MALI_DEBUG_PRINT(3, ("Group: Starting PP job 0x%08X part %u/%u on group %s\n", + job, sub_job + 1, + mali_pp_job_get_sub_job_count(job), + mali_group_core_description(group))); + + session = mali_pp_job_get_session(job); + + if (NULL != group->l2_cache_core[0]) { + mali_l2_cache_invalidate_conditional(group->l2_cache_core[0], mali_pp_job_get_cache_order(job)); + } + + if (NULL != group->l2_cache_core[1]) { + mali_l2_cache_invalidate_conditional(group->l2_cache_core[1], mali_pp_job_get_cache_order(job)); + } + + mali_group_activate_page_directory(group, session); + + if (mali_group_is_virtual(group)) { + struct mali_group *child; + struct mali_group *temp; + u32 core_num = 0; + + MALI_DEBUG_ASSERT(mali_pp_job_is_virtual(job)); + + /* Configure DLBU for the job */ + mali_dlbu_config_job(group->dlbu_core, job); + + /* Write stack address for each child group */ + _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list) { + mali_pp_write_addr_stack(child->pp_core, job); + core_num++; + } + + mali_pp_job_start(group->pp_core, job, sub_job, MALI_FALSE); + } else { + mali_pp_job_start(group->pp_core, job, sub_job, MALI_FALSE); + } + + /* if the group is virtual, loop through physical groups which belong to this group + * and call profiling events for its cores as virtual */ + if (MALI_TRUE == mali_group_is_virtual(group)) { + struct mali_group *child; + struct mali_group *temp; + + _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list) { + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | + MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(child->pp_core)) | + MALI_PROFILING_EVENT_REASON_SINGLE_HW_FLUSH, + mali_pp_job_get_frame_builder_id(job), mali_pp_job_get_flush_id(job), 0, 0, 0); + + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | + MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(child->pp_core)) | + MALI_PROFILING_EVENT_REASON_START_STOP_HW_VIRTUAL, + mali_pp_job_get_pid(job), mali_pp_job_get_tid(job), 0, 0, 0); + +#if defined(CONFIG_MALI400_PROFILING) + trace_mali_core_active(mali_pp_job_get_pid(job), 1 /* active */, 0 /* PP */, mali_pp_core_get_id(child->pp_core), + mali_pp_job_get_frame_builder_id(job), mali_pp_job_get_flush_id(job)); +#endif + } + +#if defined(CONFIG_MALI400_PROFILING) + if (0 != group->l2_cache_core_ref_count[0]) { + if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[0])) && + (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[0]))) { + mali_group_report_l2_cache_counters_per_core(group, mali_l2_cache_get_id(group->l2_cache_core[0])); + } + } + if (0 != group->l2_cache_core_ref_count[1]) { + if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[1])) && + (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[1]))) { + mali_group_report_l2_cache_counters_per_core(group, mali_l2_cache_get_id(group->l2_cache_core[1])); + } + } +#endif /* #if defined(CONFIG_MALI400_PROFILING) */ + + } else { /* group is physical - call profiling events for physical cores */ + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | + MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(group->pp_core)) | + MALI_PROFILING_EVENT_REASON_SINGLE_HW_FLUSH, + mali_pp_job_get_frame_builder_id(job), mali_pp_job_get_flush_id(job), 0, 0, 0); + + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | + MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(group->pp_core)) | + MALI_PROFILING_EVENT_REASON_START_STOP_HW_PHYSICAL, + mali_pp_job_get_pid(job), mali_pp_job_get_tid(job), 0, 0, 0); + +#if defined(CONFIG_MALI400_PROFILING) + trace_mali_core_active(mali_pp_job_get_pid(job), 1 /* active */, 0 /* PP */, mali_pp_core_get_id(group->pp_core), + mali_pp_job_get_frame_builder_id(job), mali_pp_job_get_flush_id(job)); +#endif + +#if defined(CONFIG_MALI400_PROFILING) + if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[0])) && + (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[0]))) { + mali_group_report_l2_cache_counters_per_core(group, mali_l2_cache_get_id(group->l2_cache_core[0])); + } +#endif /* #if defined(CONFIG_MALI400_PROFILING) */ + } + +#if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS) + trace_gpu_sched_switch(mali_pp_core_description(group->pp_core), + sched_clock(), mali_pp_job_get_tid(job), + 0, mali_pp_job_get_id(job)); +#endif + + group->pp_running_job = job; + group->pp_running_sub_job = sub_job; + group->is_working = MALI_TRUE; + + /* Setup SW timer and record start time */ + group->start_time = _mali_osk_time_tickcount(); + _mali_osk_timer_mod(group->timeout_timer, _mali_osk_time_mstoticks(mali_max_job_runtime)); + + MALI_DEBUG_PRINT(4, ("Group: Started PP job 0x%08X part %u/%u on group %s at %u\n", + job, sub_job + 1, + mali_pp_job_get_sub_job_count(job), + mali_group_core_description(group), + group->start_time)); + +} + +void mali_group_resume_gp_with_new_heap(struct mali_group *group, u32 job_id, u32 start_addr, u32 end_addr) +{ + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + MALI_DEBUG_ASSERT_POINTER(group->l2_cache_core[0]); + mali_l2_cache_invalidate(group->l2_cache_core[0]); + + mali_mmu_zap_tlb_without_stall(group->mmu); + + mali_gp_resume_with_new_heap(group->gp_core, start_addr, end_addr); + + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_RESUME | + MALI_PROFILING_MAKE_EVENT_CHANNEL_GP(0), + 0, 0, 0, 0, 0); + +#if defined(CONFIG_MALI400_PROFILING) + trace_mali_core_active(mali_gp_job_get_pid(group->gp_running_job), 1 /* active */, 1 /* GP */, 0 /* core */, + mali_gp_job_get_frame_builder_id(group->gp_running_job), mali_gp_job_get_flush_id(group->gp_running_job)); +#endif +} + +static void mali_group_reset_mmu(struct mali_group *group) +{ + struct mali_group *child; + struct mali_group *temp; + _mali_osk_errcode_t err; + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + if (!mali_group_is_virtual(group)) { + /* This is a physical group or an idle virtual group -- simply wait for + * the reset to complete. */ + err = mali_mmu_reset(group->mmu); + MALI_DEBUG_ASSERT(_MALI_OSK_ERR_OK == err); + } else { /* virtual group */ + /* Loop through all members of this virtual group and wait + * until they are done resetting. + */ + _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list) { + err = mali_mmu_reset(child->mmu); + MALI_DEBUG_ASSERT(_MALI_OSK_ERR_OK == err); + } + } +} + +static void mali_group_reset_pp(struct mali_group *group) +{ + struct mali_group *child; + struct mali_group *temp; + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + mali_pp_reset_async(group->pp_core); + + if (!mali_group_is_virtual(group) || NULL == group->pp_running_job) { + /* This is a physical group or an idle virtual group -- simply wait for + * the reset to complete. */ + mali_pp_reset_wait(group->pp_core); + } else { + /* Loop through all members of this virtual group and wait until they + * are done resetting. + */ + _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list) { + mali_pp_reset_wait(child->pp_core); + } + } +} + +struct mali_pp_job *mali_group_complete_pp(struct mali_group *group, mali_bool success, u32 *sub_job) +{ + struct mali_pp_job *pp_job_to_return; + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->pp_core); + MALI_DEBUG_ASSERT_POINTER(group->pp_running_job); + MALI_DEBUG_ASSERT_POINTER(sub_job); + MALI_DEBUG_ASSERT(MALI_TRUE == group->is_working); + + /* Stop/clear the timeout timer. */ + _mali_osk_timer_del_async(group->timeout_timer); + + if (NULL != group->pp_running_job) { + + /* Deal with HW counters and profiling */ + + if (MALI_TRUE == mali_group_is_virtual(group)) { + struct mali_group *child; + struct mali_group *temp; + + /* update performance counters from each physical pp core within this virtual group */ + _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list) { + mali_pp_update_performance_counters(group->pp_core, child->pp_core, group->pp_running_job, mali_pp_core_get_id(child->pp_core)); + } + +#if defined(CONFIG_MALI400_PROFILING) + /* send profiling data per physical core */ + _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list) { + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | + MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(child->pp_core)) | + MALI_PROFILING_EVENT_REASON_START_STOP_HW_VIRTUAL, + mali_pp_job_get_perf_counter_value0(group->pp_running_job, mali_pp_core_get_id(child->pp_core)), + mali_pp_job_get_perf_counter_value1(group->pp_running_job, mali_pp_core_get_id(child->pp_core)), + mali_pp_job_get_perf_counter_src0(group->pp_running_job, group->pp_running_sub_job) | (mali_pp_job_get_perf_counter_src1(group->pp_running_job, group->pp_running_sub_job) << 8), + 0, 0); + + trace_mali_core_active(mali_pp_job_get_pid(group->pp_running_job), + 0 /* active */, 0 /* PP */, mali_pp_core_get_id(child->pp_core), + mali_pp_job_get_frame_builder_id(group->pp_running_job), + mali_pp_job_get_flush_id(group->pp_running_job)); + } + if (0 != group->l2_cache_core_ref_count[0]) { + if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[0])) && + (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[0]))) { + mali_group_report_l2_cache_counters_per_core(group, mali_l2_cache_get_id(group->l2_cache_core[0])); + } + } + if (0 != group->l2_cache_core_ref_count[1]) { + if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[1])) && + (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[1]))) { + mali_group_report_l2_cache_counters_per_core(group, mali_l2_cache_get_id(group->l2_cache_core[1])); + } + } + +#endif + } else { + /* update performance counters for a physical group's pp core */ + mali_pp_update_performance_counters(group->pp_core, group->pp_core, group->pp_running_job, group->pp_running_sub_job); + +#if defined(CONFIG_MALI400_PROFILING) + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | + MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(group->pp_core)) | + MALI_PROFILING_EVENT_REASON_START_STOP_HW_PHYSICAL, + mali_pp_job_get_perf_counter_value0(group->pp_running_job, group->pp_running_sub_job), + mali_pp_job_get_perf_counter_value1(group->pp_running_job, group->pp_running_sub_job), + mali_pp_job_get_perf_counter_src0(group->pp_running_job, group->pp_running_sub_job) | (mali_pp_job_get_perf_counter_src1(group->pp_running_job, group->pp_running_sub_job) << 8), + 0, 0); + + trace_mali_core_active(mali_pp_job_get_pid(group->pp_running_job), + 0 /* active */, 0 /* PP */, mali_pp_core_get_id(group->pp_core), + mali_pp_job_get_frame_builder_id(group->pp_running_job), + mali_pp_job_get_flush_id(group->pp_running_job)); + + if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[0])) && + (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[0]))) { + mali_group_report_l2_cache_counters_per_core(group, mali_l2_cache_get_id(group->l2_cache_core[0])); + } +#endif + } + +#if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS) + trace_gpu_sched_switch( + mali_gp_core_description(group->gp_core), + sched_clock(), 0, 0, 0); +#endif + + } + + if (success) { + /* Only do soft reset for successful jobs, a full recovery + * reset will be done for failed jobs. */ + mali_pp_reset_async(group->pp_core); + } + + pp_job_to_return = group->pp_running_job; + group->pp_running_job = NULL; + group->is_working = MALI_FALSE; + *sub_job = group->pp_running_sub_job; + + if (!success) { + MALI_DEBUG_PRINT(2, ("Mali group: Executing recovery reset due to job failure\n")); + mali_group_recovery_reset(group); + } else if (_MALI_OSK_ERR_OK != mali_pp_reset_wait(group->pp_core)) { + MALI_PRINT_ERROR(("Mali group: Executing recovery reset due to reset failure\n")); + mali_group_recovery_reset(group); + } + + return pp_job_to_return; +} + +struct mali_gp_job *mali_group_complete_gp(struct mali_group *group, mali_bool success) +{ + struct mali_gp_job *gp_job_to_return; + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->gp_core); + MALI_DEBUG_ASSERT_POINTER(group->gp_running_job); + MALI_DEBUG_ASSERT(MALI_TRUE == group->is_working); + + /* Stop/clear the timeout timer. */ + _mali_osk_timer_del_async(group->timeout_timer); + + if (NULL != group->gp_running_job) { + mali_gp_update_performance_counters(group->gp_core, group->gp_running_job); + +#if defined(CONFIG_MALI400_PROFILING) + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | MALI_PROFILING_MAKE_EVENT_CHANNEL_GP(0), + mali_gp_job_get_perf_counter_value0(group->gp_running_job), + mali_gp_job_get_perf_counter_value1(group->gp_running_job), + mali_gp_job_get_perf_counter_src0(group->gp_running_job) | (mali_gp_job_get_perf_counter_src1(group->gp_running_job) << 8), + 0, 0); + + if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[0])) && + (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[0]))) + mali_group_report_l2_cache_counters_per_core(group, 0); +#endif + +#if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS) + trace_gpu_sched_switch( + mali_pp_core_description(group->pp_core), + sched_clock(), 0, 0, 0); +#endif + +#if defined(CONFIG_MALI400_PROFILING) + trace_mali_core_active(mali_gp_job_get_pid(group->gp_running_job), 0 /* active */, 1 /* GP */, 0 /* core */, + mali_gp_job_get_frame_builder_id(group->gp_running_job), mali_gp_job_get_flush_id(group->gp_running_job)); +#endif + + mali_gp_job_set_current_heap_addr(group->gp_running_job, + mali_gp_read_plbu_alloc_start_addr(group->gp_core)); + } + + if (success) { + /* Only do soft reset for successful jobs, a full recovery + * reset will be done for failed jobs. */ + mali_gp_reset_async(group->gp_core); + } + + gp_job_to_return = group->gp_running_job; + group->gp_running_job = NULL; + group->is_working = MALI_FALSE; + + if (!success) { + MALI_DEBUG_PRINT(2, ("Mali group: Executing recovery reset due to job failure\n")); + mali_group_recovery_reset(group); + } else if (_MALI_OSK_ERR_OK != mali_gp_reset_wait(group->gp_core)) { + MALI_PRINT_ERROR(("Mali group: Executing recovery reset due to reset failure\n")); + mali_group_recovery_reset(group); + } + + return gp_job_to_return; +} + +struct mali_group *mali_group_get_glob_group(u32 index) +{ + if (mali_global_num_groups > index) { + return mali_global_groups[index]; + } + + return NULL; +} + +u32 mali_group_get_glob_num_groups(void) +{ + return mali_global_num_groups; +} + +static void mali_group_activate_page_directory(struct mali_group *group, struct mali_session_data *session) +{ + MALI_DEBUG_PRINT(5, ("Mali group: Activating page directory 0x%08X from session 0x%08X on group %s\n", + mali_session_get_page_directory(session), session, + mali_group_core_description(group))); + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + if (group->session != session) { + /* Different session than last time, so we need to do some work */ + MALI_DEBUG_PRINT(5, ("Mali group: Activate session: %08x previous: %08x on group %s\n", + session, group->session, + mali_group_core_description(group))); + mali_mmu_activate_page_directory(group->mmu, mali_session_get_page_directory(session)); + group->session = session; + } else { + /* Same session as last time, so no work required */ + MALI_DEBUG_PRINT(4, ("Mali group: Activate existing session 0x%08X on group %s\n", + session->page_directory, + mali_group_core_description(group))); + mali_mmu_zap_tlb_without_stall(group->mmu); + } +} + +static void mali_group_recovery_reset(struct mali_group *group) +{ + _mali_osk_errcode_t err; + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + /* Stop cores, bus stop */ + if (NULL != group->pp_core) { + mali_pp_stop_bus(group->pp_core); + } else { + mali_gp_stop_bus(group->gp_core); + } + + /* Flush MMU and clear page fault (if any) */ + mali_mmu_activate_fault_flush_page_directory(group->mmu); + mali_mmu_page_fault_done(group->mmu); + + /* Wait for cores to stop bus, then do a hard reset on them */ + if (NULL != group->pp_core) { + if (mali_group_is_virtual(group)) { + struct mali_group *child, *temp; + + /* Disable the broadcast unit while we do reset directly on the member cores. */ + mali_bcast_disable(group->bcast_core); + + _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list) { + mali_pp_stop_bus_wait(child->pp_core); + mali_pp_hard_reset(child->pp_core); + } + + mali_bcast_enable(group->bcast_core); + } else { + mali_pp_stop_bus_wait(group->pp_core); + mali_pp_hard_reset(group->pp_core); + } + } else { + mali_gp_stop_bus_wait(group->gp_core); + mali_gp_hard_reset(group->gp_core); + } + + /* Reset MMU */ + err = mali_mmu_reset(group->mmu); + MALI_DEBUG_ASSERT(_MALI_OSK_ERR_OK == err); + MALI_IGNORE(err); + + group->session = NULL; +} + +#if MALI_STATE_TRACKING +u32 mali_group_dump_state(struct mali_group *group, char *buf, u32 size) +{ + int n = 0; + int i; + struct mali_group *child; + struct mali_group *temp; + + if (mali_group_is_virtual(group)) { + n += _mali_osk_snprintf(buf + n, size - n, + "Virtual PP Group: %p\n", group); + } else if (mali_group_is_in_virtual(group)) { + n += _mali_osk_snprintf(buf + n, size - n, + "Child PP Group: %p\n", group); + } else if (NULL != group->pp_core) { + n += _mali_osk_snprintf(buf + n, size - n, + "Physical PP Group: %p\n", group); + } else { + MALI_DEBUG_ASSERT_POINTER(group->gp_core); + n += _mali_osk_snprintf(buf + n, size - n, + "GP Group: %p\n", group); + } + + switch (group->state) { + case MALI_GROUP_STATE_INACTIVE: + n += _mali_osk_snprintf(buf + n, size - n, + "\tstate: INACTIVE\n"); + break; + case MALI_GROUP_STATE_ACTIVATION_PENDING: + n += _mali_osk_snprintf(buf + n, size - n, + "\tstate: ACTIVATION_PENDING\n"); + break; + case MALI_GROUP_STATE_ACTIVE: + n += _mali_osk_snprintf(buf + n, size - n, + "\tstate: MALI_GROUP_STATE_ACTIVE\n"); + break; + default: + n += _mali_osk_snprintf(buf + n, size - n, + "\tstate: UNKNOWN (%d)\n", group->state); + MALI_DEBUG_ASSERT(0); + break; + } + + n += _mali_osk_snprintf(buf + n, size - n, + "\tSW power: %s\n", + group->power_is_on ? "On" : "Off"); + + n += mali_pm_dump_state_domain(group->pm_domain, buf + n, size - n); + + for (i = 0; i < 2; i++) { + if (NULL != group->l2_cache_core[i]) { + struct mali_pm_domain *domain; + domain = mali_l2_cache_get_pm_domain( + group->l2_cache_core[i]); + n += mali_pm_dump_state_domain(domain, + buf + n, size - n); + } + } + + if (group->gp_core) { + n += mali_gp_dump_state(group->gp_core, buf + n, size - n); + n += _mali_osk_snprintf(buf + n, size - n, + "\tGP running job: %p\n", group->gp_running_job); + } + + if (group->pp_core) { + n += mali_pp_dump_state(group->pp_core, buf + n, size - n); + n += _mali_osk_snprintf(buf + n, size - n, + "\tPP running job: %p, subjob %d \n", + group->pp_running_job, + group->pp_running_sub_job); + } + + _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, + struct mali_group, group_list) { + n += mali_group_dump_state(child, buf + n, size - n); + } + + return n; +} +#endif + +_mali_osk_errcode_t mali_group_upper_half_mmu(void *data) +{ + struct mali_group *group = (struct mali_group *)data; + _mali_osk_errcode_t ret; + + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->mmu); + +#if defined(CONFIG_MALI400_PROFILING) && defined (CONFIG_TRACEPOINTS) +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + mali_executor_lock(); + if (!mali_group_is_working(group)) { + /* Not working, so nothing to do */ + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } +#endif + if (NULL != group->gp_core) { + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF, + 0, 0, /* No pid and tid for interrupt handler */ + MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP_MMU(0), + mali_mmu_get_rawstat(group->mmu), 0); + } else { + MALI_DEBUG_ASSERT_POINTER(group->pp_core); + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF, + 0, 0, /* No pid and tid for interrupt handler */ + MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP_MMU( + mali_pp_core_get_id(group->pp_core)), + mali_mmu_get_rawstat(group->mmu), 0); + } +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + mali_executor_unlock(); +#endif +#endif + + ret = mali_executor_interrupt_mmu(group, MALI_TRUE); + +#if defined(CONFIG_MALI400_PROFILING) && defined (CONFIG_TRACEPOINTS) +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + mali_executor_lock(); + if (!mali_group_is_working(group)) { + /* Not working, so nothing to do */ + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } +#endif + + if (NULL != group->gp_core) { + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF, + 0, 0, /* No pid and tid for interrupt handler */ + MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP_MMU(0), + mali_mmu_get_rawstat(group->mmu), 0); + } else { + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF, + 0, 0, /* No pid and tid for interrupt handler */ + MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP_MMU( + mali_pp_core_get_id(group->pp_core)), + mali_mmu_get_rawstat(group->mmu), 0); + } +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + mali_executor_unlock(); +#endif +#endif + + return ret; +} + +static void mali_group_bottom_half_mmu(void *data) +{ + struct mali_group *group = (struct mali_group *)data; + + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->mmu); + + if (NULL != group->gp_core) { + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF, + 0, _mali_osk_get_tid(), /* pid and tid */ + MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP_MMU(0), + mali_mmu_get_rawstat(group->mmu), 0); + } else { + MALI_DEBUG_ASSERT_POINTER(group->pp_core); + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF, + 0, _mali_osk_get_tid(), /* pid and tid */ + MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP_MMU( + mali_pp_core_get_id(group->pp_core)), + mali_mmu_get_rawstat(group->mmu), 0); + } + + mali_executor_interrupt_mmu(group, MALI_FALSE); + + if (NULL != group->gp_core) { + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF, + 0, _mali_osk_get_tid(), /* pid and tid */ + MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP_MMU(0), + mali_mmu_get_rawstat(group->mmu), 0); + } else { + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF, + 0, _mali_osk_get_tid(), /* pid and tid */ + MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP_MMU( + mali_pp_core_get_id(group->pp_core)), + mali_mmu_get_rawstat(group->mmu), 0); + } +} + +_mali_osk_errcode_t mali_group_upper_half_gp(void *data) +{ + struct mali_group *group = (struct mali_group *)data; + _mali_osk_errcode_t ret; + + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->gp_core); + MALI_DEBUG_ASSERT_POINTER(group->mmu); + +#if defined(CONFIG_MALI400_PROFILING) && defined (CONFIG_TRACEPOINTS) +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + mali_executor_lock(); + if (!mali_group_is_working(group)) { + /* Not working, so nothing to do */ + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } +#endif + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF, + 0, 0, /* No pid and tid for interrupt handler */ + MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP(0), + mali_gp_get_rawstat(group->gp_core), 0); + + MALI_DEBUG_PRINT(4, ("Group: Interrupt 0x%08X from %s\n", + mali_gp_get_rawstat(group->gp_core), + mali_group_core_description(group))); +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + mali_executor_unlock(); +#endif +#endif + ret = mali_executor_interrupt_gp(group, MALI_TRUE); + +#if defined(CONFIG_MALI400_PROFILING) && defined (CONFIG_TRACEPOINTS) +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + mali_executor_lock(); + if (!mali_group_is_working(group)) { + /* Not working, so nothing to do */ + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } +#endif + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF, + 0, 0, /* No pid and tid for interrupt handler */ + MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP(0), + mali_gp_get_rawstat(group->gp_core), 0); +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + mali_executor_unlock(); +#endif +#endif + return ret; +} + +static void mali_group_bottom_half_gp(void *data) +{ + struct mali_group *group = (struct mali_group *)data; + + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->gp_core); + MALI_DEBUG_ASSERT_POINTER(group->mmu); + + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF, + 0, _mali_osk_get_tid(), /* pid and tid */ + MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP(0), + mali_gp_get_rawstat(group->gp_core), 0); + + mali_executor_interrupt_gp(group, MALI_FALSE); + + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF, + 0, _mali_osk_get_tid(), /* pid and tid */ + MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP(0), + mali_gp_get_rawstat(group->gp_core), 0); +} + +_mali_osk_errcode_t mali_group_upper_half_pp(void *data) +{ + struct mali_group *group = (struct mali_group *)data; + _mali_osk_errcode_t ret; + + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->pp_core); + MALI_DEBUG_ASSERT_POINTER(group->mmu); + +#if defined(CONFIG_MALI400_PROFILING) && defined (CONFIG_TRACEPOINTS) +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + mali_executor_lock(); + if (!mali_group_is_working(group)) { + /* Not working, so nothing to do */ + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } +#endif + + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF, + 0, 0, /* No pid and tid for interrupt handler */ + MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP( + mali_pp_core_get_id(group->pp_core)), + mali_pp_get_rawstat(group->pp_core), 0); + + MALI_DEBUG_PRINT(4, ("Group: Interrupt 0x%08X from %s\n", + mali_pp_get_rawstat(group->pp_core), + mali_group_core_description(group))); +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + mali_executor_unlock(); +#endif +#endif + + ret = mali_executor_interrupt_pp(group, MALI_TRUE); + +#if defined(CONFIG_MALI400_PROFILING) && defined (CONFIG_TRACEPOINTS) +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + mali_executor_lock(); + if (!mali_group_is_working(group)) { + /* Not working, so nothing to do */ + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } +#endif + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF, + 0, 0, /* No pid and tid for interrupt handler */ + MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP( + mali_pp_core_get_id(group->pp_core)), + mali_pp_get_rawstat(group->pp_core), 0); +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + mali_executor_unlock(); +#endif +#endif + return ret; +} + +static void mali_group_bottom_half_pp(void *data) +{ + struct mali_group *group = (struct mali_group *)data; + + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->pp_core); + MALI_DEBUG_ASSERT_POINTER(group->mmu); + + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF, + 0, _mali_osk_get_tid(), /* pid and tid */ + MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP( + mali_pp_core_get_id(group->pp_core)), + mali_pp_get_rawstat(group->pp_core), 0); + + mali_executor_interrupt_pp(group, MALI_FALSE); + + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF, + 0, _mali_osk_get_tid(), /* pid and tid */ + MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP( + mali_pp_core_get_id(group->pp_core)), + mali_pp_get_rawstat(group->pp_core), 0); +} + +static void mali_group_timeout(void *data) +{ + struct mali_group *group = (struct mali_group *)data; + MALI_DEBUG_ASSERT_POINTER(group); + + MALI_DEBUG_PRINT(2, ("Group: timeout handler for %s at %u\n", + mali_group_core_description(group), + _mali_osk_time_tickcount())); + + if (NULL != group->gp_core) { + mali_group_schedule_bottom_half_gp(group); + } else { + MALI_DEBUG_ASSERT_POINTER(group->pp_core); + mali_group_schedule_bottom_half_pp(group); + } +} + +static void mali_group_out_of_memory(void *data) +{ + struct mali_group *group = (struct mali_group *)data; + + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->gp_core); + MALI_DEBUG_ASSERT_POINTER(group->mmu); + + mali_executor_group_oom(group); +} + +mali_bool mali_group_zap_session(struct mali_group *group, + struct mali_session_data *session) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(session); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + if (group->session != session) { + /* not running from this session */ + return MALI_TRUE; /* success */ + } + + if (group->is_working) { + /* The Zap also does the stall and disable_stall */ + mali_bool zap_success = mali_mmu_zap_tlb(group->mmu); + return zap_success; + } else { + /* Just remove the session instead of zapping */ + mali_group_clear_session(group); + return MALI_TRUE; /* success */ + } +} + +#if defined(CONFIG_MALI400_PROFILING) +static void mali_group_report_l2_cache_counters_per_core(struct mali_group *group, u32 core_num) +{ + u32 source0 = 0; + u32 value0 = 0; + u32 source1 = 0; + u32 value1 = 0; + u32 profiling_channel = 0; + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + switch (core_num) { + case 0: + profiling_channel = MALI_PROFILING_EVENT_TYPE_SINGLE | + MALI_PROFILING_EVENT_CHANNEL_GPU | + MALI_PROFILING_EVENT_REASON_SINGLE_GPU_L20_COUNTERS; + break; + case 1: + profiling_channel = MALI_PROFILING_EVENT_TYPE_SINGLE | + MALI_PROFILING_EVENT_CHANNEL_GPU | + MALI_PROFILING_EVENT_REASON_SINGLE_GPU_L21_COUNTERS; + break; + case 2: + profiling_channel = MALI_PROFILING_EVENT_TYPE_SINGLE | + MALI_PROFILING_EVENT_CHANNEL_GPU | + MALI_PROFILING_EVENT_REASON_SINGLE_GPU_L22_COUNTERS; + break; + default: + profiling_channel = MALI_PROFILING_EVENT_TYPE_SINGLE | + MALI_PROFILING_EVENT_CHANNEL_GPU | + MALI_PROFILING_EVENT_REASON_SINGLE_GPU_L20_COUNTERS; + break; + } + + if (0 == core_num) { + mali_l2_cache_core_get_counter_values(group->l2_cache_core[0], &source0, &value0, &source1, &value1); + } + if (1 == core_num) { + if (1 == mali_l2_cache_get_id(group->l2_cache_core[0])) { + mali_l2_cache_core_get_counter_values(group->l2_cache_core[0], &source0, &value0, &source1, &value1); + } else if (1 == mali_l2_cache_get_id(group->l2_cache_core[1])) { + mali_l2_cache_core_get_counter_values(group->l2_cache_core[1], &source0, &value0, &source1, &value1); + } + } + if (2 == core_num) { + if (2 == mali_l2_cache_get_id(group->l2_cache_core[0])) { + mali_l2_cache_core_get_counter_values(group->l2_cache_core[0], &source0, &value0, &source1, &value1); + } else if (2 == mali_l2_cache_get_id(group->l2_cache_core[1])) { + mali_l2_cache_core_get_counter_values(group->l2_cache_core[1], &source0, &value0, &source1, &value1); + } + } + + _mali_osk_profiling_add_event(profiling_channel, source1 << 8 | source0, value0, value1, 0, 0); +} +#endif /* #if defined(CONFIG_MALI400_PROFILING) */ diff --git a/drivers/gpu/arm/utgard/common/mali_group.h b/drivers/gpu/arm/utgard/common/mali_group.h new file mode 100644 index 00000000000000..705605ec1d9f42 --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_group.h @@ -0,0 +1,467 @@ +/* + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_GROUP_H__ +#define __MALI_GROUP_H__ + +#include "mali_osk.h" +#include "mali_l2_cache.h" +#include "mali_mmu.h" +#include "mali_gp.h" +#include "mali_pp.h" +#include "mali_session.h" +#include "mali_osk_profiling.h" + +/** + * @brief Default max runtime [ms] for a core job - used by timeout timers + */ +#define MALI_MAX_JOB_RUNTIME_DEFAULT 5000 + +extern int mali_max_job_runtime; + +#define MALI_MAX_NUMBER_OF_GROUPS 10 +#define MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS 8 + +enum mali_group_state { + MALI_GROUP_STATE_INACTIVE, + MALI_GROUP_STATE_ACTIVATION_PENDING, + MALI_GROUP_STATE_ACTIVE, +}; + +/** + * The structure represents a render group + * A render group is defined by all the cores that share the same Mali MMU + */ + +struct mali_group { + struct mali_mmu_core *mmu; + struct mali_session_data *session; + + enum mali_group_state state; + mali_bool power_is_on; + + mali_bool is_working; + unsigned long start_time; /* in ticks */ + + struct mali_gp_core *gp_core; + struct mali_gp_job *gp_running_job; + + struct mali_pp_core *pp_core; + struct mali_pp_job *pp_running_job; + u32 pp_running_sub_job; + + struct mali_pm_domain *pm_domain; + + struct mali_l2_cache_core *l2_cache_core[2]; + u32 l2_cache_core_ref_count[2]; + + /* Parent virtual group (if any) */ + struct mali_group *parent_group; + + struct mali_dlbu_core *dlbu_core; + struct mali_bcast_unit *bcast_core; + + /* Used for working groups which needs to be disabled */ + mali_bool disable_requested; + + /* Used by group to link child groups (for virtual group) */ + _mali_osk_list_t group_list; + + /* Used by executor module in order to link groups of same state */ + _mali_osk_list_t executor_list; + + /* Used by PM domains to link groups of same domain */ + _mali_osk_list_t pm_domain_list; + + _mali_osk_wq_work_t *bottom_half_work_mmu; + _mali_osk_wq_work_t *bottom_half_work_gp; + _mali_osk_wq_work_t *bottom_half_work_pp; + + _mali_osk_wq_work_t *oom_work_handler; + _mali_osk_timer_t *timeout_timer; +}; + +/** @brief Create a new Mali group object + * + * @return A pointer to a new group object + */ +struct mali_group *mali_group_create(struct mali_l2_cache_core *core, + struct mali_dlbu_core *dlbu, + struct mali_bcast_unit *bcast, + u32 domain_index); + +void mali_group_dump_status(struct mali_group *group); + +void mali_group_delete(struct mali_group *group); + +_mali_osk_errcode_t mali_group_add_mmu_core(struct mali_group *group, + struct mali_mmu_core *mmu_core); +void mali_group_remove_mmu_core(struct mali_group *group); + +_mali_osk_errcode_t mali_group_add_gp_core(struct mali_group *group, + struct mali_gp_core *gp_core); +void mali_group_remove_gp_core(struct mali_group *group); + +_mali_osk_errcode_t mali_group_add_pp_core(struct mali_group *group, + struct mali_pp_core *pp_core); +void mali_group_remove_pp_core(struct mali_group *group); + +MALI_STATIC_INLINE const char *mali_group_core_description( + struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + if (NULL != group->pp_core) { + return mali_pp_core_description(group->pp_core); + } else { + MALI_DEBUG_ASSERT_POINTER(group->gp_core); + return mali_gp_core_description(group->gp_core); + } +} + +MALI_STATIC_INLINE mali_bool mali_group_is_virtual(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + +#if (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) + return (NULL != group->dlbu_core); +#else + return MALI_FALSE; +#endif +} + +/** @brief Check if a group is a part of a virtual group or not + */ +MALI_STATIC_INLINE mali_bool mali_group_is_in_virtual(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + +#if (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) + return (NULL != group->parent_group) ? MALI_TRUE : MALI_FALSE; +#else + return MALI_FALSE; +#endif +} + +/** @brief Reset group + * + * This function will reset the entire group, + * including all the cores present in the group. + * + * @param group Pointer to the group to reset + */ +void mali_group_reset(struct mali_group *group); + +MALI_STATIC_INLINE struct mali_session_data *mali_group_get_session( + struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + return group->session; +} + +MALI_STATIC_INLINE void mali_group_clear_session(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + if (NULL != group->session) { + mali_mmu_activate_empty_page_directory(group->mmu); + group->session = NULL; + } +} + +enum mali_group_state mali_group_activate(struct mali_group *group); + +/* + * Change state from ACTIVATION_PENDING to ACTIVE + * For virtual group, all childs need to be ACTIVE first + */ +mali_bool mali_group_set_active(struct mali_group *group); + +/* + * @return MALI_TRUE means one or more domains can now be powered off, + * and caller should call either mali_pm_update_async() or + * mali_pm_update_sync() in order to do so. + */ +mali_bool mali_group_deactivate(struct mali_group *group); + +MALI_STATIC_INLINE enum mali_group_state mali_group_get_state(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + return group->state; +} + +MALI_STATIC_INLINE mali_bool mali_group_power_is_on(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + return group->power_is_on; +} + +void mali_group_power_up(struct mali_group *group); +void mali_group_power_down(struct mali_group *group); + +MALI_STATIC_INLINE void mali_group_set_disable_request( + struct mali_group *group, mali_bool disable) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + group->disable_requested = disable; + + /** + * When one of child group's disable_requeset is set TRUE, then + * the disable_request of parent group should also be set to TRUE. + * While, the disable_request of parent group should only be set to FALSE + * only when all of its child group's disable_request are set to FALSE. + */ + if (NULL != group->parent_group && MALI_TRUE == disable) { + group->parent_group->disable_requested = disable; + } +} + +MALI_STATIC_INLINE mali_bool mali_group_disable_requested( + struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + return group->disable_requested; +} + +/** @brief Virtual groups */ +void mali_group_add_group(struct mali_group *parent, struct mali_group *child); +struct mali_group *mali_group_acquire_group(struct mali_group *parent); +void mali_group_remove_group(struct mali_group *parent, struct mali_group *child); + +/** @brief Checks if the group is working. + */ +MALI_STATIC_INLINE mali_bool mali_group_is_working(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + if (mali_group_is_in_virtual(group)) { + struct mali_group *tmp_group = mali_executor_get_virtual_group(); + return tmp_group->is_working; + } + return group->is_working; +} + +MALI_STATIC_INLINE struct mali_gp_job *mali_group_get_running_gp_job(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + return group->gp_running_job; +} + +/** @brief Zap MMU TLB on all groups + * + * Zap TLB on group if \a session is active. + */ +mali_bool mali_group_zap_session(struct mali_group *group, + struct mali_session_data *session); + +/** @brief Get pointer to GP core object + */ +MALI_STATIC_INLINE struct mali_gp_core *mali_group_get_gp_core(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + return group->gp_core; +} + +/** @brief Get pointer to PP core object + */ +MALI_STATIC_INLINE struct mali_pp_core *mali_group_get_pp_core(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + return group->pp_core; +} + +/** @brief Start GP job + */ +void mali_group_start_gp_job(struct mali_group *group, struct mali_gp_job *job); + +void mali_group_start_pp_job(struct mali_group *group, struct mali_pp_job *job, u32 sub_job); + +/** @brief Start virtual group Job on a virtual group +*/ +void mali_group_start_job_on_virtual(struct mali_group *group, struct mali_pp_job *job, u32 first_subjob, u32 last_subjob); + + +/** @brief Start a subjob from a particular on a specific PP group +*/ +void mali_group_start_job_on_group(struct mali_group *group, struct mali_pp_job *job, u32 subjob); + + +/** @brief remove all the unused groups in tmp_unused group list, so that the group is in consistent status. + */ +void mali_group_non_dlbu_job_done_virtual(struct mali_group *group); + + +/** @brief Resume GP job that suspended waiting for more heap memory + */ +void mali_group_resume_gp_with_new_heap(struct mali_group *group, u32 job_id, u32 start_addr, u32 end_addr); + +MALI_STATIC_INLINE enum mali_interrupt_result mali_group_get_interrupt_result_gp(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->gp_core); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + return mali_gp_get_interrupt_result(group->gp_core); +} + +MALI_STATIC_INLINE enum mali_interrupt_result mali_group_get_interrupt_result_pp(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->pp_core); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + return mali_pp_get_interrupt_result(group->pp_core); +} + +MALI_STATIC_INLINE enum mali_interrupt_result mali_group_get_interrupt_result_mmu(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->mmu); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + return mali_mmu_get_interrupt_result(group->mmu); +} + +MALI_STATIC_INLINE mali_bool mali_group_gp_is_active(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->gp_core); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + return mali_gp_is_active(group->gp_core); +} + +MALI_STATIC_INLINE mali_bool mali_group_pp_is_active(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->pp_core); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + return mali_pp_is_active(group->pp_core); +} + +MALI_STATIC_INLINE mali_bool mali_group_has_timed_out(struct mali_group *group) +{ + unsigned long time_cost; + struct mali_group *tmp_group = group; + + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + /* if the group is in virtual need to use virtual_group's start time */ + if (mali_group_is_in_virtual(group)) { + tmp_group = mali_executor_get_virtual_group(); + } + + time_cost = _mali_osk_time_tickcount() - tmp_group->start_time; + if (_mali_osk_time_mstoticks(mali_max_job_runtime) <= time_cost) { + /* + * current tick is at or after timeout end time, + * so this is a valid timeout + */ + return MALI_TRUE; + } else { + /* + * Not a valid timeout. A HW interrupt probably beat + * us to it, and the timer wasn't properly deleted + * (async deletion used due to atomic context). + */ + return MALI_FALSE; + } +} + +MALI_STATIC_INLINE void mali_group_mask_all_interrupts_gp(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->gp_core); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + return mali_gp_mask_all_interrupts(group->gp_core); +} + +MALI_STATIC_INLINE void mali_group_mask_all_interrupts_pp(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->pp_core); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + return mali_pp_mask_all_interrupts(group->pp_core); +} + +MALI_STATIC_INLINE void mali_group_enable_interrupts_gp( + struct mali_group *group, + enum mali_interrupt_result exceptions) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->gp_core); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + mali_gp_enable_interrupts(group->gp_core, exceptions); +} + +MALI_STATIC_INLINE void mali_group_schedule_bottom_half_gp(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->gp_core); + _mali_osk_wq_schedule_work(group->bottom_half_work_gp); +} + +MALI_STATIC_INLINE void mali_group_schedule_oom_work_handler(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->gp_core); + _mali_osk_wq_schedule_work(group->oom_work_handler); +} + +MALI_STATIC_INLINE void mali_group_schedule_bottom_half_pp(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->pp_core); + _mali_osk_wq_schedule_work(group->bottom_half_work_pp); +} + +MALI_STATIC_INLINE void mali_group_schedule_bottom_half_mmu(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->mmu); + _mali_osk_wq_schedule_work(group->bottom_half_work_mmu); +} + +struct mali_pp_job *mali_group_complete_pp(struct mali_group *group, mali_bool success, u32 *sub_job); + +struct mali_gp_job *mali_group_complete_gp(struct mali_group *group, mali_bool success); + +#if defined(CONFIG_MALI400_PROFILING) +MALI_STATIC_INLINE void mali_group_oom(struct mali_group *group) +{ + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SUSPEND | + MALI_PROFILING_MAKE_EVENT_CHANNEL_GP(0), + 0, 0, 0, 0, 0); +} +#endif + +struct mali_group *mali_group_get_glob_group(u32 index); +u32 mali_group_get_glob_num_groups(void); + +u32 mali_group_dump_state(struct mali_group *group, char *buf, u32 size); + + +_mali_osk_errcode_t mali_group_upper_half_mmu(void *data); +_mali_osk_errcode_t mali_group_upper_half_gp(void *data); +_mali_osk_errcode_t mali_group_upper_half_pp(void *data); + +MALI_STATIC_INLINE mali_bool mali_group_is_empty(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT(mali_group_is_virtual(group)); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + return _mali_osk_list_empty(&group->group_list); +} + +#endif /* __MALI_GROUP_H__ */ diff --git a/drivers/gpu/arm/utgard/common/mali_hw_core.c b/drivers/gpu/arm/utgard/common/mali_hw_core.c new file mode 100644 index 00000000000000..c90cf38d8516c8 --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_hw_core.c @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_hw_core.h" +#include "mali_osk.h" +#include "mali_kernel_common.h" +#include "mali_osk_mali.h" + +_mali_osk_errcode_t mali_hw_core_create(struct mali_hw_core *core, const _mali_osk_resource_t *resource, u32 reg_size) +{ + core->phys_addr = resource->base; + core->phys_offset = resource->base - _mali_osk_resource_base_address(); + core->description = resource->description; + core->size = reg_size; + + MALI_DEBUG_ASSERT(core->phys_offset < core->phys_addr); + + if (_MALI_OSK_ERR_OK == _mali_osk_mem_reqregion(core->phys_addr, core->size, core->description)) { + core->mapped_registers = _mali_osk_mem_mapioregion(core->phys_addr, core->size, core->description); + if (NULL != core->mapped_registers) { + return _MALI_OSK_ERR_OK; + } else { + MALI_PRINT_ERROR(("Failed to map memory region for core %s at phys_addr 0x%08X\n", core->description, core->phys_addr)); + } + _mali_osk_mem_unreqregion(core->phys_addr, core->size); + } else { + MALI_PRINT_ERROR(("Failed to request memory region for core %s at phys_addr 0x%08X\n", core->description, core->phys_addr)); + } + + return _MALI_OSK_ERR_FAULT; +} + +void mali_hw_core_delete(struct mali_hw_core *core) +{ + if (NULL != core->mapped_registers) { + _mali_osk_mem_unmapioregion(core->phys_addr, core->size, core->mapped_registers); + core->mapped_registers = NULL; + } + _mali_osk_mem_unreqregion(core->phys_addr, core->size); +} diff --git a/drivers/gpu/arm/utgard/common/mali_hw_core.h b/drivers/gpu/arm/utgard/common/mali_hw_core.h new file mode 100644 index 00000000000000..ac2ffbedf30858 --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_hw_core.h @@ -0,0 +1,111 @@ +/* + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_HW_CORE_H__ +#define __MALI_HW_CORE_H__ + +#include "mali_osk.h" +#include "mali_kernel_common.h" + +/** + * The common parts for all Mali HW cores (GP, PP, MMU, L2 and PMU) + * This struct is embedded inside all core specific structs. + */ +struct mali_hw_core { + uintptr_t phys_addr; /**< Physical address of the registers */ + u32 phys_offset; /**< Offset from start of Mali to registers */ + u32 size; /**< Size of registers */ + mali_io_address mapped_registers; /**< Virtual mapping of the registers */ + const char *description; /**< Name of unit (as specified in device configuration) */ +}; + +#define MALI_REG_POLL_COUNT_FAST 1000000 +#define MALI_REG_POLL_COUNT_SLOW 1000000 + +/* + * GP and PP core translate their int_stat/rawstat into one of these + */ +enum mali_interrupt_result { + MALI_INTERRUPT_RESULT_NONE, + MALI_INTERRUPT_RESULT_SUCCESS, + MALI_INTERRUPT_RESULT_SUCCESS_VS, + MALI_INTERRUPT_RESULT_SUCCESS_PLBU, + MALI_INTERRUPT_RESULT_OOM, + MALI_INTERRUPT_RESULT_ERROR +}; + +_mali_osk_errcode_t mali_hw_core_create(struct mali_hw_core *core, const _mali_osk_resource_t *resource, u32 reg_size); +void mali_hw_core_delete(struct mali_hw_core *core); + +MALI_STATIC_INLINE u32 mali_hw_core_register_read(struct mali_hw_core *core, u32 relative_address) +{ + u32 read_val; + read_val = _mali_osk_mem_ioread32(core->mapped_registers, relative_address); + MALI_DEBUG_PRINT(6, ("register_read for core %s, relative addr=0x%04X, val=0x%08X\n", + core->description, relative_address, read_val)); + return read_val; +} + +MALI_STATIC_INLINE void mali_hw_core_register_write_relaxed(struct mali_hw_core *core, u32 relative_address, u32 new_val) +{ + MALI_DEBUG_PRINT(6, ("register_write_relaxed for core %s, relative addr=0x%04X, val=0x%08X\n", + core->description, relative_address, new_val)); + _mali_osk_mem_iowrite32_relaxed(core->mapped_registers, relative_address, new_val); +} + +/* Conditionally write a register. + * The register will only be written if the new value is different from the old_value. + * If the new value is different, the old value will also be updated */ +MALI_STATIC_INLINE void mali_hw_core_register_write_relaxed_conditional(struct mali_hw_core *core, u32 relative_address, u32 new_val, const u32 old_val) +{ + MALI_DEBUG_PRINT(6, ("register_write_relaxed for core %s, relative addr=0x%04X, val=0x%08X\n", + core->description, relative_address, new_val)); + if (old_val != new_val) { + _mali_osk_mem_iowrite32_relaxed(core->mapped_registers, relative_address, new_val); + } +} + +MALI_STATIC_INLINE void mali_hw_core_register_write(struct mali_hw_core *core, u32 relative_address, u32 new_val) +{ + MALI_DEBUG_PRINT(6, ("register_write for core %s, relative addr=0x%04X, val=0x%08X\n", + core->description, relative_address, new_val)); + _mali_osk_mem_iowrite32(core->mapped_registers, relative_address, new_val); +} + +MALI_STATIC_INLINE void mali_hw_core_register_write_array_relaxed(struct mali_hw_core *core, u32 relative_address, u32 *write_array, u32 nr_of_regs) +{ + u32 i; + MALI_DEBUG_PRINT(6, ("register_write_array: for core %s, relative addr=0x%04X, nr of regs=%u\n", + core->description, relative_address, nr_of_regs)); + + /* Do not use burst writes against the registers */ + for (i = 0; i < nr_of_regs; i++) { + mali_hw_core_register_write_relaxed(core, relative_address + i * 4, write_array[i]); + } +} + +/* Conditionally write a set of registers. + * The register will only be written if the new value is different from the old_value. + * If the new value is different, the old value will also be updated */ +MALI_STATIC_INLINE void mali_hw_core_register_write_array_relaxed_conditional(struct mali_hw_core *core, u32 relative_address, u32 *write_array, u32 nr_of_regs, const u32 *old_array) +{ + u32 i; + MALI_DEBUG_PRINT(6, ("register_write_array: for core %s, relative addr=0x%04X, nr of regs=%u\n", + core->description, relative_address, nr_of_regs)); + + /* Do not use burst writes against the registers */ + for (i = 0; i < nr_of_regs; i++) { + if (old_array[i] != write_array[i]) { + mali_hw_core_register_write_relaxed(core, relative_address + i * 4, write_array[i]); + } + } +} + +#endif /* __MALI_HW_CORE_H__ */ diff --git a/drivers/gpu/arm/utgard/common/mali_kernel_common.h b/drivers/gpu/arm/utgard/common/mali_kernel_common.h new file mode 100644 index 00000000000000..990cf3ac78e021 --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_kernel_common.h @@ -0,0 +1,181 @@ +/* + * Copyright (C) 2010, 2012-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_KERNEL_COMMON_H__ +#define __MALI_KERNEL_COMMON_H__ + +#include "mali_osk.h" + +/* Make sure debug is defined when it should be */ +#ifndef DEBUG +#if defined(_DEBUG) +#define DEBUG +#endif +#endif + +/* The file include several useful macros for error checking, debugging and printing. + * - MALI_PRINTF(...) Do not use this function: Will be included in Release builds. + * - MALI_DEBUG_PRINT(nr, (X) ) Prints the second argument if nr<=MALI_DEBUG_LEVEL. + * - MALI_DEBUG_ERROR( (X) ) Prints an errortext, a source trace, and the given error message. + * - MALI_DEBUG_ASSERT(exp,(X)) If the asserted expr is false, the program will exit. + * - MALI_DEBUG_ASSERT_POINTER(pointer) Triggers if the pointer is a zero pointer. + * - MALI_DEBUG_CODE( X ) The code inside the macro is only compiled in Debug builds. + * + * The (X) means that you must add an extra parenthesis around the argumentlist. + * + * The printf function: MALI_PRINTF(...) is routed to _mali_osk_debugmsg + * + * Suggested range for the DEBUG-LEVEL is [1:6] where + * [1:2] Is messages with highest priority, indicate possible errors. + * [3:4] Is messages with medium priority, output important variables. + * [5:6] Is messages with low priority, used during extensive debugging. + */ + +/** +* Fundamental error macro. Reports an error code. This is abstracted to allow us to +* easily switch to a different error reporting method if we want, and also to allow +* us to search for error returns easily. +* +* Note no closing semicolon - this is supplied in typical usage: +* +* MALI_ERROR(MALI_ERROR_OUT_OF_MEMORY); +*/ +#define MALI_ERROR(error_code) return (error_code) + +/** + * Basic error macro, to indicate success. + * Note no closing semicolon - this is supplied in typical usage: + * + * MALI_SUCCESS; + */ +#define MALI_SUCCESS MALI_ERROR(_MALI_OSK_ERR_OK) + +/** + * Basic error macro. This checks whether the given condition is true, and if not returns + * from this function with the supplied error code. This is a macro so that we can override it + * for stress testing. + * + * Note that this uses the do-while-0 wrapping to ensure that we don't get problems with dangling + * else clauses. Note also no closing semicolon - this is supplied in typical usage: + * + * MALI_CHECK((p!=NULL), ERROR_NO_OBJECT); + */ +#define MALI_CHECK(condition, error_code) do { if(!(condition)) MALI_ERROR(error_code); } while(0) + +/** + * Error propagation macro. If the expression given is anything other than + * _MALI_OSK_NO_ERROR, then the value is returned from the enclosing function + * as an error code. This effectively acts as a guard clause, and propagates + * error values up the call stack. This uses a temporary value to ensure that + * the error expression is not evaluated twice. + * If the counter for forcing a failure has been set using _mali_force_error, + * this error will be returned without evaluating the expression in + * MALI_CHECK_NO_ERROR + */ +#define MALI_CHECK_NO_ERROR(expression) \ + do { _mali_osk_errcode_t _check_no_error_result=(expression); \ + if(_check_no_error_result != _MALI_OSK_ERR_OK) \ + MALI_ERROR(_check_no_error_result); \ + } while(0) + +/** + * Pointer check macro. Checks non-null pointer. + */ +#define MALI_CHECK_NON_NULL(pointer, error_code) MALI_CHECK( ((pointer)!=NULL), (error_code) ) + +/** + * Error macro with goto. This checks whether the given condition is true, and if not jumps + * to the specified label using a goto. The label must therefore be local to the function in + * which this macro appears. This is most usually used to execute some clean-up code before + * exiting with a call to ERROR. + * + * Like the other macros, this is a macro to allow us to override the condition if we wish, + * e.g. to force an error during stress testing. + */ +#define MALI_CHECK_GOTO(condition, label) do { if(!(condition)) goto label; } while(0) + +/** + * Explicitly ignore a parameter passed into a function, to suppress compiler warnings. + * Should only be used with parameter names. + */ +#define MALI_IGNORE(x) x=x + +#if defined(CONFIG_MALI_QUIET) +#define MALI_PRINTF(args) +#else +#define MALI_PRINTF(args) _mali_osk_dbgmsg args; +#endif + +#define MALI_PRINT_ERROR(args) do{ \ + MALI_PRINTF(("Mali: ERR: %s\n" ,__FILE__)); \ + MALI_PRINTF((" %s()%4d\n ", __FUNCTION__, __LINE__)) ; \ + MALI_PRINTF(args); \ + MALI_PRINTF(("\n")); \ + } while(0) + +#define MALI_PRINT(args) do{ \ + MALI_PRINTF(("Mali: ")); \ + MALI_PRINTF(args); \ + } while (0) + +#ifdef DEBUG +#ifndef mali_debug_level +extern int mali_debug_level; +#endif + +#define MALI_DEBUG_CODE(code) code +#define MALI_DEBUG_PRINT(level, args) do { \ + if((level) <= mali_debug_level)\ + {MALI_PRINTF(("Mali<" #level ">: ")); MALI_PRINTF(args); } \ + } while (0) + +#define MALI_DEBUG_PRINT_ERROR(args) MALI_PRINT_ERROR(args) + +#define MALI_DEBUG_PRINT_IF(level,condition,args) \ + if((condition)&&((level) <= mali_debug_level))\ + {MALI_PRINTF(("Mali<" #level ">: ")); MALI_PRINTF(args); } + +#define MALI_DEBUG_PRINT_ELSE(level, args)\ + else if((level) <= mali_debug_level)\ + { MALI_PRINTF(("Mali<" #level ">: ")); MALI_PRINTF(args); } + +/** + * @note these variants of DEBUG ASSERTS will cause a debugger breakpoint + * to be entered (see _mali_osk_break() ). An alternative would be to call + * _mali_osk_abort(), on OSs that support it. + */ +#define MALI_DEBUG_PRINT_ASSERT(condition, args) do {if( !(condition)) { MALI_PRINT_ERROR(args); _mali_osk_break(); } } while(0) +#define MALI_DEBUG_ASSERT_POINTER(pointer) do {if( (pointer)== NULL) {MALI_PRINT_ERROR(("NULL pointer " #pointer)); _mali_osk_break();} } while(0) +#define MALI_DEBUG_ASSERT(condition) do {if( !(condition)) {MALI_PRINT_ERROR(("ASSERT failed: " #condition )); _mali_osk_break();} } while(0) + +#else /* DEBUG */ + +#define MALI_DEBUG_CODE(code) +#define MALI_DEBUG_PRINT(string,args) do {} while(0) +#define MALI_DEBUG_PRINT_ERROR(args) do {} while(0) +#define MALI_DEBUG_PRINT_IF(level,condition,args) do {} while(0) +#define MALI_DEBUG_PRINT_ELSE(level,condition,args) do {} while(0) +#define MALI_DEBUG_PRINT_ASSERT(condition,args) do {} while(0) +#define MALI_DEBUG_ASSERT_POINTER(pointer) do {} while(0) +#define MALI_DEBUG_ASSERT(condition) do {} while(0) + +#endif /* DEBUG */ + +/** + * variables from user space cannot be dereferenced from kernel space; tagging them + * with __user allows the GCC compiler to generate a warning. Other compilers may + * not support this so we define it here as an empty macro if the compiler doesn't + * define it. + */ +#ifndef __user +#define __user +#endif + +#endif /* __MALI_KERNEL_COMMON_H__ */ diff --git a/drivers/gpu/arm/utgard/common/mali_kernel_core.c b/drivers/gpu/arm/utgard/common/mali_kernel_core.c new file mode 100644 index 00000000000000..b9a6b7a0a27798 --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_kernel_core.c @@ -0,0 +1,1314 @@ +/* + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_kernel_common.h" +#include "mali_session.h" +#include "mali_osk.h" +#include "mali_osk_mali.h" +#include "mali_ukk.h" +#include "mali_kernel_core.h" +#include "mali_memory.h" +#include "mali_mem_validation.h" +#include "mali_mmu.h" +#include "mali_mmu_page_directory.h" +#include "mali_dlbu.h" +#include "mali_broadcast.h" +#include "mali_gp.h" +#include "mali_pp.h" +#include "mali_executor.h" +#include "mali_pp_job.h" +#include "mali_group.h" +#include "mali_pm.h" +#include "mali_pmu.h" +#include "mali_scheduler.h" +#include "mali_kernel_utilization.h" +#include "mali_l2_cache.h" +#include "mali_timeline.h" +#include "mali_soft_job.h" +#include "mali_pm_domain.h" +#if defined(CONFIG_MALI400_PROFILING) +#include "mali_osk_profiling.h" +#endif +#if defined(CONFIG_MALI400_INTERNAL_PROFILING) +#include "mali_profiling_internal.h" +#endif +#include "mali_control_timer.h" +#include "mali_dvfs_policy.h" +#include + +#define MALI_SHARED_MEMORY_DEFAULT_SIZE 0xffffffff + +/* Mali GPU memory. Real values come from module parameter or from device specific data */ +unsigned int mali_dedicated_mem_start = 0; +unsigned int mali_dedicated_mem_size = 0; + +/* Default shared memory size is set to 4G. */ +unsigned int mali_shared_mem_size = MALI_SHARED_MEMORY_DEFAULT_SIZE; + +/* Frame buffer memory to be accessible by Mali GPU */ +int mali_fb_start = 0; +int mali_fb_size = 0; + +/* Mali max job runtime */ +extern int mali_max_job_runtime; + +/** Start profiling from module load? */ +int mali_boot_profiling = 0; + +/** Limits for the number of PP cores behind each L2 cache. */ +int mali_max_pp_cores_group_1 = 0xFF; +int mali_max_pp_cores_group_2 = 0xFF; + +int mali_inited_pp_cores_group_1 = 0; +int mali_inited_pp_cores_group_2 = 0; + +static _mali_product_id_t global_product_id = _MALI_PRODUCT_ID_UNKNOWN; +static uintptr_t global_gpu_base_address = 0; +static u32 global_gpu_major_version = 0; +static u32 global_gpu_minor_version = 0; + +mali_bool mali_gpu_class_is_mali450 = MALI_FALSE; +mali_bool mali_gpu_class_is_mali470 = MALI_FALSE; + +static _mali_osk_errcode_t mali_set_global_gpu_base_address(void) +{ + _mali_osk_errcode_t err = _MALI_OSK_ERR_OK; + + global_gpu_base_address = _mali_osk_resource_base_address(); + if (0 == global_gpu_base_address) { + err = _MALI_OSK_ERR_ITEM_NOT_FOUND; + } + + return err; +} + +static u32 mali_get_bcast_id(_mali_osk_resource_t *resource_pp) +{ + switch (resource_pp->base - global_gpu_base_address) { + case 0x08000: + case 0x20000: /* fall-through for aliased mapping */ + return 0x01; + case 0x0A000: + case 0x22000: /* fall-through for aliased mapping */ + return 0x02; + case 0x0C000: + case 0x24000: /* fall-through for aliased mapping */ + return 0x04; + case 0x0E000: + case 0x26000: /* fall-through for aliased mapping */ + return 0x08; + case 0x28000: + return 0x10; + case 0x2A000: + return 0x20; + case 0x2C000: + return 0x40; + case 0x2E000: + return 0x80; + default: + return 0; + } +} + +static _mali_osk_errcode_t mali_parse_product_info(void) +{ + _mali_osk_resource_t first_pp_resource; + + /* Find the first PP core resource (again) */ + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI_OFFSET_PP0, &first_pp_resource)) { + /* Create a dummy PP object for this core so that we can read the version register */ + struct mali_group *group = mali_group_create(NULL, NULL, NULL, MALI_DOMAIN_INDEX_PP0); + if (NULL != group) { + struct mali_pp_core *pp_core = mali_pp_create(&first_pp_resource, group, MALI_FALSE, mali_get_bcast_id(&first_pp_resource)); + if (NULL != pp_core) { + u32 pp_version; + + pp_version = mali_pp_core_get_version(pp_core); + + mali_group_delete(group); + + global_gpu_major_version = (pp_version >> 8) & 0xFF; + global_gpu_minor_version = pp_version & 0xFF; + + switch (pp_version >> 16) { + case MALI200_PP_PRODUCT_ID: + global_product_id = _MALI_PRODUCT_ID_MALI200; + MALI_DEBUG_PRINT(2, ("Found Mali GPU Mali-200 r%up%u\n", global_gpu_major_version, global_gpu_minor_version)); + MALI_PRINT_ERROR(("Mali-200 is not supported by this driver.\n")); + _mali_osk_abort(); + break; + case MALI300_PP_PRODUCT_ID: + global_product_id = _MALI_PRODUCT_ID_MALI300; + MALI_DEBUG_PRINT(2, ("Found Mali GPU Mali-300 r%up%u\n", global_gpu_major_version, global_gpu_minor_version)); + break; + case MALI400_PP_PRODUCT_ID: + global_product_id = _MALI_PRODUCT_ID_MALI400; + MALI_DEBUG_PRINT(2, ("Found Mali GPU Mali-400 MP r%up%u\n", global_gpu_major_version, global_gpu_minor_version)); + break; + case MALI450_PP_PRODUCT_ID: + global_product_id = _MALI_PRODUCT_ID_MALI450; + MALI_DEBUG_PRINT(2, ("Found Mali GPU Mali-450 MP r%up%u\n", global_gpu_major_version, global_gpu_minor_version)); + break; + case MALI470_PP_PRODUCT_ID: + global_product_id = _MALI_PRODUCT_ID_MALI470; + MALI_DEBUG_PRINT(2, ("Found Mali GPU Mali-470 MP r%up%u\n", global_gpu_major_version, global_gpu_minor_version)); + break; + default: + MALI_DEBUG_PRINT(2, ("Found unknown Mali GPU (r%up%u)\n", global_gpu_major_version, global_gpu_minor_version)); + return _MALI_OSK_ERR_FAULT; + } + + return _MALI_OSK_ERR_OK; + } else { + MALI_PRINT_ERROR(("Failed to create initial PP object\n")); + } + } else { + MALI_PRINT_ERROR(("Failed to create initial group object\n")); + } + } else { + MALI_PRINT_ERROR(("First PP core not specified in config file\n")); + } + + return _MALI_OSK_ERR_FAULT; +} + +static void mali_delete_groups(void) +{ + struct mali_group *group; + + group = mali_group_get_glob_group(0); + while (NULL != group) { + mali_group_delete(group); + group = mali_group_get_glob_group(0); + } + + MALI_DEBUG_ASSERT(0 == mali_group_get_glob_num_groups()); +} + +static void mali_delete_l2_cache_cores(void) +{ + struct mali_l2_cache_core *l2; + + l2 = mali_l2_cache_core_get_glob_l2_core(0); + while (NULL != l2) { + mali_l2_cache_delete(l2); + l2 = mali_l2_cache_core_get_glob_l2_core(0); + } + + MALI_DEBUG_ASSERT(0 == mali_l2_cache_core_get_glob_num_l2_cores()); +} + +static struct mali_l2_cache_core *mali_create_l2_cache_core(_mali_osk_resource_t *resource, u32 domain_index) +{ + struct mali_l2_cache_core *l2_cache = NULL; + + if (NULL != resource) { + + MALI_DEBUG_PRINT(3, ("Found L2 cache %s\n", resource->description)); + + l2_cache = mali_l2_cache_create(resource, domain_index); + if (NULL == l2_cache) { + MALI_PRINT_ERROR(("Failed to create L2 cache object\n")); + return NULL; + } + } + MALI_DEBUG_PRINT(3, ("Created L2 cache core object\n")); + + return l2_cache; +} + +static _mali_osk_errcode_t mali_parse_config_l2_cache(void) +{ + struct mali_l2_cache_core *l2_cache = NULL; + + if (mali_is_mali400()) { + _mali_osk_resource_t l2_resource; + if (_MALI_OSK_ERR_OK != _mali_osk_resource_find(MALI400_OFFSET_L2_CACHE0, &l2_resource)) { + MALI_DEBUG_PRINT(3, ("Did not find required Mali L2 cache in config file\n")); + return _MALI_OSK_ERR_FAULT; + } + + l2_cache = mali_create_l2_cache_core(&l2_resource, MALI_DOMAIN_INDEX_L20); + if (NULL == l2_cache) { + return _MALI_OSK_ERR_FAULT; + } + } else if (mali_is_mali450()) { + /* + * L2 for GP at 0x10000 + * L2 for PP0-3 at 0x01000 + * L2 for PP4-7 at 0x11000 (optional) + */ + + _mali_osk_resource_t l2_gp_resource; + _mali_osk_resource_t l2_pp_grp0_resource; + _mali_osk_resource_t l2_pp_grp1_resource; + + /* Make cluster for GP's L2 */ + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI450_OFFSET_L2_CACHE0, &l2_gp_resource)) { + MALI_DEBUG_PRINT(3, ("Creating Mali-450 L2 cache core for GP\n")); + l2_cache = mali_create_l2_cache_core(&l2_gp_resource, MALI_DOMAIN_INDEX_L20); + if (NULL == l2_cache) { + return _MALI_OSK_ERR_FAULT; + } + } else { + MALI_DEBUG_PRINT(3, ("Did not find required Mali L2 cache for GP in config file\n")); + return _MALI_OSK_ERR_FAULT; + } + + /* Find corresponding l2 domain */ + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI450_OFFSET_L2_CACHE1, &l2_pp_grp0_resource)) { + MALI_DEBUG_PRINT(3, ("Creating Mali-450 L2 cache core for PP group 0\n")); + l2_cache = mali_create_l2_cache_core(&l2_pp_grp0_resource, MALI_DOMAIN_INDEX_L21); + if (NULL == l2_cache) { + return _MALI_OSK_ERR_FAULT; + } + } else { + MALI_DEBUG_PRINT(3, ("Did not find required Mali L2 cache for PP group 0 in config file\n")); + return _MALI_OSK_ERR_FAULT; + } + + /* Second PP core group is optional, don't fail if we don't find it */ + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI450_OFFSET_L2_CACHE2, &l2_pp_grp1_resource)) { + MALI_DEBUG_PRINT(3, ("Creating Mali-450 L2 cache core for PP group 1\n")); + l2_cache = mali_create_l2_cache_core(&l2_pp_grp1_resource, MALI_DOMAIN_INDEX_L22); + if (NULL == l2_cache) { + return _MALI_OSK_ERR_FAULT; + } + } + } else if (mali_is_mali470()) { + _mali_osk_resource_t l2c1_resource; + + /* Make cluster for L2C1 */ + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI470_OFFSET_L2_CACHE1, &l2c1_resource)) { + MALI_DEBUG_PRINT(3, ("Creating Mali-470 L2 cache 1\n")); + l2_cache = mali_create_l2_cache_core(&l2c1_resource, MALI_DOMAIN_INDEX_L21); + if (NULL == l2_cache) { + return _MALI_OSK_ERR_FAULT; + } + } else { + MALI_DEBUG_PRINT(3, ("Did not find required Mali L2 cache for L2C1\n")); + return _MALI_OSK_ERR_FAULT; + } + } + + return _MALI_OSK_ERR_OK; +} + +static struct mali_group *mali_create_group(struct mali_l2_cache_core *cache, + _mali_osk_resource_t *resource_mmu, + _mali_osk_resource_t *resource_gp, + _mali_osk_resource_t *resource_pp, + u32 domain_index) +{ + struct mali_mmu_core *mmu; + struct mali_group *group; + + MALI_DEBUG_PRINT(3, ("Starting new group for MMU %s\n", resource_mmu->description)); + + /* Create the group object */ + group = mali_group_create(cache, NULL, NULL, domain_index); + if (NULL == group) { + MALI_PRINT_ERROR(("Failed to create group object for MMU %s\n", resource_mmu->description)); + return NULL; + } + + /* Create the MMU object inside group */ + mmu = mali_mmu_create(resource_mmu, group, MALI_FALSE); + if (NULL == mmu) { + MALI_PRINT_ERROR(("Failed to create MMU object\n")); + mali_group_delete(group); + return NULL; + } + + if (NULL != resource_gp) { + /* Create the GP core object inside this group */ + struct mali_gp_core *gp_core = mali_gp_create(resource_gp, group); + if (NULL == gp_core) { + /* No need to clean up now, as we will clean up everything linked in from the cluster when we fail this function */ + MALI_PRINT_ERROR(("Failed to create GP object\n")); + mali_group_delete(group); + return NULL; + } + } + + if (NULL != resource_pp) { + struct mali_pp_core *pp_core; + + /* Create the PP core object inside this group */ + pp_core = mali_pp_create(resource_pp, group, MALI_FALSE, mali_get_bcast_id(resource_pp)); + if (NULL == pp_core) { + /* No need to clean up now, as we will clean up everything linked in from the cluster when we fail this function */ + MALI_PRINT_ERROR(("Failed to create PP object\n")); + mali_group_delete(group); + return NULL; + } + } + + return group; +} + +static _mali_osk_errcode_t mali_create_virtual_group(_mali_osk_resource_t *resource_mmu_pp_bcast, + _mali_osk_resource_t *resource_pp_bcast, + _mali_osk_resource_t *resource_dlbu, + _mali_osk_resource_t *resource_bcast) +{ + struct mali_mmu_core *mmu_pp_bcast_core; + struct mali_pp_core *pp_bcast_core; + struct mali_dlbu_core *dlbu_core; + struct mali_bcast_unit *bcast_core; + struct mali_group *group; + + MALI_DEBUG_PRINT(2, ("Starting new virtual group for MMU PP broadcast core %s\n", resource_mmu_pp_bcast->description)); + + /* Create the DLBU core object */ + dlbu_core = mali_dlbu_create(resource_dlbu); + if (NULL == dlbu_core) { + MALI_PRINT_ERROR(("Failed to create DLBU object \n")); + return _MALI_OSK_ERR_FAULT; + } + + /* Create the Broadcast unit core */ + bcast_core = mali_bcast_unit_create(resource_bcast); + if (NULL == bcast_core) { + MALI_PRINT_ERROR(("Failed to create Broadcast unit object!\n")); + mali_dlbu_delete(dlbu_core); + return _MALI_OSK_ERR_FAULT; + } + + /* Create the group object */ +#if defined(DEBUG) + /* Get a physical PP group to temporarily add to broadcast unit. IRQ + * verification needs a physical group in the broadcast unit to test + * the broadcast unit interrupt line. */ + { + struct mali_group *phys_group = NULL; + int i; + for (i = 0; i < mali_group_get_glob_num_groups(); i++) { + phys_group = mali_group_get_glob_group(i); + if (NULL != mali_group_get_pp_core(phys_group)) break; + } + MALI_DEBUG_ASSERT(NULL != mali_group_get_pp_core(phys_group)); + + /* Add the group temporarily to the broadcast, and update the + * broadcast HW. Since the HW is not updated when removing the + * group the IRQ check will work when the virtual PP is created + * later. + * + * When the virtual group gets populated, the actually used + * groups will be added to the broadcast unit and the HW will + * be updated. + */ + mali_bcast_add_group(bcast_core, phys_group); + mali_bcast_reset(bcast_core); + mali_bcast_remove_group(bcast_core, phys_group); + } +#endif /* DEBUG */ + group = mali_group_create(NULL, dlbu_core, bcast_core, MALI_DOMAIN_INDEX_DUMMY); + if (NULL == group) { + MALI_PRINT_ERROR(("Failed to create group object for MMU PP broadcast core %s\n", resource_mmu_pp_bcast->description)); + mali_bcast_unit_delete(bcast_core); + mali_dlbu_delete(dlbu_core); + return _MALI_OSK_ERR_FAULT; + } + + /* Create the MMU object inside group */ + mmu_pp_bcast_core = mali_mmu_create(resource_mmu_pp_bcast, group, MALI_TRUE); + if (NULL == mmu_pp_bcast_core) { + MALI_PRINT_ERROR(("Failed to create MMU PP broadcast object\n")); + mali_group_delete(group); + return _MALI_OSK_ERR_FAULT; + } + + /* Create the PP core object inside this group */ + pp_bcast_core = mali_pp_create(resource_pp_bcast, group, MALI_TRUE, 0); + if (NULL == pp_bcast_core) { + /* No need to clean up now, as we will clean up everything linked in from the cluster when we fail this function */ + MALI_PRINT_ERROR(("Failed to create PP object\n")); + mali_group_delete(group); + return _MALI_OSK_ERR_FAULT; + } + + return _MALI_OSK_ERR_OK; +} + +static _mali_osk_errcode_t mali_parse_config_groups(void) +{ + struct mali_group *group; + int cluster_id_gp = 0; + int cluster_id_pp_grp0 = 0; + int cluster_id_pp_grp1 = 0; + int i; + + _mali_osk_resource_t resource_gp; + _mali_osk_resource_t resource_gp_mmu; + _mali_osk_resource_t resource_pp[8]; + _mali_osk_resource_t resource_pp_mmu[8]; + _mali_osk_resource_t resource_pp_mmu_bcast; + _mali_osk_resource_t resource_pp_bcast; + _mali_osk_resource_t resource_dlbu; + _mali_osk_resource_t resource_bcast; + _mali_osk_errcode_t resource_gp_found; + _mali_osk_errcode_t resource_gp_mmu_found; + _mali_osk_errcode_t resource_pp_found[8]; + _mali_osk_errcode_t resource_pp_mmu_found[8]; + _mali_osk_errcode_t resource_pp_mmu_bcast_found; + _mali_osk_errcode_t resource_pp_bcast_found; + _mali_osk_errcode_t resource_dlbu_found; + _mali_osk_errcode_t resource_bcast_found; + + if (!(mali_is_mali400() || mali_is_mali450() || mali_is_mali470())) { + /* No known HW core */ + return _MALI_OSK_ERR_FAULT; + } + + if (MALI_MAX_JOB_RUNTIME_DEFAULT == mali_max_job_runtime) { + /* Group settings are not overridden by module parameters, so use device settings */ + _mali_osk_device_data data = { 0, }; + + if (_MALI_OSK_ERR_OK == _mali_osk_device_data_get(&data)) { + /* Use device specific settings (if defined) */ + if (0 != data.max_job_runtime) { + mali_max_job_runtime = data.max_job_runtime; + } + } + } + + if (mali_is_mali450()) { + /* Mali-450 have separate L2s for GP, and PP core group(s) */ + cluster_id_pp_grp0 = 1; + cluster_id_pp_grp1 = 2; + } + + resource_gp_found = _mali_osk_resource_find(MALI_OFFSET_GP, &resource_gp); + resource_gp_mmu_found = _mali_osk_resource_find(MALI_OFFSET_GP_MMU, &resource_gp_mmu); + resource_pp_found[0] = _mali_osk_resource_find(MALI_OFFSET_PP0, &(resource_pp[0])); + resource_pp_found[1] = _mali_osk_resource_find(MALI_OFFSET_PP1, &(resource_pp[1])); + resource_pp_found[2] = _mali_osk_resource_find(MALI_OFFSET_PP2, &(resource_pp[2])); + resource_pp_found[3] = _mali_osk_resource_find(MALI_OFFSET_PP3, &(resource_pp[3])); + resource_pp_found[4] = _mali_osk_resource_find(MALI_OFFSET_PP4, &(resource_pp[4])); + resource_pp_found[5] = _mali_osk_resource_find(MALI_OFFSET_PP5, &(resource_pp[5])); + resource_pp_found[6] = _mali_osk_resource_find(MALI_OFFSET_PP6, &(resource_pp[6])); + resource_pp_found[7] = _mali_osk_resource_find(MALI_OFFSET_PP7, &(resource_pp[7])); + resource_pp_mmu_found[0] = _mali_osk_resource_find(MALI_OFFSET_PP0_MMU, &(resource_pp_mmu[0])); + resource_pp_mmu_found[1] = _mali_osk_resource_find(MALI_OFFSET_PP1_MMU, &(resource_pp_mmu[1])); + resource_pp_mmu_found[2] = _mali_osk_resource_find(MALI_OFFSET_PP2_MMU, &(resource_pp_mmu[2])); + resource_pp_mmu_found[3] = _mali_osk_resource_find(MALI_OFFSET_PP3_MMU, &(resource_pp_mmu[3])); + resource_pp_mmu_found[4] = _mali_osk_resource_find(MALI_OFFSET_PP4_MMU, &(resource_pp_mmu[4])); + resource_pp_mmu_found[5] = _mali_osk_resource_find(MALI_OFFSET_PP5_MMU, &(resource_pp_mmu[5])); + resource_pp_mmu_found[6] = _mali_osk_resource_find(MALI_OFFSET_PP6_MMU, &(resource_pp_mmu[6])); + resource_pp_mmu_found[7] = _mali_osk_resource_find(MALI_OFFSET_PP7_MMU, &(resource_pp_mmu[7])); + + + if (mali_is_mali450() || mali_is_mali470()) { + resource_bcast_found = _mali_osk_resource_find(MALI_OFFSET_BCAST, &resource_bcast); + resource_dlbu_found = _mali_osk_resource_find(MALI_OFFSET_DLBU, &resource_dlbu); + resource_pp_mmu_bcast_found = _mali_osk_resource_find(MALI_OFFSET_PP_BCAST_MMU, &resource_pp_mmu_bcast); + resource_pp_bcast_found = _mali_osk_resource_find(MALI_OFFSET_PP_BCAST, &resource_pp_bcast); + + if (_MALI_OSK_ERR_OK != resource_bcast_found || + _MALI_OSK_ERR_OK != resource_dlbu_found || + _MALI_OSK_ERR_OK != resource_pp_mmu_bcast_found || + _MALI_OSK_ERR_OK != resource_pp_bcast_found) { + /* Missing mandatory core(s) for Mali-450 or Mali-470 */ + MALI_DEBUG_PRINT(2, ("Missing mandatory resources, Mali-450 needs DLBU, Broadcast unit, virtual PP core and virtual MMU\n")); + return _MALI_OSK_ERR_FAULT; + } + } + + if (_MALI_OSK_ERR_OK != resource_gp_found || + _MALI_OSK_ERR_OK != resource_gp_mmu_found || + _MALI_OSK_ERR_OK != resource_pp_found[0] || + _MALI_OSK_ERR_OK != resource_pp_mmu_found[0]) { + /* Missing mandatory core(s) */ + MALI_DEBUG_PRINT(2, ("Missing mandatory resource, need at least one GP and one PP, both with a separate MMU\n")); + return _MALI_OSK_ERR_FAULT; + } + + MALI_DEBUG_ASSERT(1 <= mali_l2_cache_core_get_glob_num_l2_cores()); + group = mali_create_group(mali_l2_cache_core_get_glob_l2_core(cluster_id_gp), &resource_gp_mmu, &resource_gp, NULL, MALI_DOMAIN_INDEX_GP); + if (NULL == group) { + return _MALI_OSK_ERR_FAULT; + } + + /* Create group for first (and mandatory) PP core */ + MALI_DEBUG_ASSERT(mali_l2_cache_core_get_glob_num_l2_cores() >= (cluster_id_pp_grp0 + 1)); /* >= 1 on Mali-300 and Mali-400, >= 2 on Mali-450 */ + group = mali_create_group(mali_l2_cache_core_get_glob_l2_core(cluster_id_pp_grp0), &resource_pp_mmu[0], NULL, &resource_pp[0], MALI_DOMAIN_INDEX_PP0); + if (NULL == group) { + return _MALI_OSK_ERR_FAULT; + } + + mali_inited_pp_cores_group_1++; + + /* Create groups for rest of the cores in the first PP core group */ + for (i = 1; i < 4; i++) { /* First half of the PP cores belong to first core group */ + if (mali_inited_pp_cores_group_1 < mali_max_pp_cores_group_1) { + if (_MALI_OSK_ERR_OK == resource_pp_found[i] && _MALI_OSK_ERR_OK == resource_pp_mmu_found[i]) { + group = mali_create_group(mali_l2_cache_core_get_glob_l2_core(cluster_id_pp_grp0), &resource_pp_mmu[i], NULL, &resource_pp[i], MALI_DOMAIN_INDEX_PP0 + i); + if (NULL == group) { + return _MALI_OSK_ERR_FAULT; + } + + mali_inited_pp_cores_group_1++; + } + } + } + + /* Create groups for cores in the second PP core group */ + for (i = 4; i < 8; i++) { /* Second half of the PP cores belong to second core group */ + if (mali_inited_pp_cores_group_2 < mali_max_pp_cores_group_2) { + if (_MALI_OSK_ERR_OK == resource_pp_found[i] && _MALI_OSK_ERR_OK == resource_pp_mmu_found[i]) { + MALI_DEBUG_ASSERT(mali_l2_cache_core_get_glob_num_l2_cores() >= 2); /* Only Mali-450 have a second core group */ + group = mali_create_group(mali_l2_cache_core_get_glob_l2_core(cluster_id_pp_grp1), &resource_pp_mmu[i], NULL, &resource_pp[i], MALI_DOMAIN_INDEX_PP0 + i); + if (NULL == group) { + return _MALI_OSK_ERR_FAULT; + } + + mali_inited_pp_cores_group_2++; + } + } + } + + if (mali_is_mali450() || mali_is_mali470()) { + _mali_osk_errcode_t err = mali_create_virtual_group(&resource_pp_mmu_bcast, &resource_pp_bcast, &resource_dlbu, &resource_bcast); + if (_MALI_OSK_ERR_OK != err) { + return err; + } + } + + mali_max_pp_cores_group_1 = mali_inited_pp_cores_group_1; + mali_max_pp_cores_group_2 = mali_inited_pp_cores_group_2; + MALI_DEBUG_PRINT(2, ("%d+%d PP cores initialized\n", mali_inited_pp_cores_group_1, mali_inited_pp_cores_group_2)); + + return _MALI_OSK_ERR_OK; +} + +static _mali_osk_errcode_t mali_check_shared_interrupts(void) +{ +#if !defined(CONFIG_MALI_SHARED_INTERRUPTS) + if (MALI_TRUE == _mali_osk_shared_interrupts()) { + MALI_PRINT_ERROR(("Shared interrupts detected, but driver support is not enabled\n")); + return _MALI_OSK_ERR_FAULT; + } +#endif /* !defined(CONFIG_MALI_SHARED_INTERRUPTS) */ + + /* It is OK to compile support for shared interrupts even if Mali is not using it. */ + return _MALI_OSK_ERR_OK; +} + +static _mali_osk_errcode_t mali_parse_config_pmu(void) +{ + _mali_osk_resource_t resource_pmu; + + MALI_DEBUG_ASSERT(0 != global_gpu_base_address); + + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI_OFFSET_PMU, &resource_pmu)) { + struct mali_pmu_core *pmu; + + pmu = mali_pmu_create(&resource_pmu); + if (NULL == pmu) { + MALI_PRINT_ERROR(("Failed to create PMU\n")); + return _MALI_OSK_ERR_FAULT; + } + } + + /* It's ok if the PMU doesn't exist */ + return _MALI_OSK_ERR_OK; +} + +static _mali_osk_errcode_t mali_parse_config_memory(void) +{ + _mali_osk_device_data data = { 0, }; + _mali_osk_errcode_t ret; + + /* The priority of setting the value of mali_shared_mem_size, + * mali_dedicated_mem_start and mali_dedicated_mem_size: + * 1. module parameter; + * 2. platform data; + * 3. default value; + **/ + if (_MALI_OSK_ERR_OK == _mali_osk_device_data_get(&data)) { + /* Memory settings are not overridden by module parameters, so use device settings */ + if (0 == mali_dedicated_mem_start && 0 == mali_dedicated_mem_size) { + /* Use device specific settings (if defined) */ + mali_dedicated_mem_start = data.dedicated_mem_start; + mali_dedicated_mem_size = data.dedicated_mem_size; + } + + if (MALI_SHARED_MEMORY_DEFAULT_SIZE == mali_shared_mem_size && + 0 != data.shared_mem_size) { + mali_shared_mem_size = data.shared_mem_size; + } + } + + if (0 < mali_dedicated_mem_size && 0 != mali_dedicated_mem_start) { + MALI_DEBUG_PRINT(2, ("Mali memory settings (dedicated: 0x%08X@0x%08X)\n", + mali_dedicated_mem_size, mali_dedicated_mem_start)); + + /* Dedicated memory */ + ret = mali_memory_core_resource_dedicated_memory(mali_dedicated_mem_start, mali_dedicated_mem_size); + if (_MALI_OSK_ERR_OK != ret) { + MALI_PRINT_ERROR(("Failed to register dedicated memory\n")); + mali_memory_terminate(); + return ret; + } + } + + if (0 < mali_shared_mem_size) { + MALI_DEBUG_PRINT(2, ("Mali memory settings (shared: 0x%08X)\n", mali_shared_mem_size)); + + /* Shared OS memory */ + ret = mali_memory_core_resource_os_memory(mali_shared_mem_size); + if (_MALI_OSK_ERR_OK != ret) { + MALI_PRINT_ERROR(("Failed to register shared OS memory\n")); + mali_memory_terminate(); + return ret; + } + } + + if (0 == mali_fb_start && 0 == mali_fb_size) { + /* Frame buffer settings are not overridden by module parameters, so use device settings */ + _mali_osk_device_data data = { 0, }; + + if (_MALI_OSK_ERR_OK == _mali_osk_device_data_get(&data)) { + /* Use device specific settings (if defined) */ + mali_fb_start = data.fb_start; + mali_fb_size = data.fb_size; + } + + MALI_DEBUG_PRINT(2, ("Using device defined frame buffer settings (0x%08X@0x%08X)\n", + mali_fb_size, mali_fb_start)); + } else { + MALI_DEBUG_PRINT(2, ("Using module defined frame buffer settings (0x%08X@0x%08X)\n", + mali_fb_size, mali_fb_start)); + } + + if (0 != mali_fb_size) { + /* Register frame buffer */ + ret = mali_mem_validation_add_range(mali_fb_start, mali_fb_size); + if (_MALI_OSK_ERR_OK != ret) { + MALI_PRINT_ERROR(("Failed to register frame buffer memory region\n")); + mali_memory_terminate(); + return ret; + } + } + + return _MALI_OSK_ERR_OK; +} + +static void mali_detect_gpu_class(void) +{ + if (_mali_osk_identify_gpu_resource() == 0x450) + mali_gpu_class_is_mali450 = MALI_TRUE; + + if (_mali_osk_identify_gpu_resource() == 0x470) + mali_gpu_class_is_mali470 = MALI_TRUE; +} + +static _mali_osk_errcode_t mali_init_hw_reset(void) +{ +#if (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) + _mali_osk_resource_t resource_bcast; + + /* Ensure broadcast unit is in a good state before we start creating + * groups and cores. + */ + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI_OFFSET_BCAST, &resource_bcast)) { + struct mali_bcast_unit *bcast_core; + + bcast_core = mali_bcast_unit_create(&resource_bcast); + if (NULL == bcast_core) { + MALI_PRINT_ERROR(("Failed to create Broadcast unit object!\n")); + return _MALI_OSK_ERR_FAULT; + } + mali_bcast_unit_delete(bcast_core); + } +#endif /* (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) */ + + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t mali_initialize_subsystems(void) +{ + _mali_osk_errcode_t err; + +#if defined(CONFIG_MALI_DT) && !defined(CONFIG_MALI_PLAT_SPECIFIC_DT) + err = _mali_osk_resource_initialize(); + if (_MALI_OSK_ERR_OK != err) { + mali_terminate_subsystems(); + return err; + } +#endif + + mali_pp_job_initialize(); + + mali_timeline_initialize(); + + err = mali_session_initialize(); + if (_MALI_OSK_ERR_OK != err) { + mali_terminate_subsystems(); + return err; + } + +#if defined(CONFIG_MALI400_PROFILING) + err = _mali_osk_profiling_init(mali_boot_profiling ? MALI_TRUE : MALI_FALSE); + if (_MALI_OSK_ERR_OK != err) { + /* No biggie if we weren't able to initialize the profiling */ + MALI_PRINT_ERROR(("Failed to initialize profiling, feature will be unavailable\n")); + } +#endif + + err = mali_memory_initialize(); + if (_MALI_OSK_ERR_OK != err) { + mali_terminate_subsystems(); + return err; + } + + err = mali_executor_initialize(); + if (_MALI_OSK_ERR_OK != err) { + mali_terminate_subsystems(); + return err; + } + + err = mali_scheduler_initialize(); + if (_MALI_OSK_ERR_OK != err) { + mali_terminate_subsystems(); + return err; + } + + /* Configure memory early, needed by mali_mmu_initialize. */ + err = mali_parse_config_memory(); + if (_MALI_OSK_ERR_OK != err) { + mali_terminate_subsystems(); + return err; + } + + err = mali_set_global_gpu_base_address(); + if (_MALI_OSK_ERR_OK != err) { + mali_terminate_subsystems(); + return err; + } + + /* Detect GPU class (uses L2 cache count) */ + mali_detect_gpu_class(); + + err = mali_check_shared_interrupts(); + if (_MALI_OSK_ERR_OK != err) { + mali_terminate_subsystems(); + return err; + } + + /* Initialize the MALI PMU (will not touch HW!) */ + err = mali_parse_config_pmu(); + if (_MALI_OSK_ERR_OK != err) { + mali_terminate_subsystems(); + return err; + } + + /* Initialize the power management module */ + err = mali_pm_initialize(); + if (_MALI_OSK_ERR_OK != err) { + mali_terminate_subsystems(); + return err; + } + + /* Make sure the entire GPU stays on for the rest of this function */ + mali_pm_init_begin(); + + /* Ensure HW is in a good state before starting to access cores. */ + err = mali_init_hw_reset(); + if (_MALI_OSK_ERR_OK != err) { + mali_terminate_subsystems(); + return err; + } + + /* Detect which Mali GPU we are dealing with */ + err = mali_parse_product_info(); + if (_MALI_OSK_ERR_OK != err) { + mali_pm_init_end(); + mali_terminate_subsystems(); + return err; + } + + /* The global_product_id is now populated with the correct Mali GPU */ + + /* Start configuring the actual Mali hardware. */ + + err = mali_mmu_initialize(); + if (_MALI_OSK_ERR_OK != err) { + mali_pm_init_end(); + mali_terminate_subsystems(); + return err; + } + + if (mali_is_mali450() || mali_is_mali470()) { + err = mali_dlbu_initialize(); + if (_MALI_OSK_ERR_OK != err) { + mali_pm_init_end(); + mali_terminate_subsystems(); + return err; + } + } + + err = mali_parse_config_l2_cache(); + if (_MALI_OSK_ERR_OK != err) { + mali_pm_init_end(); + mali_terminate_subsystems(); + return err; + } + + err = mali_parse_config_groups(); + if (_MALI_OSK_ERR_OK != err) { + mali_pm_init_end(); + mali_terminate_subsystems(); + return err; + } + + /* Move groups into executor */ + mali_executor_populate(); + + /* Need call after all group has assigned a domain */ + mali_pm_power_cost_setup(); + + /* Initialize the GPU timer */ + err = mali_control_timer_init(); + if (_MALI_OSK_ERR_OK != err) { + mali_pm_init_end(); + mali_terminate_subsystems(); + return err; + } + + /* Initialize the GPU utilization tracking */ + err = mali_utilization_init(); + if (_MALI_OSK_ERR_OK != err) { + mali_pm_init_end(); + mali_terminate_subsystems(); + return err; + } + +#if defined(CONFIG_MALI_DVFS) + err = mali_dvfs_policy_init(); + if (_MALI_OSK_ERR_OK != err) { + mali_pm_init_end(); + mali_terminate_subsystems(); + return err; + } +#endif + + /* Allowing the system to be turned off */ + mali_pm_init_end(); + + return _MALI_OSK_ERR_OK; /* all ok */ +} + +void mali_terminate_subsystems(void) +{ + struct mali_pmu_core *pmu = mali_pmu_get_global_pmu_core(); + + MALI_DEBUG_PRINT(2, ("terminate_subsystems() called\n")); + + mali_utilization_term(); + mali_control_timer_term(); + + mali_executor_depopulate(); + mali_delete_groups(); /* Delete groups not added to executor */ + mali_executor_terminate(); + + mali_scheduler_terminate(); + mali_pp_job_terminate(); + mali_delete_l2_cache_cores(); + mali_mmu_terminate(); + + if (mali_is_mali450() || mali_is_mali470()) { + mali_dlbu_terminate(); + } + + mali_pm_terminate(); + + if (NULL != pmu) { + mali_pmu_delete(pmu); + } + +#if defined(CONFIG_MALI400_PROFILING) + _mali_osk_profiling_term(); +#endif + + mali_memory_terminate(); + + mali_session_terminate(); + + mali_timeline_terminate(); + + global_gpu_base_address = 0; +} + +_mali_product_id_t mali_kernel_core_get_product_id(void) +{ + return global_product_id; +} + +u32 mali_kernel_core_get_gpu_major_version(void) +{ + return global_gpu_major_version; +} + +u32 mali_kernel_core_get_gpu_minor_version(void) +{ + return global_gpu_minor_version; +} + +_mali_osk_errcode_t _mali_ukk_get_api_version(_mali_uk_get_api_version_s *args) +{ + MALI_DEBUG_ASSERT_POINTER(args); + MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); + + /* check compatability */ + if (args->version == _MALI_UK_API_VERSION) { + args->compatible = 1; + } else { + args->compatible = 0; + } + + args->version = _MALI_UK_API_VERSION; /* report our version */ + + /* success regardless of being compatible or not */ + MALI_SUCCESS; +} + +_mali_osk_errcode_t _mali_ukk_get_api_version_v2(_mali_uk_get_api_version_v2_s *args) +{ + MALI_DEBUG_ASSERT_POINTER(args); + MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); + + /* check compatability */ + if (args->version == _MALI_UK_API_VERSION) { + args->compatible = 1; + } else { + args->compatible = 0; + } + + args->version = _MALI_UK_API_VERSION; /* report our version */ + + /* success regardless of being compatible or not */ + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t _mali_ukk_wait_for_notification(_mali_uk_wait_for_notification_s *args) +{ + _mali_osk_errcode_t err; + _mali_osk_notification_t *notification; + _mali_osk_notification_queue_t *queue; + struct mali_session_data *session; + + /* check input */ + MALI_DEBUG_ASSERT_POINTER(args); + MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); + + session = (struct mali_session_data *)(uintptr_t)args->ctx; + queue = session->ioctl_queue; + + /* if the queue does not exist we're currently shutting down */ + if (NULL == queue) { + MALI_DEBUG_PRINT(1, ("No notification queue registered with the session. Asking userspace to stop querying\n")); + args->type = _MALI_NOTIFICATION_CORE_SHUTDOWN_IN_PROGRESS; + return _MALI_OSK_ERR_OK; + } + + /* receive a notification, might sleep */ + err = _mali_osk_notification_queue_receive(queue, ¬ification); + if (_MALI_OSK_ERR_OK != err) { + MALI_ERROR(err); /* errcode returned, pass on to caller */ + } + + /* copy the buffer to the user */ + args->type = (_mali_uk_notification_type)notification->notification_type; + _mali_osk_memcpy(&args->data, notification->result_buffer, notification->result_buffer_size); + + /* finished with the notification */ + _mali_osk_notification_delete(notification); + + return _MALI_OSK_ERR_OK; /* all ok */ +} + +_mali_osk_errcode_t _mali_ukk_post_notification(_mali_uk_post_notification_s *args) +{ + _mali_osk_notification_t *notification; + _mali_osk_notification_queue_t *queue; + struct mali_session_data *session; + + /* check input */ + MALI_DEBUG_ASSERT_POINTER(args); + MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); + + session = (struct mali_session_data *)(uintptr_t)args->ctx; + queue = session->ioctl_queue; + + /* if the queue does not exist we're currently shutting down */ + if (NULL == queue) { + MALI_DEBUG_PRINT(1, ("No notification queue registered with the session. Asking userspace to stop querying\n")); + return _MALI_OSK_ERR_OK; + } + + notification = _mali_osk_notification_create(args->type, 0); + if (NULL == notification) { + MALI_PRINT_ERROR(("Failed to create notification object\n")); + return _MALI_OSK_ERR_NOMEM; + } + + _mali_osk_notification_queue_send(queue, notification); + + return _MALI_OSK_ERR_OK; /* all ok */ +} + +_mali_osk_errcode_t _mali_ukk_pending_submit(_mali_uk_pending_submit_s *args) +{ + wait_queue_head_t *queue; + + /* check input */ + MALI_DEBUG_ASSERT_POINTER(args); + MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); + + queue = mali_session_get_wait_queue(); + + /* check pending big job number, might sleep if larger than MAX allowed number */ + if (wait_event_interruptible(*queue, MALI_MAX_PENDING_BIG_JOB > mali_scheduler_job_gp_big_job_count())) { + return _MALI_OSK_ERR_RESTARTSYSCALL; + } + + return _MALI_OSK_ERR_OK; /* all ok */ +} + + +_mali_osk_errcode_t _mali_ukk_request_high_priority(_mali_uk_request_high_priority_s *args) +{ + struct mali_session_data *session; + + MALI_DEBUG_ASSERT_POINTER(args); + MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); + + session = (struct mali_session_data *)(uintptr_t)args->ctx; + + if (!session->use_high_priority_job_queue) { + session->use_high_priority_job_queue = MALI_TRUE; + MALI_DEBUG_PRINT(2, ("Session 0x%08X with pid %d was granted higher priority.\n", session, _mali_osk_get_pid())); + } + + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t _mali_ukk_open(void **context) +{ + u32 i; + struct mali_session_data *session; + + /* allocated struct to track this session */ + session = (struct mali_session_data *)_mali_osk_calloc(1, sizeof(struct mali_session_data)); + MALI_CHECK_NON_NULL(session, _MALI_OSK_ERR_NOMEM); + + MALI_DEBUG_PRINT(3, ("Session starting\n")); + + /* create a response queue for this session */ + session->ioctl_queue = _mali_osk_notification_queue_init(); + if (NULL == session->ioctl_queue) { + goto err; + } + + session->page_directory = mali_mmu_pagedir_alloc(); + if (NULL == session->page_directory) { + goto err_mmu; + } + + if (_MALI_OSK_ERR_OK != mali_mmu_pagedir_map(session->page_directory, MALI_DLBU_VIRT_ADDR, _MALI_OSK_MALI_PAGE_SIZE)) { + MALI_PRINT_ERROR(("Failed to map DLBU page into session\n")); + goto err_mmu; + } + + if (0 != mali_dlbu_phys_addr) { + mali_mmu_pagedir_update(session->page_directory, MALI_DLBU_VIRT_ADDR, mali_dlbu_phys_addr, + _MALI_OSK_MALI_PAGE_SIZE, MALI_MMU_FLAGS_DEFAULT); + } + + if (_MALI_OSK_ERR_OK != mali_memory_session_begin(session)) { + goto err_session; + } + + /* Create soft system. */ + session->soft_job_system = mali_soft_job_system_create(session); + if (NULL == session->soft_job_system) { + goto err_soft; + } + + /* Create timeline system. */ + session->timeline_system = mali_timeline_system_create(session); + if (NULL == session->timeline_system) { + goto err_time_line; + } + +#if defined(CONFIG_MALI_DVFS) + _mali_osk_atomic_init(&session->number_of_window_jobs, 0); +#endif + + session->use_high_priority_job_queue = MALI_FALSE; + + /* Initialize list of PP jobs on this session. */ + _MALI_OSK_INIT_LIST_HEAD(&session->pp_job_list); + + /* Initialize the pp_job_fb_lookup_list array used to quickly lookup jobs from a given frame builder */ + for (i = 0; i < MALI_PP_JOB_FB_LOOKUP_LIST_SIZE; ++i) { + _MALI_OSK_INIT_LIST_HEAD(&session->pp_job_fb_lookup_list[i]); + } + + session->pid = _mali_osk_get_pid(); + session->comm = _mali_osk_get_comm(); + session->max_mali_mem_allocated_size = 0; + for (i = 0; i < MALI_MEM_TYPE_MAX; i ++) { + atomic_set(&session->mali_mem_array[i], 0); + } + atomic_set(&session->mali_mem_allocated_pages, 0); + *context = (void *)session; + + /* Add session to the list of all sessions. */ + mali_session_add(session); + + MALI_DEBUG_PRINT(3, ("Session started\n")); + return _MALI_OSK_ERR_OK; + +err_time_line: + mali_soft_job_system_destroy(session->soft_job_system); +err_soft: + mali_memory_session_end(session); +err_session: + mali_mmu_pagedir_free(session->page_directory); +err_mmu: + _mali_osk_notification_queue_term(session->ioctl_queue); +err: + _mali_osk_free(session); + MALI_ERROR(_MALI_OSK_ERR_NOMEM); + +} + +#if defined(DEBUG) +/* parameter used for debug */ +extern u32 num_pm_runtime_resume; +extern u32 num_pm_updates; +extern u32 num_pm_updates_up; +extern u32 num_pm_updates_down; +#endif + +_mali_osk_errcode_t _mali_ukk_close(void **context) +{ + struct mali_session_data *session; + MALI_CHECK_NON_NULL(context, _MALI_OSK_ERR_INVALID_ARGS); + session = (struct mali_session_data *)*context; + + MALI_DEBUG_PRINT(3, ("Session ending\n")); + + MALI_DEBUG_ASSERT_POINTER(session->soft_job_system); + MALI_DEBUG_ASSERT_POINTER(session->timeline_system); + + /* Remove session from list of all sessions. */ + mali_session_remove(session); + + /* This flag is used to prevent queueing of jobs due to activation. */ + session->is_aborting = MALI_TRUE; + + /* Stop the soft job timer. */ + mali_timeline_system_stop_timer(session->timeline_system); + + /* Abort queued jobs */ + mali_scheduler_abort_session(session); + + /* Abort executing jobs */ + mali_executor_abort_session(session); + + /* Abort the soft job system. */ + mali_soft_job_system_abort(session->soft_job_system); + + /* Force execution of all pending bottom half processing for GP and PP. */ + _mali_osk_wq_flush(); + + /* The session PP list should now be empty. */ + MALI_DEBUG_ASSERT(_mali_osk_list_empty(&session->pp_job_list)); + + /* At this point the GP and PP scheduler no longer has any jobs queued or running from this + * session, and all soft jobs in the soft job system has been destroyed. */ + + /* Any trackers left in the timeline system are directly or indirectly waiting on external + * sync fences. Cancel all sync fence waiters to trigger activation of all remaining + * trackers. This call will sleep until all timelines are empty. */ + mali_timeline_system_abort(session->timeline_system); + + /* Flush pending work. + * Needed to make sure all bottom half processing related to this + * session has been completed, before we free internal data structures. + */ + _mali_osk_wq_flush(); + + /* Destroy timeline system. */ + mali_timeline_system_destroy(session->timeline_system); + session->timeline_system = NULL; + + /* Destroy soft system. */ + mali_soft_job_system_destroy(session->soft_job_system); + session->soft_job_system = NULL; + + MALI_DEBUG_CODE({ + /* Check that the pp_job_fb_lookup_list array is empty. */ + u32 i; + for (i = 0; i < MALI_PP_JOB_FB_LOOKUP_LIST_SIZE; ++i) + { + MALI_DEBUG_ASSERT(_mali_osk_list_empty(&session->pp_job_fb_lookup_list[i])); + } + }); + + /* Free remaining memory allocated to this session */ + mali_memory_session_end(session); + +#if defined(CONFIG_MALI_DVFS) + _mali_osk_atomic_term(&session->number_of_window_jobs); +#endif + +#if defined(CONFIG_MALI400_PROFILING) + _mali_osk_profiling_stop_sampling(session->pid); +#endif + + /* Free session data structures */ + mali_mmu_pagedir_unmap(session->page_directory, MALI_DLBU_VIRT_ADDR, _MALI_OSK_MALI_PAGE_SIZE); + mali_mmu_pagedir_free(session->page_directory); + _mali_osk_notification_queue_term(session->ioctl_queue); + _mali_osk_free(session); + + *context = NULL; + + MALI_DEBUG_PRINT(3, ("Session has ended\n")); + +#if defined(DEBUG) + MALI_DEBUG_PRINT(3, ("Stats: # runtime resumes: %u\n", num_pm_runtime_resume)); + MALI_DEBUG_PRINT(3, (" # PM updates: .... %u (up %u, down %u)\n", num_pm_updates, num_pm_updates_up, num_pm_updates_down)); + + num_pm_runtime_resume = 0; + num_pm_updates = 0; + num_pm_updates_up = 0; + num_pm_updates_down = 0; +#endif + + return _MALI_OSK_ERR_OK;; +} + +#if MALI_STATE_TRACKING +u32 _mali_kernel_core_dump_state(char *buf, u32 size) +{ + int n = 0; /* Number of bytes written to buf */ + + n += mali_scheduler_dump_state(buf + n, size - n); + n += mali_executor_dump_state(buf + n, size - n); + + return n; +} +#endif diff --git a/drivers/gpu/arm/utgard/common/mali_kernel_core.h b/drivers/gpu/arm/utgard/common/mali_kernel_core.h new file mode 100644 index 00000000000000..8cdbc5af3205fd --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_kernel_core.h @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_KERNEL_CORE_H__ +#define __MALI_KERNEL_CORE_H__ + +#include "mali_osk.h" + +typedef enum { + _MALI_PRODUCT_ID_UNKNOWN, + _MALI_PRODUCT_ID_MALI200, + _MALI_PRODUCT_ID_MALI300, + _MALI_PRODUCT_ID_MALI400, + _MALI_PRODUCT_ID_MALI450, + _MALI_PRODUCT_ID_MALI470, +} _mali_product_id_t; + +extern mali_bool mali_gpu_class_is_mali450; +extern mali_bool mali_gpu_class_is_mali470; + +_mali_osk_errcode_t mali_initialize_subsystems(void); + +void mali_terminate_subsystems(void); + +_mali_product_id_t mali_kernel_core_get_product_id(void); + +u32 mali_kernel_core_get_gpu_major_version(void); + +u32 mali_kernel_core_get_gpu_minor_version(void); + +u32 _mali_kernel_core_dump_state(char *buf, u32 size); + +MALI_STATIC_INLINE mali_bool mali_is_mali470(void) +{ + return mali_gpu_class_is_mali470; +} + +MALI_STATIC_INLINE mali_bool mali_is_mali450(void) +{ + return mali_gpu_class_is_mali450; +} + +MALI_STATIC_INLINE mali_bool mali_is_mali400(void) +{ + if (mali_gpu_class_is_mali450 || mali_gpu_class_is_mali470) + return MALI_FALSE; + + return MALI_TRUE; +} +#endif /* __MALI_KERNEL_CORE_H__ */ diff --git a/drivers/gpu/arm/utgard/common/mali_kernel_utilization.c b/drivers/gpu/arm/utgard/common/mali_kernel_utilization.c new file mode 100644 index 00000000000000..63b94174224963 --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_kernel_utilization.c @@ -0,0 +1,440 @@ +/* + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_kernel_utilization.h" +#include "mali_osk.h" +#include "mali_osk_mali.h" +#include "mali_kernel_common.h" +#include "mali_session.h" +#include "mali_scheduler.h" + +#include "mali_executor.h" +#include "mali_dvfs_policy.h" +#include "mali_control_timer.h" + +/* Thresholds for GP bound detection. */ +#define MALI_GP_BOUND_GP_UTILIZATION_THRESHOLD 240 +#define MALI_GP_BOUND_PP_UTILIZATION_THRESHOLD 250 + +static _mali_osk_spinlock_irq_t *utilization_data_lock; + +static u32 num_running_gp_cores = 0; +static u32 num_running_pp_cores = 0; + +static u64 work_start_time_gpu = 0; +static u64 work_start_time_gp = 0; +static u64 work_start_time_pp = 0; +static u64 accumulated_work_time_gpu = 0; +static u64 accumulated_work_time_gp = 0; +static u64 accumulated_work_time_pp = 0; + +static u32 last_utilization_gpu = 0 ; +static u32 last_utilization_gp = 0 ; +static u32 last_utilization_pp = 0 ; + +void (*mali_utilization_callback)(struct mali_gpu_utilization_data *data) = NULL; + +/* Define the first timer control timer timeout in milliseconds */ +static u32 mali_control_first_timeout = 100; +static struct mali_gpu_utilization_data mali_util_data = {0, }; + +struct mali_gpu_utilization_data *mali_utilization_calculate(u64 *start_time, u64 *time_period, mali_bool *need_add_timer) +{ + u64 time_now; + u32 leading_zeroes; + u32 shift_val; + u32 work_normalized_gpu; + u32 work_normalized_gp; + u32 work_normalized_pp; + u32 period_normalized; + u32 utilization_gpu; + u32 utilization_gp; + u32 utilization_pp; + + mali_utilization_data_lock(); + + time_now = _mali_osk_time_get_ns(); + + *time_period = time_now - *start_time; + + if (accumulated_work_time_gpu == 0 && work_start_time_gpu == 0) { + mali_control_timer_pause(); + /* + * No work done for this period + * - No need to reschedule timer + * - Report zero usage + */ + last_utilization_gpu = 0; + last_utilization_gp = 0; + last_utilization_pp = 0; + + mali_util_data.utilization_gpu = last_utilization_gpu; + mali_util_data.utilization_gp = last_utilization_gp; + mali_util_data.utilization_pp = last_utilization_pp; + + mali_utilization_data_unlock(); + + *need_add_timer = MALI_FALSE; + + mali_executor_hint_disable(MALI_EXECUTOR_HINT_GP_BOUND); + + MALI_DEBUG_PRINT(4, ("last_utilization_gpu = %d \n", last_utilization_gpu)); + MALI_DEBUG_PRINT(4, ("last_utilization_gp = %d \n", last_utilization_gp)); + MALI_DEBUG_PRINT(4, ("last_utilization_pp = %d \n", last_utilization_pp)); + + return &mali_util_data; + } + + /* If we are currently busy, update working period up to now */ + if (work_start_time_gpu != 0) { + accumulated_work_time_gpu += (time_now - work_start_time_gpu); + work_start_time_gpu = time_now; + + /* GP and/or PP will also be busy if the GPU is busy at this point */ + + if (work_start_time_gp != 0) { + accumulated_work_time_gp += (time_now - work_start_time_gp); + work_start_time_gp = time_now; + } + + if (work_start_time_pp != 0) { + accumulated_work_time_pp += (time_now - work_start_time_pp); + work_start_time_pp = time_now; + } + } + + /* + * We have two 64-bit values, a dividend and a divisor. + * To avoid dependencies to a 64-bit divider, we shift down the two values + * equally first. + * We shift the dividend up and possibly the divisor down, making the result X in 256. + */ + + /* Shift the 64-bit values down so they fit inside a 32-bit integer */ + leading_zeroes = _mali_osk_clz((u32)(*time_period >> 32)); + shift_val = 32 - leading_zeroes; + work_normalized_gpu = (u32)(accumulated_work_time_gpu >> shift_val); + work_normalized_gp = (u32)(accumulated_work_time_gp >> shift_val); + work_normalized_pp = (u32)(accumulated_work_time_pp >> shift_val); + period_normalized = (u32)(*time_period >> shift_val); + + /* + * Now, we should report the usage in parts of 256 + * this means we must shift up the dividend or down the divisor by 8 + * (we could do a combination, but we just use one for simplicity, + * but the end result should be good enough anyway) + */ + if (period_normalized > 0x00FFFFFF) { + /* The divisor is so big that it is safe to shift it down */ + period_normalized >>= 8; + } else { + /* + * The divisor is so small that we can shift up the dividend, without loosing any data. + * (dividend is always smaller than the divisor) + */ + work_normalized_gpu <<= 8; + work_normalized_gp <<= 8; + work_normalized_pp <<= 8; + } + + utilization_gpu = work_normalized_gpu / period_normalized; + utilization_gp = work_normalized_gp / period_normalized; + utilization_pp = work_normalized_pp / period_normalized; + + last_utilization_gpu = utilization_gpu; + last_utilization_gp = utilization_gp; + last_utilization_pp = utilization_pp; + + if ((MALI_GP_BOUND_GP_UTILIZATION_THRESHOLD < last_utilization_gp) && + (MALI_GP_BOUND_PP_UTILIZATION_THRESHOLD > last_utilization_pp)) { + mali_executor_hint_enable(MALI_EXECUTOR_HINT_GP_BOUND); + } else { + mali_executor_hint_disable(MALI_EXECUTOR_HINT_GP_BOUND); + } + + /* starting a new period */ + accumulated_work_time_gpu = 0; + accumulated_work_time_gp = 0; + accumulated_work_time_pp = 0; + + *start_time = time_now; + + mali_util_data.utilization_gp = last_utilization_gp; + mali_util_data.utilization_gpu = last_utilization_gpu; + mali_util_data.utilization_pp = last_utilization_pp; + + mali_utilization_data_unlock(); + + *need_add_timer = MALI_TRUE; + + MALI_DEBUG_PRINT(4, ("last_utilization_gpu = %d \n", last_utilization_gpu)); + MALI_DEBUG_PRINT(4, ("last_utilization_gp = %d \n", last_utilization_gp)); + MALI_DEBUG_PRINT(4, ("last_utilization_pp = %d \n", last_utilization_pp)); + + return &mali_util_data; +} + +_mali_osk_errcode_t mali_utilization_init(void) +{ +#if USING_GPU_UTILIZATION + _mali_osk_device_data data; + + if (_MALI_OSK_ERR_OK == _mali_osk_device_data_get(&data)) { + if (NULL != data.utilization_callback) { + mali_utilization_callback = data.utilization_callback; + MALI_DEBUG_PRINT(2, ("Mali GPU Utilization: Utilization handler installed \n")); + } + } +#endif /* defined(USING_GPU_UTILIZATION) */ + + if (NULL == mali_utilization_callback) { + MALI_DEBUG_PRINT(2, ("Mali GPU Utilization: No platform utilization handler installed\n")); + } + + utilization_data_lock = _mali_osk_spinlock_irq_init(_MALI_OSK_LOCKFLAG_ORDERED, _MALI_OSK_LOCK_ORDER_UTILIZATION); + if (NULL == utilization_data_lock) { + return _MALI_OSK_ERR_FAULT; + } + + num_running_gp_cores = 0; + num_running_pp_cores = 0; + + return _MALI_OSK_ERR_OK; +} + +void mali_utilization_term(void) +{ + if (NULL != utilization_data_lock) { + _mali_osk_spinlock_irq_term(utilization_data_lock); + } +} + +void mali_utilization_gp_start(void) +{ + mali_utilization_data_lock(); + + ++num_running_gp_cores; + if (1 == num_running_gp_cores) { + u64 time_now = _mali_osk_time_get_ns(); + + /* First GP core started, consider GP busy from now and onwards */ + work_start_time_gp = time_now; + + if (0 == num_running_pp_cores) { + mali_bool is_resume = MALI_FALSE; + /* + * There are no PP cores running, so this is also the point + * at which we consider the GPU to be busy as well. + */ + work_start_time_gpu = time_now; + + is_resume = mali_control_timer_resume(time_now); + + mali_utilization_data_unlock(); + + if (is_resume) { + /* Do some policy in new period for performance consideration */ +#if defined(CONFIG_MALI_DVFS) + /* Clear session->number_of_window_jobs, prepare parameter for dvfs */ + mali_session_max_window_num(); + if (0 == last_utilization_gpu) { + /* + * for mali_dev_pause is called in set clock, + * so each time we change clock, we will set clock to + * highest step even if under down clock case, + * it is not nessesary, so we only set the clock under + * last time utilization equal 0, we stop the timer then + * start the GPU again case + */ + mali_dvfs_policy_new_period(); + } +#endif + /* + * First timeout using short interval for power consideration + * because we give full power in the new period, but if the + * job loading is light, finish in 10ms, the other time all keep + * in high freq it will wast time. + */ + mali_control_timer_add(mali_control_first_timeout); + } + } else { + mali_utilization_data_unlock(); + } + + } else { + /* Nothing to do */ + mali_utilization_data_unlock(); + } +} + +void mali_utilization_pp_start(void) +{ + mali_utilization_data_lock(); + + ++num_running_pp_cores; + if (1 == num_running_pp_cores) { + u64 time_now = _mali_osk_time_get_ns(); + + /* First PP core started, consider PP busy from now and onwards */ + work_start_time_pp = time_now; + + if (0 == num_running_gp_cores) { + mali_bool is_resume = MALI_FALSE; + /* + * There are no GP cores running, so this is also the point + * at which we consider the GPU to be busy as well. + */ + work_start_time_gpu = time_now; + + /* Start a new period if stoped */ + is_resume = mali_control_timer_resume(time_now); + + mali_utilization_data_unlock(); + + if (is_resume) { +#if defined(CONFIG_MALI_DVFS) + /* Clear session->number_of_window_jobs, prepare parameter for dvfs */ + mali_session_max_window_num(); + if (0 == last_utilization_gpu) { + /* + * for mali_dev_pause is called in set clock, + * so each time we change clock, we will set clock to + * highest step even if under down clock case, + * it is not nessesary, so we only set the clock under + * last time utilization equal 0, we stop the timer then + * start the GPU again case + */ + mali_dvfs_policy_new_period(); + } +#endif + + /* + * First timeout using short interval for power consideration + * because we give full power in the new period, but if the + * job loading is light, finish in 10ms, the other time all keep + * in high freq it will wast time. + */ + mali_control_timer_add(mali_control_first_timeout); + } + } else { + mali_utilization_data_unlock(); + } + } else { + /* Nothing to do */ + mali_utilization_data_unlock(); + } +} + +void mali_utilization_gp_end(void) +{ + mali_utilization_data_lock(); + + --num_running_gp_cores; + if (0 == num_running_gp_cores) { + u64 time_now = _mali_osk_time_get_ns(); + + /* Last GP core ended, consider GP idle from now and onwards */ + accumulated_work_time_gp += (time_now - work_start_time_gp); + work_start_time_gp = 0; + + if (0 == num_running_pp_cores) { + /* + * There are no PP cores running, so this is also the point + * at which we consider the GPU to be idle as well. + */ + accumulated_work_time_gpu += (time_now - work_start_time_gpu); + work_start_time_gpu = 0; + } + } + + mali_utilization_data_unlock(); +} + +void mali_utilization_pp_end(void) +{ + mali_utilization_data_lock(); + + --num_running_pp_cores; + if (0 == num_running_pp_cores) { + u64 time_now = _mali_osk_time_get_ns(); + + /* Last PP core ended, consider PP idle from now and onwards */ + accumulated_work_time_pp += (time_now - work_start_time_pp); + work_start_time_pp = 0; + + if (0 == num_running_gp_cores) { + /* + * There are no GP cores running, so this is also the point + * at which we consider the GPU to be idle as well. + */ + accumulated_work_time_gpu += (time_now - work_start_time_gpu); + work_start_time_gpu = 0; + } + } + + mali_utilization_data_unlock(); +} + +mali_bool mali_utilization_enabled(void) +{ +#if defined(CONFIG_MALI_DVFS) + return mali_dvfs_policy_enabled(); +#else + return (NULL != mali_utilization_callback); +#endif /* defined(CONFIG_MALI_DVFS) */ +} + +void mali_utilization_platform_realize(struct mali_gpu_utilization_data *util_data) +{ + MALI_DEBUG_ASSERT_POINTER(mali_utilization_callback); + + mali_utilization_callback(util_data); +} + +void mali_utilization_reset(void) +{ + accumulated_work_time_gpu = 0; + accumulated_work_time_gp = 0; + accumulated_work_time_pp = 0; + + last_utilization_gpu = 0; + last_utilization_gp = 0; + last_utilization_pp = 0; +} + +void mali_utilization_data_lock(void) +{ + _mali_osk_spinlock_irq_lock(utilization_data_lock); +} + +void mali_utilization_data_unlock(void) +{ + _mali_osk_spinlock_irq_unlock(utilization_data_lock); +} + +void mali_utilization_data_assert_locked(void) +{ + MALI_DEBUG_ASSERT_LOCK_HELD(utilization_data_lock); +} + +u32 _mali_ukk_utilization_gp_pp(void) +{ + return last_utilization_gpu; +} + +u32 _mali_ukk_utilization_gp(void) +{ + return last_utilization_gp; +} + +u32 _mali_ukk_utilization_pp(void) +{ + return last_utilization_pp; +} diff --git a/drivers/gpu/arm/utgard/common/mali_kernel_utilization.h b/drivers/gpu/arm/utgard/common/mali_kernel_utilization.h new file mode 100644 index 00000000000000..3c20b1983762ae --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_kernel_utilization.h @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_KERNEL_UTILIZATION_H__ +#define __MALI_KERNEL_UTILIZATION_H__ + +#include +#include "mali_osk.h" + +/** + * Initialize/start the Mali GPU utilization metrics reporting. + * + * @return _MALI_OSK_ERR_OK on success, otherwise failure. + */ +_mali_osk_errcode_t mali_utilization_init(void); + +/** + * Terminate the Mali GPU utilization metrics reporting + */ +void mali_utilization_term(void); + +/** + * Check if Mali utilization is enabled + */ +mali_bool mali_utilization_enabled(void); + +/** + * Should be called when a job is about to execute a GP job + */ +void mali_utilization_gp_start(void); + +/** + * Should be called when a job has completed executing a GP job + */ +void mali_utilization_gp_end(void); + +/** + * Should be called when a job is about to execute a PP job + */ +void mali_utilization_pp_start(void); + +/** + * Should be called when a job has completed executing a PP job + */ +void mali_utilization_pp_end(void); + +/** + * Should be called to calcution the GPU utilization + */ +struct mali_gpu_utilization_data *mali_utilization_calculate(u64 *start_time, u64 *time_period, mali_bool *need_add_timer); + +_mali_osk_spinlock_irq_t *mali_utilization_get_lock(void); + +void mali_utilization_platform_realize(struct mali_gpu_utilization_data *util_data); + +void mali_utilization_data_lock(void); + +void mali_utilization_data_unlock(void); + +void mali_utilization_data_assert_locked(void); + +void mali_utilization_reset(void); + + +#endif /* __MALI_KERNEL_UTILIZATION_H__ */ diff --git a/drivers/gpu/arm/utgard/common/mali_kernel_vsync.c b/drivers/gpu/arm/utgard/common/mali_kernel_vsync.c new file mode 100644 index 00000000000000..2eed4c88cf5616 --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_kernel_vsync.c @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_kernel_common.h" +#include "mali_osk.h" +#include "mali_ukk.h" + +#include "mali_osk_profiling.h" + +_mali_osk_errcode_t _mali_ukk_vsync_event_report(_mali_uk_vsync_event_report_s *args) +{ + _mali_uk_vsync_event event = (_mali_uk_vsync_event)args->event; + MALI_IGNORE(event); /* event is not used for release code, and that is OK */ + + /* + * Manually generate user space events in kernel space. + * This saves user space from calling kernel space twice in this case. + * We just need to remember to add pid and tid manually. + */ + if (event == _MALI_UK_VSYNC_EVENT_BEGIN_WAIT) { + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SUSPEND | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_VSYNC, + _mali_osk_get_pid(), _mali_osk_get_tid(), 0, 0, 0); + } + + if (event == _MALI_UK_VSYNC_EVENT_END_WAIT) { + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_RESUME | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_VSYNC, + _mali_osk_get_pid(), _mali_osk_get_tid(), 0, 0, 0); + } + + + MALI_DEBUG_PRINT(4, ("Received VSYNC event: %d\n", event)); + MALI_SUCCESS; +} + diff --git a/drivers/gpu/arm/utgard/common/mali_l2_cache.c b/drivers/gpu/arm/utgard/common/mali_l2_cache.c new file mode 100644 index 00000000000000..494ba789cd08c6 --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_l2_cache.c @@ -0,0 +1,534 @@ +/* + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#include "mali_kernel_common.h" +#include "mali_osk.h" +#include "mali_l2_cache.h" +#include "mali_hw_core.h" +#include "mali_scheduler.h" +#include "mali_pm.h" +#include "mali_pm_domain.h" + +/** + * Size of the Mali L2 cache registers in bytes + */ +#define MALI400_L2_CACHE_REGISTERS_SIZE 0x30 + +/** + * Mali L2 cache register numbers + * Used in the register read/write routines. + * See the hardware documentation for more information about each register + */ +typedef enum mali_l2_cache_register { + MALI400_L2_CACHE_REGISTER_SIZE = 0x0004, + MALI400_L2_CACHE_REGISTER_STATUS = 0x0008, + /*unused = 0x000C */ + MALI400_L2_CACHE_REGISTER_COMMAND = 0x0010, + MALI400_L2_CACHE_REGISTER_CLEAR_PAGE = 0x0014, + MALI400_L2_CACHE_REGISTER_MAX_READS = 0x0018, + MALI400_L2_CACHE_REGISTER_ENABLE = 0x001C, + MALI400_L2_CACHE_REGISTER_PERFCNT_SRC0 = 0x0020, + MALI400_L2_CACHE_REGISTER_PERFCNT_VAL0 = 0x0024, + MALI400_L2_CACHE_REGISTER_PERFCNT_SRC1 = 0x0028, + MALI400_L2_CACHE_REGISTER_PERFCNT_VAL1 = 0x002C, +} mali_l2_cache_register; + +/** + * Mali L2 cache commands + * These are the commands that can be sent to the Mali L2 cache unit + */ +typedef enum mali_l2_cache_command { + MALI400_L2_CACHE_COMMAND_CLEAR_ALL = 0x01, +} mali_l2_cache_command; + +/** + * Mali L2 cache commands + * These are the commands that can be sent to the Mali L2 cache unit + */ +typedef enum mali_l2_cache_enable { + MALI400_L2_CACHE_ENABLE_DEFAULT = 0x0, /* Default */ + MALI400_L2_CACHE_ENABLE_ACCESS = 0x01, + MALI400_L2_CACHE_ENABLE_READ_ALLOCATE = 0x02, +} mali_l2_cache_enable; + +/** + * Mali L2 cache status bits + */ +typedef enum mali_l2_cache_status { + MALI400_L2_CACHE_STATUS_COMMAND_BUSY = 0x01, + MALI400_L2_CACHE_STATUS_DATA_BUSY = 0x02, +} mali_l2_cache_status; + +#define MALI400_L2_MAX_READS_NOT_SET -1 + +static struct mali_l2_cache_core * + mali_global_l2s[MALI_MAX_NUMBER_OF_L2_CACHE_CORES] = { NULL, }; +static u32 mali_global_num_l2s = 0; + +int mali_l2_max_reads = MALI400_L2_MAX_READS_NOT_SET; + + +/* Local helper functions */ + +static void mali_l2_cache_reset(struct mali_l2_cache_core *cache); + +static _mali_osk_errcode_t mali_l2_cache_send_command( + struct mali_l2_cache_core *cache, u32 reg, u32 val); + +static void mali_l2_cache_lock(struct mali_l2_cache_core *cache) +{ + MALI_DEBUG_ASSERT_POINTER(cache); + _mali_osk_spinlock_irq_lock(cache->lock); +} + +static void mali_l2_cache_unlock(struct mali_l2_cache_core *cache) +{ + MALI_DEBUG_ASSERT_POINTER(cache); + _mali_osk_spinlock_irq_unlock(cache->lock); +} + +/* Implementation of the L2 cache interface */ + +struct mali_l2_cache_core *mali_l2_cache_create( + _mali_osk_resource_t *resource, u32 domain_index) +{ + struct mali_l2_cache_core *cache = NULL; +#if defined(DEBUG) + u32 cache_size; +#endif + + MALI_DEBUG_PRINT(4, ("Mali L2 cache: Creating Mali L2 cache: %s\n", + resource->description)); + + if (mali_global_num_l2s >= MALI_MAX_NUMBER_OF_L2_CACHE_CORES) { + MALI_PRINT_ERROR(("Mali L2 cache: Too many L2 caches\n")); + return NULL; + } + + cache = _mali_osk_malloc(sizeof(struct mali_l2_cache_core)); + if (NULL == cache) { + MALI_PRINT_ERROR(("Mali L2 cache: Failed to allocate memory for L2 cache core\n")); + return NULL; + } + + cache->core_id = mali_global_num_l2s; + cache->counter_src0 = MALI_HW_CORE_NO_COUNTER; + cache->counter_src1 = MALI_HW_CORE_NO_COUNTER; + cache->counter_value0_base = 0; + cache->counter_value1_base = 0; + cache->pm_domain = NULL; + cache->power_is_on = MALI_FALSE; + cache->last_invalidated_id = 0; + + if (_MALI_OSK_ERR_OK != mali_hw_core_create(&cache->hw_core, + resource, MALI400_L2_CACHE_REGISTERS_SIZE)) { + _mali_osk_free(cache); + return NULL; + } + +#if defined(DEBUG) + cache_size = mali_hw_core_register_read(&cache->hw_core, + MALI400_L2_CACHE_REGISTER_SIZE); + MALI_DEBUG_PRINT(2, ("Mali L2 cache: Created %s: % 3uK, %u-way, % 2ubyte cache line, % 3ubit external bus\n", + resource->description, + 1 << (((cache_size >> 16) & 0xff) - 10), + 1 << ((cache_size >> 8) & 0xff), + 1 << (cache_size & 0xff), + 1 << ((cache_size >> 24) & 0xff))); +#endif + + cache->lock = _mali_osk_spinlock_irq_init(_MALI_OSK_LOCKFLAG_ORDERED, + _MALI_OSK_LOCK_ORDER_L2); + if (NULL == cache->lock) { + MALI_PRINT_ERROR(("Mali L2 cache: Failed to create counter lock for L2 cache core %s\n", + cache->hw_core.description)); + mali_hw_core_delete(&cache->hw_core); + _mali_osk_free(cache); + return NULL; + } + + /* register with correct power domain */ + cache->pm_domain = mali_pm_register_l2_cache( + domain_index, cache); + + mali_global_l2s[mali_global_num_l2s] = cache; + mali_global_num_l2s++; + + return cache; +} + +void mali_l2_cache_delete(struct mali_l2_cache_core *cache) +{ + u32 i; + for (i = 0; i < mali_global_num_l2s; i++) { + if (mali_global_l2s[i] != cache) { + continue; + } + + mali_global_l2s[i] = NULL; + mali_global_num_l2s--; + + if (i == mali_global_num_l2s) { + /* Removed last element, nothing more to do */ + break; + } + + /* + * We removed a l2 cache from the middle of the array, + * so move the last l2 cache to current position + */ + mali_global_l2s[i] = mali_global_l2s[mali_global_num_l2s]; + mali_global_l2s[mali_global_num_l2s] = NULL; + + /* All good */ + break; + } + + _mali_osk_spinlock_irq_term(cache->lock); + mali_hw_core_delete(&cache->hw_core); + _mali_osk_free(cache); +} + +void mali_l2_cache_power_up(struct mali_l2_cache_core *cache) +{ + MALI_DEBUG_ASSERT_POINTER(cache); + + mali_l2_cache_lock(cache); + + mali_l2_cache_reset(cache); + + if ((1 << MALI_DOMAIN_INDEX_DUMMY) != cache->pm_domain->pmu_mask) + MALI_DEBUG_ASSERT(MALI_FALSE == cache->power_is_on); + cache->power_is_on = MALI_TRUE; + + mali_l2_cache_unlock(cache); +} + +void mali_l2_cache_power_down(struct mali_l2_cache_core *cache) +{ + MALI_DEBUG_ASSERT_POINTER(cache); + + mali_l2_cache_lock(cache); + + MALI_DEBUG_ASSERT(MALI_TRUE == cache->power_is_on); + + /* + * The HW counters will start from zero again when we resume, + * but we should report counters as always increasing. + * Take a copy of the HW values now in order to add this to + * the values we report after being powered up. + * + * The physical power off of the L2 cache might be outside our + * own control (e.g. runtime PM). That is why we must manually + * set set the counter value to zero as well. + */ + + if (cache->counter_src0 != MALI_HW_CORE_NO_COUNTER) { + cache->counter_value0_base += mali_hw_core_register_read( + &cache->hw_core, + MALI400_L2_CACHE_REGISTER_PERFCNT_VAL0); + mali_hw_core_register_write(&cache->hw_core, + MALI400_L2_CACHE_REGISTER_PERFCNT_VAL0, 0); + } + + if (cache->counter_src1 != MALI_HW_CORE_NO_COUNTER) { + cache->counter_value1_base += mali_hw_core_register_read( + &cache->hw_core, + MALI400_L2_CACHE_REGISTER_PERFCNT_VAL1); + mali_hw_core_register_write(&cache->hw_core, + MALI400_L2_CACHE_REGISTER_PERFCNT_VAL1, 0); + } + + + cache->power_is_on = MALI_FALSE; + + mali_l2_cache_unlock(cache); +} + +void mali_l2_cache_core_set_counter_src( + struct mali_l2_cache_core *cache, u32 source_id, u32 counter) +{ + u32 reg_offset_src; + u32 reg_offset_val; + + MALI_DEBUG_ASSERT_POINTER(cache); + MALI_DEBUG_ASSERT(source_id >= 0 && source_id <= 1); + + mali_l2_cache_lock(cache); + + if (0 == source_id) { + /* start counting from 0 */ + cache->counter_value0_base = 0; + cache->counter_src0 = counter; + reg_offset_src = MALI400_L2_CACHE_REGISTER_PERFCNT_SRC0; + reg_offset_val = MALI400_L2_CACHE_REGISTER_PERFCNT_VAL0; + } else { + /* start counting from 0 */ + cache->counter_value1_base = 0; + cache->counter_src1 = counter; + reg_offset_src = MALI400_L2_CACHE_REGISTER_PERFCNT_SRC1; + reg_offset_val = MALI400_L2_CACHE_REGISTER_PERFCNT_VAL1; + } + + if (cache->power_is_on) { + u32 hw_src; + + if (MALI_HW_CORE_NO_COUNTER != counter) { + hw_src = counter; + } else { + hw_src = 0; /* disable value for HW */ + } + + /* Set counter src */ + mali_hw_core_register_write(&cache->hw_core, + reg_offset_src, hw_src); + + /* Make sure the HW starts counting from 0 again */ + mali_hw_core_register_write(&cache->hw_core, + reg_offset_val, 0); + } + + mali_l2_cache_unlock(cache); +} + +void mali_l2_cache_core_get_counter_values( + struct mali_l2_cache_core *cache, + u32 *src0, u32 *value0, u32 *src1, u32 *value1) +{ + MALI_DEBUG_ASSERT_POINTER(cache); + MALI_DEBUG_ASSERT(NULL != src0); + MALI_DEBUG_ASSERT(NULL != value0); + MALI_DEBUG_ASSERT(NULL != src1); + MALI_DEBUG_ASSERT(NULL != value1); + + mali_l2_cache_lock(cache); + + *src0 = cache->counter_src0; + *src1 = cache->counter_src1; + + if (cache->counter_src0 != MALI_HW_CORE_NO_COUNTER) { + if (MALI_TRUE == cache->power_is_on) { + *value0 = mali_hw_core_register_read(&cache->hw_core, + MALI400_L2_CACHE_REGISTER_PERFCNT_VAL0); + } else { + *value0 = 0; + } + + /* Add base offset value (in case we have been power off) */ + *value0 += cache->counter_value0_base; + } + + if (cache->counter_src1 != MALI_HW_CORE_NO_COUNTER) { + if (MALI_TRUE == cache->power_is_on) { + *value1 = mali_hw_core_register_read(&cache->hw_core, + MALI400_L2_CACHE_REGISTER_PERFCNT_VAL1); + } else { + *value1 = 0; + } + + /* Add base offset value (in case we have been power off) */ + *value1 += cache->counter_value1_base; + } + + mali_l2_cache_unlock(cache); +} + +struct mali_l2_cache_core *mali_l2_cache_core_get_glob_l2_core(u32 index) +{ + if (mali_global_num_l2s > index) { + return mali_global_l2s[index]; + } + + return NULL; +} + +u32 mali_l2_cache_core_get_glob_num_l2_cores(void) +{ + return mali_global_num_l2s; +} + +void mali_l2_cache_invalidate(struct mali_l2_cache_core *cache) +{ + MALI_DEBUG_ASSERT_POINTER(cache); + + if (NULL == cache) { + return; + } + + mali_l2_cache_lock(cache); + + cache->last_invalidated_id = mali_scheduler_get_new_cache_order(); + mali_l2_cache_send_command(cache, MALI400_L2_CACHE_REGISTER_COMMAND, + MALI400_L2_CACHE_COMMAND_CLEAR_ALL); + + mali_l2_cache_unlock(cache); +} + +void mali_l2_cache_invalidate_conditional( + struct mali_l2_cache_core *cache, u32 id) +{ + MALI_DEBUG_ASSERT_POINTER(cache); + + if (NULL == cache) { + return; + } + + /* + * If the last cache invalidation was done by a job with a higher id we + * don't have to flush. Since user space will store jobs w/ their + * corresponding memory in sequence (first job #0, then job #1, ...), + * we don't have to flush for job n-1 if job n has already invalidated + * the cache since we know for sure that job n-1's memory was already + * written when job n was started. + */ + + mali_l2_cache_lock(cache); + + if (((s32)id) > ((s32)cache->last_invalidated_id)) { + /* Set latest invalidated id to current "point in time" */ + cache->last_invalidated_id = + mali_scheduler_get_new_cache_order(); + mali_l2_cache_send_command(cache, + MALI400_L2_CACHE_REGISTER_COMMAND, + MALI400_L2_CACHE_COMMAND_CLEAR_ALL); + } + + mali_l2_cache_unlock(cache); +} + +void mali_l2_cache_invalidate_all(void) +{ + u32 i; + for (i = 0; i < mali_global_num_l2s; i++) { + struct mali_l2_cache_core *cache = mali_global_l2s[i]; + _mali_osk_errcode_t ret; + + MALI_DEBUG_ASSERT_POINTER(cache); + + mali_l2_cache_lock(cache); + + if (MALI_TRUE != cache->power_is_on) { + mali_l2_cache_unlock(cache); + continue; + } + + cache->last_invalidated_id = + mali_scheduler_get_new_cache_order(); + + ret = mali_l2_cache_send_command(cache, + MALI400_L2_CACHE_REGISTER_COMMAND, + MALI400_L2_CACHE_COMMAND_CLEAR_ALL); + if (_MALI_OSK_ERR_OK != ret) { + MALI_PRINT_ERROR(("Failed to invalidate cache\n")); + } + + mali_l2_cache_unlock(cache); + } +} + +void mali_l2_cache_invalidate_all_pages(u32 *pages, u32 num_pages) +{ + u32 i; + for (i = 0; i < mali_global_num_l2s; i++) { + struct mali_l2_cache_core *cache = mali_global_l2s[i]; + u32 j; + + MALI_DEBUG_ASSERT_POINTER(cache); + + mali_l2_cache_lock(cache); + + if (MALI_TRUE != cache->power_is_on) { + mali_l2_cache_unlock(cache); + continue; + } + + for (j = 0; j < num_pages; j++) { + _mali_osk_errcode_t ret; + + ret = mali_l2_cache_send_command(cache, + MALI400_L2_CACHE_REGISTER_CLEAR_PAGE, + pages[j]); + if (_MALI_OSK_ERR_OK != ret) { + MALI_PRINT_ERROR(("Failed to invalidate cache (page)\n")); + } + } + + mali_l2_cache_unlock(cache); + } +} + +/* -------- local helper functions below -------- */ + +static void mali_l2_cache_reset(struct mali_l2_cache_core *cache) +{ + MALI_DEBUG_ASSERT_POINTER(cache); + MALI_DEBUG_ASSERT_LOCK_HELD(cache->lock); + + /* Invalidate cache (just to keep it in a known state at startup) */ + mali_l2_cache_send_command(cache, MALI400_L2_CACHE_REGISTER_COMMAND, + MALI400_L2_CACHE_COMMAND_CLEAR_ALL); + + /* Enable cache */ + mali_hw_core_register_write(&cache->hw_core, + MALI400_L2_CACHE_REGISTER_ENABLE, + (u32)MALI400_L2_CACHE_ENABLE_ACCESS | + (u32)MALI400_L2_CACHE_ENABLE_READ_ALLOCATE); + + if (MALI400_L2_MAX_READS_NOT_SET != mali_l2_max_reads) { + mali_hw_core_register_write(&cache->hw_core, + MALI400_L2_CACHE_REGISTER_MAX_READS, + (u32)mali_l2_max_reads); + } + + /* Restart any performance counters (if enabled) */ + if (cache->counter_src0 != MALI_HW_CORE_NO_COUNTER) { + + mali_hw_core_register_write(&cache->hw_core, + MALI400_L2_CACHE_REGISTER_PERFCNT_SRC0, + cache->counter_src0); + } + + if (cache->counter_src1 != MALI_HW_CORE_NO_COUNTER) { + mali_hw_core_register_write(&cache->hw_core, + MALI400_L2_CACHE_REGISTER_PERFCNT_SRC1, + cache->counter_src1); + } +} + +static _mali_osk_errcode_t mali_l2_cache_send_command( + struct mali_l2_cache_core *cache, u32 reg, u32 val) +{ + int i = 0; + const int loop_count = 100000; + + MALI_DEBUG_ASSERT_POINTER(cache); + MALI_DEBUG_ASSERT_LOCK_HELD(cache->lock); + + /* + * First, wait for L2 cache command handler to go idle. + * (Commands received while processing another command will be ignored) + */ + for (i = 0; i < loop_count; i++) { + if (!(mali_hw_core_register_read(&cache->hw_core, + MALI400_L2_CACHE_REGISTER_STATUS) & + (u32)MALI400_L2_CACHE_STATUS_COMMAND_BUSY)) { + break; + } + } + + if (i == loop_count) { + MALI_DEBUG_PRINT(1, ("Mali L2 cache: aborting wait for command interface to go idle\n")); + return _MALI_OSK_ERR_FAULT; + } + + /* then issue the command */ + mali_hw_core_register_write(&cache->hw_core, reg, val); + + return _MALI_OSK_ERR_OK; +} diff --git a/drivers/gpu/arm/utgard/common/mali_l2_cache.h b/drivers/gpu/arm/utgard/common/mali_l2_cache.h new file mode 100644 index 00000000000000..6dc8ec22d6deae --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_l2_cache.h @@ -0,0 +1,124 @@ +/* + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_KERNEL_L2_CACHE_H__ +#define __MALI_KERNEL_L2_CACHE_H__ + +#include "mali_osk.h" +#include "mali_hw_core.h" + +#define MALI_MAX_NUMBER_OF_L2_CACHE_CORES 3 +/* Maximum 1 GP and 4 PP for an L2 cache core (Mali-400 MP4) */ +#define MALI_MAX_NUMBER_OF_GROUPS_PER_L2_CACHE 5 + +/** + * Definition of the L2 cache core struct + * Used to track a L2 cache unit in the system. + * Contains information about the mapping of the registers + */ +struct mali_l2_cache_core { + /* Common HW core functionality */ + struct mali_hw_core hw_core; + + /* Synchronize L2 cache access */ + _mali_osk_spinlock_irq_t *lock; + + /* Unique core ID */ + u32 core_id; + + /* The power domain this L2 cache belongs to */ + struct mali_pm_domain *pm_domain; + + /* MALI_TRUE if power is on for this L2 cache */ + mali_bool power_is_on; + + /* A "timestamp" to avoid unnecessary flushes */ + u32 last_invalidated_id; + + /* Performance counter 0, MALI_HW_CORE_NO_COUNTER for disabled */ + u32 counter_src0; + + /* Performance counter 1, MALI_HW_CORE_NO_COUNTER for disabled */ + u32 counter_src1; + + /* + * Performance counter 0 value base/offset + * (allows accumulative reporting even after power off) + */ + u32 counter_value0_base; + + /* + * Performance counter 0 value base/offset + * (allows accumulative reporting even after power off) + */ + u32 counter_value1_base; + + /* Used by PM domains to link L2 caches of same domain */ + _mali_osk_list_t pm_domain_list; +}; + +_mali_osk_errcode_t mali_l2_cache_initialize(void); +void mali_l2_cache_terminate(void); + +struct mali_l2_cache_core *mali_l2_cache_create( + _mali_osk_resource_t *resource, u32 domain_index); +void mali_l2_cache_delete(struct mali_l2_cache_core *cache); + +MALI_STATIC_INLINE u32 mali_l2_cache_get_id(struct mali_l2_cache_core *cache) +{ + MALI_DEBUG_ASSERT_POINTER(cache); + return cache->core_id; +} + +MALI_STATIC_INLINE struct mali_pm_domain *mali_l2_cache_get_pm_domain( + struct mali_l2_cache_core *cache) +{ + MALI_DEBUG_ASSERT_POINTER(cache); + return cache->pm_domain; +} + +void mali_l2_cache_power_up(struct mali_l2_cache_core *cache); +void mali_l2_cache_power_down(struct mali_l2_cache_core *cache); + +void mali_l2_cache_core_set_counter_src( + struct mali_l2_cache_core *cache, u32 source_id, u32 counter); + +MALI_STATIC_INLINE u32 mali_l2_cache_core_get_counter_src0( + struct mali_l2_cache_core *cache) +{ + MALI_DEBUG_ASSERT_POINTER(cache); + return cache->counter_src0; +} + +MALI_STATIC_INLINE u32 mali_l2_cache_core_get_counter_src1( + struct mali_l2_cache_core *cache) +{ + MALI_DEBUG_ASSERT_POINTER(cache); + return cache->counter_src1; +} + +void mali_l2_cache_core_get_counter_values( + struct mali_l2_cache_core *cache, + u32 *src0, u32 *value0, u32 *src1, u32 *value1); + +struct mali_l2_cache_core *mali_l2_cache_core_get_glob_l2_core(u32 index); +u32 mali_l2_cache_core_get_glob_num_l2_cores(void); + +struct mali_group *mali_l2_cache_get_group( + struct mali_l2_cache_core *cache, u32 index); + +void mali_l2_cache_invalidate(struct mali_l2_cache_core *cache); +void mali_l2_cache_invalidate_conditional( + struct mali_l2_cache_core *cache, u32 id); + +void mali_l2_cache_invalidate_all(void); +void mali_l2_cache_invalidate_all_pages(u32 *pages, u32 num_pages); + +#endif /* __MALI_KERNEL_L2_CACHE_H__ */ diff --git a/drivers/gpu/arm/utgard/common/mali_mem_validation.c b/drivers/gpu/arm/utgard/common/mali_mem_validation.c new file mode 100644 index 00000000000000..e2b5b2a7f739cb --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_mem_validation.c @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_mem_validation.h" +#include "mali_osk.h" +#include "mali_kernel_common.h" + +#define MALI_INVALID_MEM_ADDR 0xFFFFFFFF + +typedef struct { + u32 phys_base; /**< Mali physical base of the memory, page aligned */ + u32 size; /**< size in bytes of the memory, multiple of page size */ +} _mali_mem_validation_t; + +static _mali_mem_validation_t mali_mem_validator = { MALI_INVALID_MEM_ADDR, MALI_INVALID_MEM_ADDR }; + +_mali_osk_errcode_t mali_mem_validation_add_range(u32 start, u32 size) +{ + /* Check that no other MEM_VALIDATION resources exist */ + if (MALI_INVALID_MEM_ADDR != mali_mem_validator.phys_base) { + MALI_PRINT_ERROR(("Failed to add frame buffer memory; another range is already specified\n")); + return _MALI_OSK_ERR_FAULT; + } + + /* Check restrictions on page alignment */ + if ((0 != (start & (~_MALI_OSK_CPU_PAGE_MASK))) || + (0 != (size & (~_MALI_OSK_CPU_PAGE_MASK)))) { + MALI_PRINT_ERROR(("Failed to add frame buffer memory; incorrect alignment\n")); + return _MALI_OSK_ERR_FAULT; + } + + mali_mem_validator.phys_base = start; + mali_mem_validator.size = size; + MALI_DEBUG_PRINT(2, ("Memory Validator installed for Mali physical address base=0x%08X, size=0x%08X\n", + mali_mem_validator.phys_base, mali_mem_validator.size)); + + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t mali_mem_validation_check(u32 phys_addr, u32 size) +{ + if (phys_addr < (phys_addr + size)) { /* Don't allow overflow (or zero size) */ + if ((0 == (phys_addr & (~_MALI_OSK_CPU_PAGE_MASK))) && + (0 == (size & (~_MALI_OSK_CPU_PAGE_MASK)))) { + if ((phys_addr >= mali_mem_validator.phys_base) && + ((phys_addr + (size - 1)) >= mali_mem_validator.phys_base) && + (phys_addr <= (mali_mem_validator.phys_base + (mali_mem_validator.size - 1))) && + ((phys_addr + (size - 1)) <= (mali_mem_validator.phys_base + (mali_mem_validator.size - 1)))) { + MALI_DEBUG_PRINT(3, ("Accepted range 0x%08X + size 0x%08X (= 0x%08X)\n", phys_addr, size, (phys_addr + size - 1))); + return _MALI_OSK_ERR_OK; + } + } + } + + MALI_PRINT_ERROR(("MALI PHYSICAL RANGE VALIDATION ERROR: The range supplied was: phys_base=0x%08X, size=0x%08X\n", phys_addr, size)); + + return _MALI_OSK_ERR_FAULT; +} diff --git a/drivers/gpu/arm/utgard/common/mali_mem_validation.h b/drivers/gpu/arm/utgard/common/mali_mem_validation.h new file mode 100644 index 00000000000000..267720625d87ee --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_mem_validation.h @@ -0,0 +1,19 @@ +/* + * Copyright (C) 2011-2013, 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_MEM_VALIDATION_H__ +#define __MALI_MEM_VALIDATION_H__ + +#include "mali_osk.h" + +_mali_osk_errcode_t mali_mem_validation_add_range(u32 start, u32 size); +_mali_osk_errcode_t mali_mem_validation_check(u32 phys_addr, u32 size); + +#endif /* __MALI_MEM_VALIDATION_H__ */ diff --git a/drivers/gpu/arm/utgard/common/mali_mmu.c b/drivers/gpu/arm/utgard/common/mali_mmu.c new file mode 100644 index 00000000000000..b975c1468d6783 --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_mmu.c @@ -0,0 +1,433 @@ +/* + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_kernel_common.h" +#include "mali_osk.h" +#include "mali_osk_list.h" +#include "mali_ukk.h" + +#include "mali_mmu.h" +#include "mali_hw_core.h" +#include "mali_group.h" +#include "mali_mmu_page_directory.h" + +/** + * Size of the MMU registers in bytes + */ +#define MALI_MMU_REGISTERS_SIZE 0x24 + +/** + * MMU commands + * These are the commands that can be sent + * to the MMU unit. + */ +typedef enum mali_mmu_command { + MALI_MMU_COMMAND_ENABLE_PAGING = 0x00, /**< Enable paging (memory translation) */ + MALI_MMU_COMMAND_DISABLE_PAGING = 0x01, /**< Disable paging (memory translation) */ + MALI_MMU_COMMAND_ENABLE_STALL = 0x02, /**< Enable stall on page fault */ + MALI_MMU_COMMAND_DISABLE_STALL = 0x03, /**< Disable stall on page fault */ + MALI_MMU_COMMAND_ZAP_CACHE = 0x04, /**< Zap the entire page table cache */ + MALI_MMU_COMMAND_PAGE_FAULT_DONE = 0x05, /**< Page fault processed */ + MALI_MMU_COMMAND_HARD_RESET = 0x06 /**< Reset the MMU back to power-on settings */ +} mali_mmu_command; + +static void mali_mmu_probe_trigger(void *data); +static _mali_osk_errcode_t mali_mmu_probe_ack(void *data); + +MALI_STATIC_INLINE _mali_osk_errcode_t mali_mmu_raw_reset(struct mali_mmu_core *mmu); + +/* page fault queue flush helper pages + * note that the mapping pointers are currently unused outside of the initialization functions */ +static mali_dma_addr mali_page_fault_flush_page_directory = MALI_INVALID_PAGE; +static mali_io_address mali_page_fault_flush_page_directory_mapping = NULL; +static mali_dma_addr mali_page_fault_flush_page_table = MALI_INVALID_PAGE; +static mali_io_address mali_page_fault_flush_page_table_mapping = NULL; +static mali_dma_addr mali_page_fault_flush_data_page = MALI_INVALID_PAGE; +static mali_io_address mali_page_fault_flush_data_page_mapping = NULL; + +/* an empty page directory (no address valid) which is active on any MMU not currently marked as in use */ +static mali_dma_addr mali_empty_page_directory_phys = MALI_INVALID_PAGE; +static mali_io_address mali_empty_page_directory_virt = NULL; + + +_mali_osk_errcode_t mali_mmu_initialize(void) +{ + /* allocate the helper pages */ + mali_empty_page_directory_phys = mali_allocate_empty_page(&mali_empty_page_directory_virt); + if (0 == mali_empty_page_directory_phys) { + MALI_DEBUG_PRINT_ERROR(("Mali MMU: Could not allocate empty page directory.\n")); + mali_empty_page_directory_phys = MALI_INVALID_PAGE; + return _MALI_OSK_ERR_NOMEM; + } + + if (_MALI_OSK_ERR_OK != mali_create_fault_flush_pages(&mali_page_fault_flush_page_directory, + &mali_page_fault_flush_page_directory_mapping, + &mali_page_fault_flush_page_table, + &mali_page_fault_flush_page_table_mapping, + &mali_page_fault_flush_data_page, + &mali_page_fault_flush_data_page_mapping)) { + MALI_DEBUG_PRINT_ERROR(("Mali MMU: Could not allocate fault flush pages\n")); + mali_free_empty_page(mali_empty_page_directory_phys, mali_empty_page_directory_virt); + mali_empty_page_directory_phys = MALI_INVALID_PAGE; + mali_empty_page_directory_virt = NULL; + return _MALI_OSK_ERR_NOMEM; + } + + return _MALI_OSK_ERR_OK; +} + +void mali_mmu_terminate(void) +{ + MALI_DEBUG_PRINT(3, ("Mali MMU: terminating\n")); + + /* Free global helper pages */ + mali_free_empty_page(mali_empty_page_directory_phys, mali_empty_page_directory_virt); + mali_empty_page_directory_phys = MALI_INVALID_PAGE; + mali_empty_page_directory_virt = NULL; + + /* Free the page fault flush pages */ + mali_destroy_fault_flush_pages(&mali_page_fault_flush_page_directory, + &mali_page_fault_flush_page_directory_mapping, + &mali_page_fault_flush_page_table, + &mali_page_fault_flush_page_table_mapping, + &mali_page_fault_flush_data_page, + &mali_page_fault_flush_data_page_mapping); +} + +struct mali_mmu_core *mali_mmu_create(_mali_osk_resource_t *resource, struct mali_group *group, mali_bool is_virtual) +{ + struct mali_mmu_core *mmu = NULL; + + MALI_DEBUG_ASSERT_POINTER(resource); + + MALI_DEBUG_PRINT(2, ("Mali MMU: Creating Mali MMU: %s\n", resource->description)); + + mmu = _mali_osk_calloc(1, sizeof(struct mali_mmu_core)); + if (NULL != mmu) { + if (_MALI_OSK_ERR_OK == mali_hw_core_create(&mmu->hw_core, resource, MALI_MMU_REGISTERS_SIZE)) { + if (_MALI_OSK_ERR_OK == mali_group_add_mmu_core(group, mmu)) { + if (is_virtual) { + /* Skip reset and IRQ setup for virtual MMU */ + return mmu; + } + + if (_MALI_OSK_ERR_OK == mali_mmu_reset(mmu)) { + /* Setup IRQ handlers (which will do IRQ probing if needed) */ + mmu->irq = _mali_osk_irq_init(resource->irq, + mali_group_upper_half_mmu, + group, + mali_mmu_probe_trigger, + mali_mmu_probe_ack, + mmu, + resource->description); + if (NULL != mmu->irq) { + return mmu; + } else { + MALI_PRINT_ERROR(("Mali MMU: Failed to setup interrupt handlers for MMU %s\n", mmu->hw_core.description)); + } + } + mali_group_remove_mmu_core(group); + } else { + MALI_PRINT_ERROR(("Mali MMU: Failed to add core %s to group\n", mmu->hw_core.description)); + } + mali_hw_core_delete(&mmu->hw_core); + } + + _mali_osk_free(mmu); + } else { + MALI_PRINT_ERROR(("Failed to allocate memory for MMU\n")); + } + + return NULL; +} + +void mali_mmu_delete(struct mali_mmu_core *mmu) +{ + if (NULL != mmu->irq) { + _mali_osk_irq_term(mmu->irq); + } + + mali_hw_core_delete(&mmu->hw_core); + _mali_osk_free(mmu); +} + +static void mali_mmu_enable_paging(struct mali_mmu_core *mmu) +{ + int i; + + mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_ENABLE_PAGING); + + for (i = 0; i < MALI_REG_POLL_COUNT_FAST; ++i) { + if (mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS) & MALI_MMU_STATUS_BIT_PAGING_ENABLED) { + break; + } + } + if (MALI_REG_POLL_COUNT_FAST == i) { + MALI_PRINT_ERROR(("Enable paging request failed, MMU status is 0x%08X\n", mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS))); + } +} + +/** + * Issues the enable stall command to the MMU and waits for HW to complete the request + * @param mmu The MMU to enable paging for + * @return MALI_TRUE if HW stall was successfully engaged, otherwise MALI_FALSE (req timed out) + */ +static mali_bool mali_mmu_enable_stall(struct mali_mmu_core *mmu) +{ + int i; + u32 mmu_status = mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS); + + if (0 == (mmu_status & MALI_MMU_STATUS_BIT_PAGING_ENABLED)) { + MALI_DEBUG_PRINT(4, ("MMU stall is implicit when Paging is not enabled.\n")); + return MALI_TRUE; + } + + if (mmu_status & MALI_MMU_STATUS_BIT_PAGE_FAULT_ACTIVE) { + MALI_DEBUG_PRINT(3, ("Aborting MMU stall request since it is in pagefault state.\n")); + return MALI_FALSE; + } + + mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_ENABLE_STALL); + + for (i = 0; i < MALI_REG_POLL_COUNT_FAST; ++i) { + mmu_status = mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS); + if (mmu_status & MALI_MMU_STATUS_BIT_PAGE_FAULT_ACTIVE) { + break; + } + if ((mmu_status & MALI_MMU_STATUS_BIT_STALL_ACTIVE) && (0 == (mmu_status & MALI_MMU_STATUS_BIT_STALL_NOT_ACTIVE))) { + break; + } + if (0 == (mmu_status & (MALI_MMU_STATUS_BIT_PAGING_ENABLED))) { + break; + } + } + if (MALI_REG_POLL_COUNT_FAST == i) { + MALI_DEBUG_PRINT(2, ("Enable stall request failed, MMU status is 0x%08X\n", mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS))); + return MALI_FALSE; + } + + if (mmu_status & MALI_MMU_STATUS_BIT_PAGE_FAULT_ACTIVE) { + MALI_DEBUG_PRINT(2, ("Aborting MMU stall request since it has a pagefault.\n")); + return MALI_FALSE; + } + + return MALI_TRUE; +} + +/** + * Issues the disable stall command to the MMU and waits for HW to complete the request + * @param mmu The MMU to enable paging for + */ +static void mali_mmu_disable_stall(struct mali_mmu_core *mmu) +{ + int i; + u32 mmu_status = mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS); + + if (0 == (mmu_status & MALI_MMU_STATUS_BIT_PAGING_ENABLED)) { + MALI_DEBUG_PRINT(3, ("MMU disable skipped since it was not enabled.\n")); + return; + } + if (mmu_status & MALI_MMU_STATUS_BIT_PAGE_FAULT_ACTIVE) { + MALI_DEBUG_PRINT(2, ("Aborting MMU disable stall request since it is in pagefault state.\n")); + return; + } + + mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_DISABLE_STALL); + + for (i = 0; i < MALI_REG_POLL_COUNT_FAST; ++i) { + u32 status = mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS); + if (0 == (status & MALI_MMU_STATUS_BIT_STALL_ACTIVE)) { + break; + } + if (status & MALI_MMU_STATUS_BIT_PAGE_FAULT_ACTIVE) { + break; + } + if (0 == (mmu_status & MALI_MMU_STATUS_BIT_PAGING_ENABLED)) { + break; + } + } + if (MALI_REG_POLL_COUNT_FAST == i) MALI_DEBUG_PRINT(1, ("Disable stall request failed, MMU status is 0x%08X\n", mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS))); +} + +void mali_mmu_page_fault_done(struct mali_mmu_core *mmu) +{ + MALI_DEBUG_PRINT(4, ("Mali MMU: %s: Leaving page fault mode\n", mmu->hw_core.description)); + mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_PAGE_FAULT_DONE); +} + +MALI_STATIC_INLINE _mali_osk_errcode_t mali_mmu_raw_reset(struct mali_mmu_core *mmu) +{ + int i; + + mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_DTE_ADDR, 0xCAFEBABE); + MALI_DEBUG_ASSERT(0xCAFEB000 == mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_DTE_ADDR)); + mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_HARD_RESET); + + for (i = 0; i < MALI_REG_POLL_COUNT_FAST; ++i) { + if (mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_DTE_ADDR) == 0) { + break; + } + } + if (MALI_REG_POLL_COUNT_FAST == i) { + MALI_PRINT_ERROR(("Reset request failed, MMU status is 0x%08X\n", mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS))); + return _MALI_OSK_ERR_FAULT; + } + + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t mali_mmu_reset(struct mali_mmu_core *mmu) +{ + _mali_osk_errcode_t err = _MALI_OSK_ERR_FAULT; + mali_bool stall_success; + MALI_DEBUG_ASSERT_POINTER(mmu); + + stall_success = mali_mmu_enable_stall(mmu); + if (!stall_success) { + err = _MALI_OSK_ERR_BUSY; + } + + MALI_DEBUG_PRINT(3, ("Mali MMU: mali_kernel_mmu_reset: %s\n", mmu->hw_core.description)); + + if (_MALI_OSK_ERR_OK == mali_mmu_raw_reset(mmu)) { + mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_INT_MASK, MALI_MMU_INTERRUPT_PAGE_FAULT | MALI_MMU_INTERRUPT_READ_BUS_ERROR); + /* no session is active, so just activate the empty page directory */ + mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_DTE_ADDR, mali_empty_page_directory_phys); + mali_mmu_enable_paging(mmu); + err = _MALI_OSK_ERR_OK; + } + mali_mmu_disable_stall(mmu); + + return err; +} + +mali_bool mali_mmu_zap_tlb(struct mali_mmu_core *mmu) +{ + mali_bool stall_success = mali_mmu_enable_stall(mmu); + + mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_ZAP_CACHE); + + if (MALI_FALSE == stall_success) { + /* False means that it is in Pagefault state. Not possible to disable_stall then */ + return MALI_FALSE; + } + + mali_mmu_disable_stall(mmu); + return MALI_TRUE; +} + +void mali_mmu_zap_tlb_without_stall(struct mali_mmu_core *mmu) +{ + mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_ZAP_CACHE); +} + + +void mali_mmu_invalidate_page(struct mali_mmu_core *mmu, u32 mali_address) +{ + mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_ZAP_ONE_LINE, MALI_MMU_PDE_ENTRY(mali_address)); +} + +static void mali_mmu_activate_address_space(struct mali_mmu_core *mmu, u32 page_directory) +{ + /* The MMU must be in stalled or page fault mode, for this writing to work */ + MALI_DEBUG_ASSERT(0 != (mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS) + & (MALI_MMU_STATUS_BIT_STALL_ACTIVE | MALI_MMU_STATUS_BIT_PAGE_FAULT_ACTIVE))); + mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_DTE_ADDR, page_directory); + mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_ZAP_CACHE); + +} + +void mali_mmu_activate_page_directory(struct mali_mmu_core *mmu, struct mali_page_directory *pagedir) +{ + mali_bool stall_success; + MALI_DEBUG_ASSERT_POINTER(mmu); + + MALI_DEBUG_PRINT(5, ("Asked to activate page directory 0x%x on MMU %s\n", pagedir, mmu->hw_core.description)); + + stall_success = mali_mmu_enable_stall(mmu); + MALI_DEBUG_ASSERT(stall_success); + MALI_IGNORE(stall_success); + mali_mmu_activate_address_space(mmu, pagedir->page_directory); + mali_mmu_disable_stall(mmu); +} + +void mali_mmu_activate_empty_page_directory(struct mali_mmu_core *mmu) +{ + mali_bool stall_success; + + MALI_DEBUG_ASSERT_POINTER(mmu); + MALI_DEBUG_PRINT(3, ("Activating the empty page directory on MMU %s\n", mmu->hw_core.description)); + + stall_success = mali_mmu_enable_stall(mmu); + + /* This function can only be called when the core is idle, so it could not fail. */ + MALI_DEBUG_ASSERT(stall_success); + MALI_IGNORE(stall_success); + + mali_mmu_activate_address_space(mmu, mali_empty_page_directory_phys); + mali_mmu_disable_stall(mmu); +} + +void mali_mmu_activate_fault_flush_page_directory(struct mali_mmu_core *mmu) +{ + mali_bool stall_success; + MALI_DEBUG_ASSERT_POINTER(mmu); + + MALI_DEBUG_PRINT(3, ("Activating the page fault flush page directory on MMU %s\n", mmu->hw_core.description)); + stall_success = mali_mmu_enable_stall(mmu); + /* This function is expect to fail the stalling, since it might be in PageFault mode when it is called */ + mali_mmu_activate_address_space(mmu, mali_page_fault_flush_page_directory); + if (MALI_TRUE == stall_success) mali_mmu_disable_stall(mmu); +} + +/* Is called when we want the mmu to give an interrupt */ +static void mali_mmu_probe_trigger(void *data) +{ + struct mali_mmu_core *mmu = (struct mali_mmu_core *)data; + mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_INT_RAWSTAT, MALI_MMU_INTERRUPT_PAGE_FAULT | MALI_MMU_INTERRUPT_READ_BUS_ERROR); +} + +/* Is called when the irq probe wants the mmu to acknowledge an interrupt from the hw */ +static _mali_osk_errcode_t mali_mmu_probe_ack(void *data) +{ + struct mali_mmu_core *mmu = (struct mali_mmu_core *)data; + u32 int_stat; + + int_stat = mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_INT_STATUS); + + MALI_DEBUG_PRINT(2, ("mali_mmu_probe_irq_acknowledge: intstat 0x%x\n", int_stat)); + if (int_stat & MALI_MMU_INTERRUPT_PAGE_FAULT) { + MALI_DEBUG_PRINT(2, ("Probe: Page fault detect: PASSED\n")); + mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_INT_CLEAR, MALI_MMU_INTERRUPT_PAGE_FAULT); + } else { + MALI_DEBUG_PRINT(1, ("Probe: Page fault detect: FAILED\n")); + } + + if (int_stat & MALI_MMU_INTERRUPT_READ_BUS_ERROR) { + MALI_DEBUG_PRINT(2, ("Probe: Bus read error detect: PASSED\n")); + mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_INT_CLEAR, MALI_MMU_INTERRUPT_READ_BUS_ERROR); + } else { + MALI_DEBUG_PRINT(1, ("Probe: Bus read error detect: FAILED\n")); + } + + if ((int_stat & (MALI_MMU_INTERRUPT_PAGE_FAULT | MALI_MMU_INTERRUPT_READ_BUS_ERROR)) == + (MALI_MMU_INTERRUPT_PAGE_FAULT | MALI_MMU_INTERRUPT_READ_BUS_ERROR)) { + return _MALI_OSK_ERR_OK; + } + + return _MALI_OSK_ERR_FAULT; +} + +#if 0 +void mali_mmu_print_state(struct mali_mmu_core *mmu) +{ + MALI_DEBUG_PRINT(2, ("MMU: State of %s is 0x%08x\n", mmu->hw_core.description, mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS))); +} +#endif diff --git a/drivers/gpu/arm/utgard/common/mali_mmu.h b/drivers/gpu/arm/utgard/common/mali_mmu.h new file mode 100644 index 00000000000000..101c968bd45d33 --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_mmu.h @@ -0,0 +1,123 @@ +/* + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_MMU_H__ +#define __MALI_MMU_H__ + +#include "mali_osk.h" +#include "mali_mmu_page_directory.h" +#include "mali_hw_core.h" + +/* Forward declaration from mali_group.h */ +struct mali_group; + +/** + * MMU register numbers + * Used in the register read/write routines. + * See the hardware documentation for more information about each register + */ +typedef enum mali_mmu_register { + MALI_MMU_REGISTER_DTE_ADDR = 0x0000, /**< Current Page Directory Pointer */ + MALI_MMU_REGISTER_STATUS = 0x0004, /**< Status of the MMU */ + MALI_MMU_REGISTER_COMMAND = 0x0008, /**< Command register, used to control the MMU */ + MALI_MMU_REGISTER_PAGE_FAULT_ADDR = 0x000C, /**< Logical address of the last page fault */ + MALI_MMU_REGISTER_ZAP_ONE_LINE = 0x010, /**< Used to invalidate the mapping of a single page from the MMU */ + MALI_MMU_REGISTER_INT_RAWSTAT = 0x0014, /**< Raw interrupt status, all interrupts visible */ + MALI_MMU_REGISTER_INT_CLEAR = 0x0018, /**< Indicate to the MMU that the interrupt has been received */ + MALI_MMU_REGISTER_INT_MASK = 0x001C, /**< Enable/disable types of interrupts */ + MALI_MMU_REGISTER_INT_STATUS = 0x0020 /**< Interrupt status based on the mask */ +} mali_mmu_register; + +/** + * MMU interrupt register bits + * Each cause of the interrupt is reported + * through the (raw) interrupt status registers. + * Multiple interrupts can be pending, so multiple bits + * can be set at once. + */ +typedef enum mali_mmu_interrupt { + MALI_MMU_INTERRUPT_PAGE_FAULT = 0x01, /**< A page fault occured */ + MALI_MMU_INTERRUPT_READ_BUS_ERROR = 0x02 /**< A bus read error occured */ +} mali_mmu_interrupt; + +typedef enum mali_mmu_status_bits { + MALI_MMU_STATUS_BIT_PAGING_ENABLED = 1 << 0, + MALI_MMU_STATUS_BIT_PAGE_FAULT_ACTIVE = 1 << 1, + MALI_MMU_STATUS_BIT_STALL_ACTIVE = 1 << 2, + MALI_MMU_STATUS_BIT_IDLE = 1 << 3, + MALI_MMU_STATUS_BIT_REPLAY_BUFFER_EMPTY = 1 << 4, + MALI_MMU_STATUS_BIT_PAGE_FAULT_IS_WRITE = 1 << 5, + MALI_MMU_STATUS_BIT_STALL_NOT_ACTIVE = 1 << 31, +} mali_mmu_status_bits; + +/** + * Definition of the MMU struct + * Used to track a MMU unit in the system. + * Contains information about the mapping of the registers + */ +struct mali_mmu_core { + struct mali_hw_core hw_core; /**< Common for all HW cores */ + _mali_osk_irq_t *irq; /**< IRQ handler */ +}; + +_mali_osk_errcode_t mali_mmu_initialize(void); + +void mali_mmu_terminate(void); + +struct mali_mmu_core *mali_mmu_create(_mali_osk_resource_t *resource, struct mali_group *group, mali_bool is_virtual); +void mali_mmu_delete(struct mali_mmu_core *mmu); + +_mali_osk_errcode_t mali_mmu_reset(struct mali_mmu_core *mmu); +mali_bool mali_mmu_zap_tlb(struct mali_mmu_core *mmu); +void mali_mmu_zap_tlb_without_stall(struct mali_mmu_core *mmu); +void mali_mmu_invalidate_page(struct mali_mmu_core *mmu, u32 mali_address); + +void mali_mmu_activate_page_directory(struct mali_mmu_core *mmu, struct mali_page_directory *pagedir); +void mali_mmu_activate_empty_page_directory(struct mali_mmu_core *mmu); +void mali_mmu_activate_fault_flush_page_directory(struct mali_mmu_core *mmu); + +void mali_mmu_page_fault_done(struct mali_mmu_core *mmu); + +MALI_STATIC_INLINE enum mali_interrupt_result mali_mmu_get_interrupt_result(struct mali_mmu_core *mmu) +{ + u32 rawstat_used = mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_INT_RAWSTAT); + if (0 == rawstat_used) { + return MALI_INTERRUPT_RESULT_NONE; + } + return MALI_INTERRUPT_RESULT_ERROR; +} + + +MALI_STATIC_INLINE u32 mali_mmu_get_int_status(struct mali_mmu_core *mmu) +{ + return mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_INT_STATUS); +} + +MALI_STATIC_INLINE u32 mali_mmu_get_rawstat(struct mali_mmu_core *mmu) +{ + return mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_INT_RAWSTAT); +} + +MALI_STATIC_INLINE void mali_mmu_mask_all_interrupts(struct mali_mmu_core *mmu) +{ + mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_INT_MASK, 0); +} + +MALI_STATIC_INLINE u32 mali_mmu_get_status(struct mali_mmu_core *mmu) +{ + return mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS); +} + +MALI_STATIC_INLINE u32 mali_mmu_get_page_fault_addr(struct mali_mmu_core *mmu) +{ + return mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_PAGE_FAULT_ADDR); +} + +#endif /* __MALI_MMU_H__ */ diff --git a/drivers/gpu/arm/utgard/common/mali_mmu_page_directory.c b/drivers/gpu/arm/utgard/common/mali_mmu_page_directory.c new file mode 100644 index 00000000000000..126fd77ec9c993 --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_mmu_page_directory.c @@ -0,0 +1,495 @@ +/* + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_kernel_common.h" +#include "mali_osk.h" +#include "mali_ukk.h" +#include "mali_uk_types.h" +#include "mali_mmu_page_directory.h" +#include "mali_memory.h" +#include "mali_l2_cache.h" + +static _mali_osk_errcode_t fill_page(mali_io_address mapping, u32 data); + +u32 mali_allocate_empty_page(mali_io_address *virt_addr) +{ + _mali_osk_errcode_t err; + mali_io_address mapping; + mali_dma_addr address; + + if (_MALI_OSK_ERR_OK != mali_mmu_get_table_page(&address, &mapping)) { + /* Allocation failed */ + MALI_DEBUG_PRINT(2, ("Mali MMU: Failed to get table page for empty pgdir\n")); + return 0; + } + + MALI_DEBUG_ASSERT_POINTER(mapping); + + err = fill_page(mapping, 0); + if (_MALI_OSK_ERR_OK != err) { + mali_mmu_release_table_page(address, mapping); + MALI_DEBUG_PRINT(2, ("Mali MMU: Failed to zero page\n")); + return 0; + } + + *virt_addr = mapping; + return address; +} + +void mali_free_empty_page(mali_dma_addr address, mali_io_address virt_addr) +{ + if (MALI_INVALID_PAGE != address) { + mali_mmu_release_table_page(address, virt_addr); + } +} + +_mali_osk_errcode_t mali_create_fault_flush_pages(mali_dma_addr *page_directory, + mali_io_address *page_directory_mapping, + mali_dma_addr *page_table, mali_io_address *page_table_mapping, + mali_dma_addr *data_page, mali_io_address *data_page_mapping) +{ + _mali_osk_errcode_t err; + + err = mali_mmu_get_table_page(data_page, data_page_mapping); + if (_MALI_OSK_ERR_OK == err) { + err = mali_mmu_get_table_page(page_table, page_table_mapping); + if (_MALI_OSK_ERR_OK == err) { + err = mali_mmu_get_table_page(page_directory, page_directory_mapping); + if (_MALI_OSK_ERR_OK == err) { + fill_page(*data_page_mapping, 0); + fill_page(*page_table_mapping, *data_page | MALI_MMU_FLAGS_DEFAULT); + fill_page(*page_directory_mapping, *page_table | MALI_MMU_FLAGS_PRESENT); + MALI_SUCCESS; + } + mali_mmu_release_table_page(*page_table, *page_table_mapping); + *page_table = MALI_INVALID_PAGE; + } + mali_mmu_release_table_page(*data_page, *data_page_mapping); + *data_page = MALI_INVALID_PAGE; + } + return err; +} + +void mali_destroy_fault_flush_pages( + mali_dma_addr *page_directory, mali_io_address *page_directory_mapping, + mali_dma_addr *page_table, mali_io_address *page_table_mapping, + mali_dma_addr *data_page, mali_io_address *data_page_mapping) +{ + if (MALI_INVALID_PAGE != *page_directory) { + mali_mmu_release_table_page(*page_directory, *page_directory_mapping); + *page_directory = MALI_INVALID_PAGE; + *page_directory_mapping = NULL; + } + + if (MALI_INVALID_PAGE != *page_table) { + mali_mmu_release_table_page(*page_table, *page_table_mapping); + *page_table = MALI_INVALID_PAGE; + *page_table_mapping = NULL; + } + + if (MALI_INVALID_PAGE != *data_page) { + mali_mmu_release_table_page(*data_page, *data_page_mapping); + *data_page = MALI_INVALID_PAGE; + *data_page_mapping = NULL; + } +} + +static _mali_osk_errcode_t fill_page(mali_io_address mapping, u32 data) +{ + int i; + MALI_DEBUG_ASSERT_POINTER(mapping); + + for (i = 0; i < MALI_MMU_PAGE_SIZE / 4; i++) { + _mali_osk_mem_iowrite32_relaxed(mapping, i * sizeof(u32), data); + } + _mali_osk_mem_barrier(); + MALI_SUCCESS; +} + +_mali_osk_errcode_t mali_mmu_pagedir_map(struct mali_page_directory *pagedir, u32 mali_address, u32 size) +{ + const int first_pde = MALI_MMU_PDE_ENTRY(mali_address); + const int last_pde = MALI_MMU_PDE_ENTRY(mali_address + size - 1); + _mali_osk_errcode_t err; + mali_io_address pde_mapping; + mali_dma_addr pde_phys; + int i, page_count; + u32 start_address; + if (last_pde < first_pde) + return _MALI_OSK_ERR_INVALID_ARGS; + + for (i = first_pde; i <= last_pde; i++) { + if (0 == (_mali_osk_mem_ioread32(pagedir->page_directory_mapped, + i * sizeof(u32)) & MALI_MMU_FLAGS_PRESENT)) { + /* Page table not present */ + MALI_DEBUG_ASSERT(0 == pagedir->page_entries_usage_count[i]); + MALI_DEBUG_ASSERT(NULL == pagedir->page_entries_mapped[i]); + + err = mali_mmu_get_table_page(&pde_phys, &pde_mapping); + if (_MALI_OSK_ERR_OK != err) { + MALI_PRINT_ERROR(("Failed to allocate page table page.\n")); + return err; + } + pagedir->page_entries_mapped[i] = pde_mapping; + + /* Update PDE, mark as present */ + _mali_osk_mem_iowrite32_relaxed(pagedir->page_directory_mapped, i * sizeof(u32), + pde_phys | MALI_MMU_FLAGS_PRESENT); + + MALI_DEBUG_ASSERT(0 == pagedir->page_entries_usage_count[i]); + } + + if (first_pde == last_pde) { + pagedir->page_entries_usage_count[i] += size / MALI_MMU_PAGE_SIZE; + } else if (i == first_pde) { + start_address = i * MALI_MMU_VIRTUAL_PAGE_SIZE; + page_count = (start_address + MALI_MMU_VIRTUAL_PAGE_SIZE - mali_address) / MALI_MMU_PAGE_SIZE; + pagedir->page_entries_usage_count[i] += page_count; + } else if (i == last_pde) { + start_address = i * MALI_MMU_VIRTUAL_PAGE_SIZE; + page_count = (mali_address + size - start_address) / MALI_MMU_PAGE_SIZE; + pagedir->page_entries_usage_count[i] += page_count; + } else { + pagedir->page_entries_usage_count[i] = 1024; + } + } + _mali_osk_write_mem_barrier(); + + return _MALI_OSK_ERR_OK; +} + +MALI_STATIC_INLINE void mali_mmu_zero_pte(mali_io_address page_table, u32 mali_address, u32 size) +{ + int i; + const int first_pte = MALI_MMU_PTE_ENTRY(mali_address); + const int last_pte = MALI_MMU_PTE_ENTRY(mali_address + size - 1); + + for (i = first_pte; i <= last_pte; i++) { + _mali_osk_mem_iowrite32_relaxed(page_table, i * sizeof(u32), 0); + } +} + +static u32 mali_page_directory_get_phys_address(struct mali_page_directory *pagedir, u32 index) +{ + return (_mali_osk_mem_ioread32(pagedir->page_directory_mapped, + index * sizeof(u32)) & ~MALI_MMU_FLAGS_MASK); +} + + +_mali_osk_errcode_t mali_mmu_pagedir_unmap(struct mali_page_directory *pagedir, u32 mali_address, u32 size) +{ + const int first_pde = MALI_MMU_PDE_ENTRY(mali_address); + const int last_pde = MALI_MMU_PDE_ENTRY(mali_address + size - 1); + u32 left = size; + int i; + mali_bool pd_changed = MALI_FALSE; + u32 pages_to_invalidate[3]; /* hard-coded to 3: max two pages from the PT level plus max one page from PD level */ + u32 num_pages_inv = 0; + mali_bool invalidate_all = MALI_FALSE; /* safety mechanism in case page_entries_usage_count is unreliable */ + + /* For all page directory entries in range. */ + for (i = first_pde; i <= last_pde; i++) { + u32 size_in_pde, offset; + + MALI_DEBUG_ASSERT_POINTER(pagedir->page_entries_mapped[i]); + MALI_DEBUG_ASSERT(0 != pagedir->page_entries_usage_count[i]); + + /* Offset into page table, 0 if mali_address is 4MiB aligned */ + offset = (mali_address & (MALI_MMU_VIRTUAL_PAGE_SIZE - 1)); + if (left < MALI_MMU_VIRTUAL_PAGE_SIZE - offset) { + size_in_pde = left; + } else { + size_in_pde = MALI_MMU_VIRTUAL_PAGE_SIZE - offset; + } + + pagedir->page_entries_usage_count[i] -= size_in_pde / MALI_MMU_PAGE_SIZE; + + /* If entire page table is unused, free it */ + if (0 == pagedir->page_entries_usage_count[i]) { + u32 page_phys; + void *page_virt; + MALI_DEBUG_PRINT(4, ("Releasing page table as this is the last reference\n")); + /* last reference removed, no need to zero out each PTE */ + + page_phys = MALI_MMU_ENTRY_ADDRESS(_mali_osk_mem_ioread32(pagedir->page_directory_mapped, i * sizeof(u32))); + page_virt = pagedir->page_entries_mapped[i]; + pagedir->page_entries_mapped[i] = NULL; + _mali_osk_mem_iowrite32_relaxed(pagedir->page_directory_mapped, i * sizeof(u32), 0); + + mali_mmu_release_table_page(page_phys, page_virt); + pd_changed = MALI_TRUE; + } else { + MALI_DEBUG_ASSERT(num_pages_inv < 2); + if (num_pages_inv < 2) { + pages_to_invalidate[num_pages_inv] = mali_page_directory_get_phys_address(pagedir, i); + num_pages_inv++; + } else { + invalidate_all = MALI_TRUE; + } + + /* If part of the page table is still in use, zero the relevant PTEs */ + mali_mmu_zero_pte(pagedir->page_entries_mapped[i], mali_address, size_in_pde); + } + + left -= size_in_pde; + mali_address += size_in_pde; + } + _mali_osk_write_mem_barrier(); + + /* L2 pages invalidation */ + if (MALI_TRUE == pd_changed) { + MALI_DEBUG_ASSERT(num_pages_inv < 3); + if (num_pages_inv < 3) { + pages_to_invalidate[num_pages_inv] = pagedir->page_directory; + num_pages_inv++; + } else { + invalidate_all = MALI_TRUE; + } + } + + if (invalidate_all) { + mali_l2_cache_invalidate_all(); + } else { + mali_l2_cache_invalidate_all_pages(pages_to_invalidate, num_pages_inv); + } + + MALI_SUCCESS; +} + +struct mali_page_directory *mali_mmu_pagedir_alloc(void) +{ + struct mali_page_directory *pagedir; + _mali_osk_errcode_t err; + mali_dma_addr phys; + + pagedir = _mali_osk_calloc(1, sizeof(struct mali_page_directory)); + if (NULL == pagedir) { + return NULL; + } + + err = mali_mmu_get_table_page(&phys, &pagedir->page_directory_mapped); + if (_MALI_OSK_ERR_OK != err) { + _mali_osk_free(pagedir); + return NULL; + } + + pagedir->page_directory = (u32)phys; + + /* Zero page directory */ + fill_page(pagedir->page_directory_mapped, 0); + + return pagedir; +} + +void mali_mmu_pagedir_free(struct mali_page_directory *pagedir) +{ + const int num_page_table_entries = sizeof(pagedir->page_entries_mapped) / sizeof(pagedir->page_entries_mapped[0]); + int i; + + /* Free referenced page tables and zero PDEs. */ + for (i = 0; i < num_page_table_entries; i++) { + if (pagedir->page_directory_mapped && (_mali_osk_mem_ioread32( + pagedir->page_directory_mapped, + sizeof(u32)*i) & MALI_MMU_FLAGS_PRESENT)) { + mali_dma_addr phys = _mali_osk_mem_ioread32(pagedir->page_directory_mapped, + i * sizeof(u32)) & ~MALI_MMU_FLAGS_MASK; + _mali_osk_mem_iowrite32_relaxed(pagedir->page_directory_mapped, i * sizeof(u32), 0); + mali_mmu_release_table_page(phys, pagedir->page_entries_mapped[i]); + } + } + _mali_osk_write_mem_barrier(); + + /* Free the page directory page. */ + mali_mmu_release_table_page(pagedir->page_directory, pagedir->page_directory_mapped); + + _mali_osk_free(pagedir); +} + + +void mali_mmu_pagedir_update(struct mali_page_directory *pagedir, u32 mali_address, + mali_dma_addr phys_address, u32 size, u32 permission_bits) +{ + u32 end_address = mali_address + size; + u32 mali_phys = (u32)phys_address; + + /* Map physical pages into MMU page tables */ + for (; mali_address < end_address; mali_address += MALI_MMU_PAGE_SIZE, mali_phys += MALI_MMU_PAGE_SIZE) { + MALI_DEBUG_ASSERT_POINTER(pagedir->page_entries_mapped[MALI_MMU_PDE_ENTRY(mali_address)]); + _mali_osk_mem_iowrite32_relaxed(pagedir->page_entries_mapped[MALI_MMU_PDE_ENTRY(mali_address)], + MALI_MMU_PTE_ENTRY(mali_address) * sizeof(u32), + mali_phys | permission_bits); + } +} + +void mali_mmu_pagedir_diag(struct mali_page_directory *pagedir, u32 fault_addr) +{ +#if defined(DEBUG) + u32 pde_index, pte_index; + u32 pde, pte; + + pde_index = MALI_MMU_PDE_ENTRY(fault_addr); + pte_index = MALI_MMU_PTE_ENTRY(fault_addr); + + + pde = _mali_osk_mem_ioread32(pagedir->page_directory_mapped, + pde_index * sizeof(u32)); + + + if (pde & MALI_MMU_FLAGS_PRESENT) { + u32 pte_addr = MALI_MMU_ENTRY_ADDRESS(pde); + + pte = _mali_osk_mem_ioread32(pagedir->page_entries_mapped[pde_index], + pte_index * sizeof(u32)); + + MALI_DEBUG_PRINT(2, ("\tMMU: %08x: Page table present: %08x\n" + "\t\tPTE: %08x, page %08x is %s\n", + fault_addr, pte_addr, pte, + MALI_MMU_ENTRY_ADDRESS(pte), + pte & MALI_MMU_FLAGS_DEFAULT ? "rw" : "not present")); + } else { + MALI_DEBUG_PRINT(2, ("\tMMU: %08x: Page table not present: %08x\n", + fault_addr, pde)); + } +#else + MALI_IGNORE(pagedir); + MALI_IGNORE(fault_addr); +#endif +} + +/* For instrumented */ +struct dump_info { + u32 buffer_left; + u32 register_writes_size; + u32 page_table_dump_size; + u32 *buffer; +}; + +static _mali_osk_errcode_t writereg(u32 where, u32 what, const char *comment, struct dump_info *info) +{ + if (NULL != info) { + info->register_writes_size += sizeof(u32) * 2; /* two 32-bit words */ + + if (NULL != info->buffer) { + /* check that we have enough space */ + if (info->buffer_left < sizeof(u32) * 2) MALI_ERROR(_MALI_OSK_ERR_NOMEM); + + *info->buffer = where; + info->buffer++; + + *info->buffer = what; + info->buffer++; + + info->buffer_left -= sizeof(u32) * 2; + } + } + + MALI_SUCCESS; +} + +static _mali_osk_errcode_t mali_mmu_dump_page(mali_io_address page, u32 phys_addr, struct dump_info *info) +{ + if (NULL != info) { + /* 4096 for the page and 4 bytes for the address */ + const u32 page_size_in_elements = MALI_MMU_PAGE_SIZE / 4; + const u32 page_size_in_bytes = MALI_MMU_PAGE_SIZE; + const u32 dump_size_in_bytes = MALI_MMU_PAGE_SIZE + 4; + + info->page_table_dump_size += dump_size_in_bytes; + + if (NULL != info->buffer) { + if (info->buffer_left < dump_size_in_bytes) MALI_ERROR(_MALI_OSK_ERR_NOMEM); + + *info->buffer = phys_addr; + info->buffer++; + + _mali_osk_memcpy(info->buffer, page, page_size_in_bytes); + info->buffer += page_size_in_elements; + + info->buffer_left -= dump_size_in_bytes; + } + } + + MALI_SUCCESS; +} + +static _mali_osk_errcode_t dump_mmu_page_table(struct mali_page_directory *pagedir, struct dump_info *info) +{ + MALI_DEBUG_ASSERT_POINTER(pagedir); + MALI_DEBUG_ASSERT_POINTER(info); + + if (NULL != pagedir->page_directory_mapped) { + int i; + + MALI_CHECK_NO_ERROR( + mali_mmu_dump_page(pagedir->page_directory_mapped, pagedir->page_directory, info) + ); + + for (i = 0; i < 1024; i++) { + if (NULL != pagedir->page_entries_mapped[i]) { + MALI_CHECK_NO_ERROR( + mali_mmu_dump_page(pagedir->page_entries_mapped[i], + _mali_osk_mem_ioread32(pagedir->page_directory_mapped, + i * sizeof(u32)) & ~MALI_MMU_FLAGS_MASK, info) + ); + } + } + } + + MALI_SUCCESS; +} + +static _mali_osk_errcode_t dump_mmu_registers(struct mali_page_directory *pagedir, struct dump_info *info) +{ + MALI_CHECK_NO_ERROR(writereg(0x00000000, pagedir->page_directory, + "set the page directory address", info)); + MALI_CHECK_NO_ERROR(writereg(0x00000008, 4, "zap???", info)); + MALI_CHECK_NO_ERROR(writereg(0x00000008, 0, "enable paging", info)); + MALI_SUCCESS; +} + +_mali_osk_errcode_t _mali_ukk_query_mmu_page_table_dump_size(_mali_uk_query_mmu_page_table_dump_size_s *args) +{ + struct dump_info info = { 0, 0, 0, NULL }; + struct mali_session_data *session_data; + + session_data = (struct mali_session_data *)(uintptr_t)(args->ctx); + MALI_DEBUG_ASSERT_POINTER(session_data); + MALI_DEBUG_ASSERT_POINTER(args); + + MALI_CHECK_NO_ERROR(dump_mmu_registers(session_data->page_directory, &info)); + MALI_CHECK_NO_ERROR(dump_mmu_page_table(session_data->page_directory, &info)); + args->size = info.register_writes_size + info.page_table_dump_size; + MALI_SUCCESS; +} + +_mali_osk_errcode_t _mali_ukk_dump_mmu_page_table(_mali_uk_dump_mmu_page_table_s *args) +{ + struct dump_info info = { 0, 0, 0, NULL }; + struct mali_session_data *session_data; + + MALI_DEBUG_ASSERT_POINTER(args); + + session_data = (struct mali_session_data *)(uintptr_t)(args->ctx); + MALI_DEBUG_ASSERT_POINTER(session_data); + + info.buffer_left = args->size; + info.buffer = (u32 *)(uintptr_t)args->buffer; + + args->register_writes = (uintptr_t)info.buffer; + MALI_CHECK_NO_ERROR(dump_mmu_registers(session_data->page_directory, &info)); + + args->page_table_dump = (uintptr_t)info.buffer; + MALI_CHECK_NO_ERROR(dump_mmu_page_table(session_data->page_directory, &info)); + + args->register_writes_size = info.register_writes_size; + args->page_table_dump_size = info.page_table_dump_size; + + MALI_SUCCESS; +} diff --git a/drivers/gpu/arm/utgard/common/mali_mmu_page_directory.h b/drivers/gpu/arm/utgard/common/mali_mmu_page_directory.h new file mode 100644 index 00000000000000..561fb60b9803ee --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_mmu_page_directory.h @@ -0,0 +1,110 @@ +/* + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_MMU_PAGE_DIRECTORY_H__ +#define __MALI_MMU_PAGE_DIRECTORY_H__ + +#include "mali_osk.h" + +/** + * Size of an MMU page in bytes + */ +#define MALI_MMU_PAGE_SIZE 0x1000 + +/* + * Size of the address space referenced by a page table page + */ +#define MALI_MMU_VIRTUAL_PAGE_SIZE 0x400000 /* 4 MiB */ + +/** + * Page directory index from address + * Calculates the page directory index from the given address + */ +#define MALI_MMU_PDE_ENTRY(address) (((address)>>22) & 0x03FF) + +/** + * Page table index from address + * Calculates the page table index from the given address + */ +#define MALI_MMU_PTE_ENTRY(address) (((address)>>12) & 0x03FF) + +/** + * Extract the memory address from an PDE/PTE entry + */ +#define MALI_MMU_ENTRY_ADDRESS(value) ((value) & 0xFFFFFC00) + +#define MALI_INVALID_PAGE ((u32)(~0)) + +/** + * + */ +typedef enum mali_mmu_entry_flags { + MALI_MMU_FLAGS_PRESENT = 0x01, + MALI_MMU_FLAGS_READ_PERMISSION = 0x02, + MALI_MMU_FLAGS_WRITE_PERMISSION = 0x04, + MALI_MMU_FLAGS_OVERRIDE_CACHE = 0x8, + MALI_MMU_FLAGS_WRITE_CACHEABLE = 0x10, + MALI_MMU_FLAGS_WRITE_ALLOCATE = 0x20, + MALI_MMU_FLAGS_WRITE_BUFFERABLE = 0x40, + MALI_MMU_FLAGS_READ_CACHEABLE = 0x80, + MALI_MMU_FLAGS_READ_ALLOCATE = 0x100, + MALI_MMU_FLAGS_MASK = 0x1FF, +} mali_mmu_entry_flags; + + +#define MALI_MMU_FLAGS_FORCE_GP_READ_ALLOCATE ( \ + MALI_MMU_FLAGS_PRESENT | \ + MALI_MMU_FLAGS_READ_PERMISSION | \ + MALI_MMU_FLAGS_WRITE_PERMISSION | \ + MALI_MMU_FLAGS_OVERRIDE_CACHE | \ + MALI_MMU_FLAGS_WRITE_CACHEABLE | \ + MALI_MMU_FLAGS_WRITE_BUFFERABLE | \ + MALI_MMU_FLAGS_READ_CACHEABLE | \ + MALI_MMU_FLAGS_READ_ALLOCATE ) + +#define MALI_MMU_FLAGS_DEFAULT ( \ + MALI_MMU_FLAGS_PRESENT | \ + MALI_MMU_FLAGS_READ_PERMISSION | \ + MALI_MMU_FLAGS_WRITE_PERMISSION ) + + +struct mali_page_directory { + u32 page_directory; /**< Physical address of the memory session's page directory */ + mali_io_address page_directory_mapped; /**< Pointer to the mapped version of the page directory into the kernel's address space */ + + mali_io_address page_entries_mapped[1024]; /**< Pointers to the page tables which exists in the page directory mapped into the kernel's address space */ + u32 page_entries_usage_count[1024]; /**< Tracks usage count of the page table pages, so they can be releases on the last reference */ +}; + +/* Map Mali virtual address space (i.e. ensure page tables exist for the virtual range) */ +_mali_osk_errcode_t mali_mmu_pagedir_map(struct mali_page_directory *pagedir, u32 mali_address, u32 size); +_mali_osk_errcode_t mali_mmu_pagedir_unmap(struct mali_page_directory *pagedir, u32 mali_address, u32 size); + +/* Back virtual address space with actual pages. Assumes input is contiguous and 4k aligned. */ +void mali_mmu_pagedir_update(struct mali_page_directory *pagedir, u32 mali_address, + mali_dma_addr phys_address, u32 size, u32 permission_bits); + +u32 mali_allocate_empty_page(mali_io_address *virtual); +void mali_free_empty_page(mali_dma_addr address, mali_io_address virt_addr); +_mali_osk_errcode_t mali_create_fault_flush_pages(mali_dma_addr *page_directory, + mali_io_address *page_directory_mapping, + mali_dma_addr *page_table, mali_io_address *page_table_mapping, + mali_dma_addr *data_page, mali_io_address *data_page_mapping); +void mali_destroy_fault_flush_pages( + mali_dma_addr *page_directory, mali_io_address *page_directory_mapping, + mali_dma_addr *page_table, mali_io_address *page_table_mapping, + mali_dma_addr *data_page, mali_io_address *data_page_mapping); + +struct mali_page_directory *mali_mmu_pagedir_alloc(void); +void mali_mmu_pagedir_free(struct mali_page_directory *pagedir); + +void mali_mmu_pagedir_diag(struct mali_page_directory *pagedir, u32 fault_addr); + +#endif /* __MALI_MMU_PAGE_DIRECTORY_H__ */ diff --git a/drivers/gpu/arm/utgard/common/mali_osk.h b/drivers/gpu/arm/utgard/common/mali_osk.h new file mode 100644 index 00000000000000..4c9e57cba18f26 --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_osk.h @@ -0,0 +1,1397 @@ +/* + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk.h + * Defines the OS abstraction layer for the kernel device driver (OSK) + */ + +#ifndef __MALI_OSK_H__ +#define __MALI_OSK_H__ + +#include "mali_osk_types.h" +#include "mali_osk_specific.h" /* include any per-os specifics */ +#include "mali_osk_locks.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @addtogroup uddapi Unified Device Driver (UDD) APIs + * + * @{ + */ + +/** + * @addtogroup oskapi UDD OS Abstraction for Kernel-side (OSK) APIs + * + * @{ + */ + +/** @addtogroup _mali_osk_lock OSK Mutual Exclusion Locks + * @{ */ + +#ifdef DEBUG +/** @brief Macro for asserting that the current thread holds a given lock + */ +#define MALI_DEBUG_ASSERT_LOCK_HELD(l) MALI_DEBUG_ASSERT(_mali_osk_lock_get_owner((_mali_osk_lock_debug_t *)l) == _mali_osk_get_tid()); + +/** @brief returns a lock's owner (thread id) if debugging is enabled + */ +#else +#define MALI_DEBUG_ASSERT_LOCK_HELD(l) do {} while(0) +#endif + +/** @} */ /* end group _mali_osk_lock */ + +/** @addtogroup _mali_osk_miscellaneous + * @{ */ + +/** @brief Find the containing structure of another structure + * + * This is the reverse of the operation 'offsetof'. This means that the + * following condition is satisfied: + * + * ptr == _MALI_OSK_CONTAINER_OF( &ptr->member, type, member ) + * + * When ptr is of type 'type'. + * + * Its purpose it to recover a larger structure that has wrapped a smaller one. + * + * @note no type or memory checking occurs to ensure that a wrapper structure + * does in fact exist, and that it is being recovered with respect to the + * correct member. + * + * @param ptr the pointer to the member that is contained within the larger + * structure + * @param type the type of the structure that contains the member + * @param member the name of the member in the structure that ptr points to. + * @return a pointer to a \a type object which contains \a member, as pointed + * to by \a ptr. + */ +#define _MALI_OSK_CONTAINER_OF(ptr, type, member) \ + ((type *)( ((char *)ptr) - offsetof(type,member) )) + +/** @addtogroup _mali_osk_wq + * @{ */ + +/** @brief Initialize work queues (for deferred work) + * + * @return _MALI_OSK_ERR_OK on success, otherwise failure. + */ +_mali_osk_errcode_t _mali_osk_wq_init(void); + +/** @brief Terminate work queues (for deferred work) + */ +void _mali_osk_wq_term(void); + +/** @brief Create work in the work queue + * + * Creates a work object which can be scheduled in the work queue. When + * scheduled, \a handler will be called with \a data as the argument. + * + * Refer to \ref _mali_osk_wq_schedule_work() for details on how work + * is scheduled in the queue. + * + * The returned pointer must be freed with \ref _mali_osk_wq_delete_work() + * when no longer needed. + */ +_mali_osk_wq_work_t *_mali_osk_wq_create_work(_mali_osk_wq_work_handler_t handler, void *data); + +/** @brief A high priority version of \a _mali_osk_wq_create_work() + * + * Creates a work object which can be scheduled in the high priority work queue. + * + * This is unfortunately needed to get low latency scheduling of the Mali cores. Normally we would + * schedule the next job in hw_irq or tasklet, but often we can't since we need to synchronously map + * and unmap shared memory when a job is connected to external fences (timelines). And this requires + * taking a mutex. + * + * We do signal a lot of other (low priority) work also as part of the job being finished, and if we + * don't set this Mali scheduling thread as high priority, we see that the CPU scheduler often runs + * random things instead of starting the next GPU job when the GPU is idle. So setting the gpu + * scheduler to high priority does give a visually more responsive system. + * + * Start the high priority work with: \a _mali_osk_wq_schedule_work_high_pri() + */ +_mali_osk_wq_work_t *_mali_osk_wq_create_work_high_pri(_mali_osk_wq_work_handler_t handler, void *data); + +/** @brief Delete a work object + * + * This will flush the work queue to ensure that the work handler will not + * be called after deletion. + */ +void _mali_osk_wq_delete_work(_mali_osk_wq_work_t *work); + +/** @brief Delete a work object + * + * This will NOT flush the work queue, so only call this if you are sure that the work handler will + * not be called after deletion. + */ +void _mali_osk_wq_delete_work_nonflush(_mali_osk_wq_work_t *work); + +/** @brief Cause a queued, deferred call of the work handler + * + * _mali_osk_wq_schedule_work provides a mechanism for enqueuing deferred calls + * to the work handler. After calling \ref _mali_osk_wq_schedule_work(), the + * work handler will be scheduled to run at some point in the future. + * + * Typically this is called by the IRQ upper-half to defer further processing of + * IRQ-related work to the IRQ bottom-half handler. This is necessary for work + * that cannot be done in an IRQ context by the IRQ upper-half handler. Timer + * callbacks also use this mechanism, because they are treated as though they + * operate in an IRQ context. Refer to \ref _mali_osk_timer_t for more + * information. + * + * Code that operates in a kernel-process context (with no IRQ context + * restrictions) may also enqueue deferred calls to the IRQ bottom-half. The + * advantage over direct calling is that deferred calling allows the caller and + * IRQ bottom half to hold the same mutex, with a guarantee that they will not + * deadlock just by using this mechanism. + * + * _mali_osk_wq_schedule_work() places deferred call requests on a queue, to + * allow for more than one thread to make a deferred call. Therfore, if it is + * called 'K' times, then the IRQ bottom-half will be scheduled 'K' times too. + * 'K' is a number that is implementation-specific. + * + * _mali_osk_wq_schedule_work() is guaranteed to not block on: + * - enqueuing a deferred call request. + * - the completion of the work handler. + * + * This is to prevent deadlock. For example, if _mali_osk_wq_schedule_work() + * blocked, then it would cause a deadlock when the following two conditions + * hold: + * - The work handler callback (of type _mali_osk_wq_work_handler_t) locks + * a mutex + * - And, at the same time, the caller of _mali_osk_wq_schedule_work() also + * holds the same mutex + * + * @note care must be taken to not overflow the queue that + * _mali_osk_wq_schedule_work() operates on. Code must be structured to + * ensure that the number of requests made to the queue is bounded. Otherwise, + * work will be lost. + * + * The queue that _mali_osk_wq_schedule_work implements is a FIFO of N-writer, + * 1-reader type. The writers are the callers of _mali_osk_wq_schedule_work + * (all OSK-registered IRQ upper-half handlers in the system, watchdog timers, + * callers from a Kernel-process context). The reader is a single thread that + * handles all OSK-registered work. + * + * @param work a pointer to the _mali_osk_wq_work_t object corresponding to the + * work to begin processing. + */ +void _mali_osk_wq_schedule_work(_mali_osk_wq_work_t *work); + +/** @brief Cause a queued, deferred call of the high priority work handler + * + * Function is the same as \a _mali_osk_wq_schedule_work() with the only + * difference that it runs in a high (real time) priority on the system. + * + * Should only be used as a substitue for doing the same work in interrupts. + * + * This is allowed to sleep, but the work should be small since it will block + * all other applications. +*/ +void _mali_osk_wq_schedule_work_high_pri(_mali_osk_wq_work_t *work); + +/** @brief Flush the work queue + * + * This will flush the OSK work queue, ensuring all work in the queue has + * completed before returning. + * + * Since this blocks on the completion of work in the work-queue, the + * caller of this function \b must \b not hold any mutexes that are taken by + * any registered work handler. To do so may cause a deadlock. + * + */ +void _mali_osk_wq_flush(void); + +/** @brief Create work in the delayed work queue + * + * Creates a work object which can be scheduled in the work queue. When + * scheduled, a timer will be start and the \a handler will be called with + * \a data as the argument when timer out + * + * Refer to \ref _mali_osk_wq_delayed_schedule_work() for details on how work + * is scheduled in the queue. + * + * The returned pointer must be freed with \ref _mali_osk_wq_delayed_delete_work_nonflush() + * when no longer needed. + */ +_mali_osk_wq_delayed_work_t *_mali_osk_wq_delayed_create_work(_mali_osk_wq_work_handler_t handler, void *data); + +/** @brief Delete a work object + * + * This will NOT flush the work queue, so only call this if you are sure that the work handler will + * not be called after deletion. + */ +void _mali_osk_wq_delayed_delete_work_nonflush(_mali_osk_wq_delayed_work_t *work); + +/** @brief Cancel a delayed work without waiting for it to finish + * + * Note that the \a work callback function may still be running on return from + * _mali_osk_wq_delayed_cancel_work_async(). + * + * @param work The delayed work to be cancelled + */ +void _mali_osk_wq_delayed_cancel_work_async(_mali_osk_wq_delayed_work_t *work); + +/** @brief Cancel a delayed work and wait for it to finish + * + * When this function returns, the \a work was either cancelled or it finished running. + * + * @param work The delayed work to be cancelled + */ +void _mali_osk_wq_delayed_cancel_work_sync(_mali_osk_wq_delayed_work_t *work); + +/** @brief Put \a work task in global workqueue after delay + * + * After waiting for a given time this puts a job in the kernel-global + * workqueue. + * + * If \a work was already on a queue, this function will return without doing anything + * + * @param work job to be done + * @param delay number of jiffies to wait or 0 for immediate execution + */ +void _mali_osk_wq_delayed_schedule_work(_mali_osk_wq_delayed_work_t *work, u32 delay); + +/** @} */ /* end group _mali_osk_wq */ + + +/** @addtogroup _mali_osk_irq + * @{ */ + +/** @brief Initialize IRQ handling for a resource + * + * Registers an interrupt handler \a uhandler for the given IRQ number \a irqnum. + * \a data will be passed as argument to the handler when an interrupt occurs. + * + * If \a irqnum is -1, _mali_osk_irq_init will probe for the IRQ number using + * the supplied \a trigger_func and \a ack_func. These functions will also + * receive \a data as their argument. + * + * @param irqnum The IRQ number that the resource uses, as seen by the CPU. + * The value -1 has a special meaning which indicates the use of probing, and + * trigger_func and ack_func must be non-NULL. + * @param uhandler The interrupt handler, corresponding to a ISR handler for + * the resource + * @param int_data resource specific data, which will be passed to uhandler + * @param trigger_func Optional: a function to trigger the resource's irq, to + * probe for the interrupt. Use NULL if irqnum != -1. + * @param ack_func Optional: a function to acknowledge the resource's irq, to + * probe for the interrupt. Use NULL if irqnum != -1. + * @param probe_data resource-specific data, which will be passed to + * (if present) trigger_func and ack_func + * @param description textual description of the IRQ resource. + * @return on success, a pointer to a _mali_osk_irq_t object, which represents + * the IRQ handling on this resource. NULL on failure. + */ +_mali_osk_irq_t *_mali_osk_irq_init(u32 irqnum, _mali_osk_irq_uhandler_t uhandler, void *int_data, _mali_osk_irq_trigger_t trigger_func, _mali_osk_irq_ack_t ack_func, void *probe_data, const char *description); + +/** @brief Terminate IRQ handling on a resource. + * + * This will disable the interrupt from the device, and then waits for any + * currently executing IRQ handlers to complete. + * + * @note If work is deferred to an IRQ bottom-half handler through + * \ref _mali_osk_wq_schedule_work(), be sure to flush any remaining work + * with \ref _mali_osk_wq_flush() or (implicitly) with \ref _mali_osk_wq_delete_work() + * + * @param irq a pointer to the _mali_osk_irq_t object corresponding to the + * resource whose IRQ handling is to be terminated. + */ +void _mali_osk_irq_term(_mali_osk_irq_t *irq); + +/** @} */ /* end group _mali_osk_irq */ + + +/** @addtogroup _mali_osk_atomic + * @{ */ + +/** @brief Decrement an atomic counter + * + * @note It is an error to decrement the counter beyond -(1<<23) + * + * @param atom pointer to an atomic counter */ +void _mali_osk_atomic_dec(_mali_osk_atomic_t *atom); + +/** @brief Decrement an atomic counter, return new value + * + * @param atom pointer to an atomic counter + * @return The new value, after decrement */ +u32 _mali_osk_atomic_dec_return(_mali_osk_atomic_t *atom); + +/** @brief Increment an atomic counter + * + * @note It is an error to increment the counter beyond (1<<23)-1 + * + * @param atom pointer to an atomic counter */ +void _mali_osk_atomic_inc(_mali_osk_atomic_t *atom); + +/** @brief Increment an atomic counter, return new value + * + * @param atom pointer to an atomic counter */ +u32 _mali_osk_atomic_inc_return(_mali_osk_atomic_t *atom); + +/** @brief Initialize an atomic counter + * + * @note the parameter required is a u32, and so signed integers should be + * cast to u32. + * + * @param atom pointer to an atomic counter + * @param val the value to initialize the atomic counter. + */ +void _mali_osk_atomic_init(_mali_osk_atomic_t *atom, u32 val); + +/** @brief Read a value from an atomic counter + * + * This can only be safely used to determine the value of the counter when it + * is guaranteed that other threads will not be modifying the counter. This + * makes its usefulness limited. + * + * @param atom pointer to an atomic counter + */ +u32 _mali_osk_atomic_read(_mali_osk_atomic_t *atom); + +/** @brief Terminate an atomic counter + * + * @param atom pointer to an atomic counter + */ +void _mali_osk_atomic_term(_mali_osk_atomic_t *atom); + +/** @brief Assign a new val to atomic counter, and return the old atomic counter + * + * @param atom pointer to an atomic counter + * @param val the new value assign to the atomic counter + * @return the old value of the atomic counter + */ +u32 _mali_osk_atomic_xchg(_mali_osk_atomic_t *atom, u32 val); +/** @} */ /* end group _mali_osk_atomic */ + + +/** @defgroup _mali_osk_memory OSK Memory Allocation + * @{ */ + +/** @brief Allocate zero-initialized memory. + * + * Returns a buffer capable of containing at least \a n elements of \a size + * bytes each. The buffer is initialized to zero. + * + * If there is a need for a bigger block of memory (16KB or bigger), then + * consider to use _mali_osk_vmalloc() instead, as this function might + * map down to a OS function with size limitations. + * + * The buffer is suitably aligned for storage and subsequent access of every + * type that the compiler supports. Therefore, the pointer to the start of the + * buffer may be cast into any pointer type, and be subsequently accessed from + * such a pointer, without loss of information. + * + * When the buffer is no longer in use, it must be freed with _mali_osk_free(). + * Failure to do so will cause a memory leak. + * + * @note Most toolchains supply memory allocation functions that meet the + * compiler's alignment requirements. + * + * @param n Number of elements to allocate + * @param size Size of each element + * @return On success, the zero-initialized buffer allocated. NULL on failure + */ +void *_mali_osk_calloc(u32 n, u32 size); + +/** @brief Allocate memory. + * + * Returns a buffer capable of containing at least \a size bytes. The + * contents of the buffer are undefined. + * + * If there is a need for a bigger block of memory (16KB or bigger), then + * consider to use _mali_osk_vmalloc() instead, as this function might + * map down to a OS function with size limitations. + * + * The buffer is suitably aligned for storage and subsequent access of every + * type that the compiler supports. Therefore, the pointer to the start of the + * buffer may be cast into any pointer type, and be subsequently accessed from + * such a pointer, without loss of information. + * + * When the buffer is no longer in use, it must be freed with _mali_osk_free(). + * Failure to do so will cause a memory leak. + * + * @note Most toolchains supply memory allocation functions that meet the + * compiler's alignment requirements. + * + * Remember to free memory using _mali_osk_free(). + * @param size Number of bytes to allocate + * @return On success, the buffer allocated. NULL on failure. + */ +void *_mali_osk_malloc(u32 size); + +/** @brief Free memory. + * + * Reclaims the buffer pointed to by the parameter \a ptr for the system. + * All memory returned from _mali_osk_malloc() and _mali_osk_calloc() + * must be freed before the application exits. Otherwise, + * a memory leak will occur. + * + * Memory must be freed once. It is an error to free the same non-NULL pointer + * more than once. + * + * It is legal to free the NULL pointer. + * + * @param ptr Pointer to buffer to free + */ +void _mali_osk_free(void *ptr); + +/** @brief Allocate memory. + * + * Returns a buffer capable of containing at least \a size bytes. The + * contents of the buffer are undefined. + * + * This function is potentially slower than _mali_osk_malloc() and _mali_osk_calloc(), + * but do support bigger sizes. + * + * The buffer is suitably aligned for storage and subsequent access of every + * type that the compiler supports. Therefore, the pointer to the start of the + * buffer may be cast into any pointer type, and be subsequently accessed from + * such a pointer, without loss of information. + * + * When the buffer is no longer in use, it must be freed with _mali_osk_free(). + * Failure to do so will cause a memory leak. + * + * @note Most toolchains supply memory allocation functions that meet the + * compiler's alignment requirements. + * + * Remember to free memory using _mali_osk_free(). + * @param size Number of bytes to allocate + * @return On success, the buffer allocated. NULL on failure. + */ +void *_mali_osk_valloc(u32 size); + +/** @brief Free memory. + * + * Reclaims the buffer pointed to by the parameter \a ptr for the system. + * All memory returned from _mali_osk_valloc() must be freed before the + * application exits. Otherwise a memory leak will occur. + * + * Memory must be freed once. It is an error to free the same non-NULL pointer + * more than once. + * + * It is legal to free the NULL pointer. + * + * @param ptr Pointer to buffer to free + */ +void _mali_osk_vfree(void *ptr); + +/** @brief Copies memory. + * + * Copies the \a len bytes from the buffer pointed by the parameter \a src + * directly to the buffer pointed by \a dst. + * + * It is an error for \a src to overlap \a dst anywhere in \a len bytes. + * + * @param dst Pointer to the destination array where the content is to be + * copied. + * @param src Pointer to the source of data to be copied. + * @param len Number of bytes to copy. + * @return \a dst is always passed through unmodified. + */ +void *_mali_osk_memcpy(void *dst, const void *src, u32 len); + +/** @brief Fills memory. + * + * Sets the first \a n bytes of the block of memory pointed to by \a s to + * the specified value + * @param s Pointer to the block of memory to fill. + * @param c Value to be set, passed as u32. Only the 8 Least Significant Bits (LSB) + * are used. + * @param n Number of bytes to be set to the value. + * @return \a s is always passed through unmodified + */ +void *_mali_osk_memset(void *s, u32 c, u32 n); +/** @} */ /* end group _mali_osk_memory */ + + +/** @brief Checks the amount of memory allocated + * + * Checks that not more than \a max_allocated bytes are allocated. + * + * Some OS bring up an interactive out of memory dialogue when the + * system runs out of memory. This can stall non-interactive + * apps (e.g. automated test runs). This function can be used to + * not trigger the OOM dialogue by keeping allocations + * within a certain limit. + * + * @return MALI_TRUE when \a max_allocated bytes are not in use yet. MALI_FALSE + * when at least \a max_allocated bytes are in use. + */ +mali_bool _mali_osk_mem_check_allocated(u32 max_allocated); + + +/** @addtogroup _mali_osk_low_level_memory + * @{ */ + +/** @brief Issue a memory barrier + * + * This defines an arbitrary memory barrier operation, which forces an ordering constraint + * on memory read and write operations. + */ +void _mali_osk_mem_barrier(void); + +/** @brief Issue a write memory barrier + * + * This defines an write memory barrier operation which forces an ordering constraint + * on memory write operations. + */ +void _mali_osk_write_mem_barrier(void); + +/** @brief Map a physically contiguous region into kernel space + * + * This is primarily used for mapping in registers from resources, and Mali-MMU + * page tables. The mapping is only visable from kernel-space. + * + * Access has to go through _mali_osk_mem_ioread32 and _mali_osk_mem_iowrite32 + * + * @param phys CPU-physical base address of the memory to map in. This must + * be aligned to the system's page size, which is assumed to be 4K. + * @param size the number of bytes of physically contiguous address space to + * map in + * @param description A textual description of the memory being mapped in. + * @return On success, a Mali IO address through which the mapped-in + * memory/registers can be accessed. NULL on failure. + */ +mali_io_address _mali_osk_mem_mapioregion(uintptr_t phys, u32 size, const char *description); + +/** @brief Unmap a physically contiguous address range from kernel space. + * + * The address range should be one previously mapped in through + * _mali_osk_mem_mapioregion. + * + * It is a programming error to do (but not limited to) the following: + * - attempt an unmap twice + * - unmap only part of a range obtained through _mali_osk_mem_mapioregion + * - unmap more than the range obtained through _mali_osk_mem_mapioregion + * - unmap an address range that was not successfully mapped using + * _mali_osk_mem_mapioregion + * - provide a mapping that does not map to phys. + * + * @param phys CPU-physical base address of the memory that was originally + * mapped in. This must be aligned to the system's page size, which is assumed + * to be 4K + * @param size The number of bytes that were originally mapped in. + * @param mapping The Mali IO address through which the mapping is + * accessed. + */ +void _mali_osk_mem_unmapioregion(uintptr_t phys, u32 size, mali_io_address mapping); + +/** @brief Allocate and Map a physically contiguous region into kernel space + * + * This is used for allocating physically contiguous regions (such as Mali-MMU + * page tables) and mapping them into kernel space. The mapping is only + * visible from kernel-space. + * + * The alignment of the returned memory is guaranteed to be at least + * _MALI_OSK_CPU_PAGE_SIZE. + * + * Access must go through _mali_osk_mem_ioread32 and _mali_osk_mem_iowrite32 + * + * @note This function is primarily to provide support for OSs that are + * incapable of separating the tasks 'allocate physically contiguous memory' + * and 'map it into kernel space' + * + * @param[out] phys CPU-physical base address of memory that was allocated. + * (*phys) will be guaranteed to be aligned to at least + * _MALI_OSK_CPU_PAGE_SIZE on success. + * + * @param[in] size the number of bytes of physically contiguous memory to + * allocate. This must be a multiple of _MALI_OSK_CPU_PAGE_SIZE. + * + * @return On success, a Mali IO address through which the mapped-in + * memory/registers can be accessed. NULL on failure, and (*phys) is unmodified. + */ +mali_io_address _mali_osk_mem_allocioregion(u32 *phys, u32 size); + +/** @brief Free a physically contiguous address range from kernel space. + * + * The address range should be one previously mapped in through + * _mali_osk_mem_allocioregion. + * + * It is a programming error to do (but not limited to) the following: + * - attempt a free twice on the same ioregion + * - free only part of a range obtained through _mali_osk_mem_allocioregion + * - free more than the range obtained through _mali_osk_mem_allocioregion + * - free an address range that was not successfully mapped using + * _mali_osk_mem_allocioregion + * - provide a mapping that does not map to phys. + * + * @param phys CPU-physical base address of the memory that was originally + * mapped in, which was aligned to _MALI_OSK_CPU_PAGE_SIZE. + * @param size The number of bytes that were originally mapped in, which was + * a multiple of _MALI_OSK_CPU_PAGE_SIZE. + * @param mapping The Mali IO address through which the mapping is + * accessed. + */ +void _mali_osk_mem_freeioregion(u32 phys, u32 size, mali_io_address mapping); + +/** @brief Request a region of physically contiguous memory + * + * This is used to ensure exclusive access to a region of physically contigous + * memory. + * + * It is acceptable to implement this as a stub. However, it is then the job + * of the System Integrator to ensure that no other device driver will be using + * the physical address ranges used by Mali, while the Mali device driver is + * loaded. + * + * @param phys CPU-physical base address of the memory to request. This must + * be aligned to the system's page size, which is assumed to be 4K. + * @param size the number of bytes of physically contiguous address space to + * request. + * @param description A textual description of the memory being requested. + * @return _MALI_OSK_ERR_OK on success. Otherwise, a suitable + * _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_osk_mem_reqregion(uintptr_t phys, u32 size, const char *description); + +/** @brief Un-request a region of physically contiguous memory + * + * This is used to release a regious of physically contiguous memory previously + * requested through _mali_osk_mem_reqregion, so that other device drivers may + * use it. This will be called at time of Mali device driver termination. + * + * It is a programming error to attempt to: + * - unrequest a region twice + * - unrequest only part of a range obtained through _mali_osk_mem_reqregion + * - unrequest more than the range obtained through _mali_osk_mem_reqregion + * - unrequest an address range that was not successfully requested using + * _mali_osk_mem_reqregion + * + * @param phys CPU-physical base address of the memory to un-request. This must + * be aligned to the system's page size, which is assumed to be 4K + * @param size the number of bytes of physically contiguous address space to + * un-request. + */ +void _mali_osk_mem_unreqregion(uintptr_t phys, u32 size); + +/** @brief Read from a location currently mapped in through + * _mali_osk_mem_mapioregion + * + * This reads a 32-bit word from a 32-bit aligned location. It is a programming + * error to provide unaligned locations, or to read from memory that is not + * mapped in, or not mapped through either _mali_osk_mem_mapioregion() or + * _mali_osk_mem_allocioregion(). + * + * @param mapping Mali IO address to read from + * @param offset Byte offset from the given IO address to operate on, must be a multiple of 4 + * @return the 32-bit word from the specified location. + */ +u32 _mali_osk_mem_ioread32(volatile mali_io_address mapping, u32 offset); + +/** @brief Write to a location currently mapped in through + * _mali_osk_mem_mapioregion without memory barriers + * + * This write a 32-bit word to a 32-bit aligned location without using memory barrier. + * It is a programming error to provide unaligned locations, or to write to memory that is not + * mapped in, or not mapped through either _mali_osk_mem_mapioregion() or + * _mali_osk_mem_allocioregion(). + * + * @param mapping Mali IO address to write to + * @param offset Byte offset from the given IO address to operate on, must be a multiple of 4 + * @param val the 32-bit word to write. + */ +void _mali_osk_mem_iowrite32_relaxed(volatile mali_io_address addr, u32 offset, u32 val); + +/** @brief Write to a location currently mapped in through + * _mali_osk_mem_mapioregion with write memory barrier + * + * This write a 32-bit word to a 32-bit aligned location. It is a programming + * error to provide unaligned locations, or to write to memory that is not + * mapped in, or not mapped through either _mali_osk_mem_mapioregion() or + * _mali_osk_mem_allocioregion(). + * + * @param mapping Mali IO address to write to + * @param offset Byte offset from the given IO address to operate on, must be a multiple of 4 + * @param val the 32-bit word to write. + */ +void _mali_osk_mem_iowrite32(volatile mali_io_address mapping, u32 offset, u32 val); + +/** @brief Flush all CPU caches + * + * This should only be implemented if flushing of the cache is required for + * memory mapped in through _mali_osk_mem_mapregion. + */ +void _mali_osk_cache_flushall(void); + +/** @brief Flush any caches necessary for the CPU and MALI to have the same view of a range of uncached mapped memory + * + * This should only be implemented if your OS doesn't do a full cache flush (inner & outer) + * after allocating uncached mapped memory. + * + * Some OS do not perform a full cache flush (including all outer caches) for uncached mapped memory. + * They zero the memory through a cached mapping, then flush the inner caches but not the outer caches. + * This is required for MALI to have the correct view of the memory. + */ +void _mali_osk_cache_ensure_uncached_range_flushed(void *uncached_mapping, u32 offset, u32 size); + +/** @brief Safely copy as much data as possible from src to dest + * + * Do not crash if src or dest isn't available. + * + * @param dest Destination buffer (limited to user space mapped Mali memory) + * @param src Source buffer + * @param size Number of bytes to copy + * @return Number of bytes actually copied + */ +u32 _mali_osk_mem_write_safe(void *dest, const void *src, u32 size); + +/** @} */ /* end group _mali_osk_low_level_memory */ + + +/** @addtogroup _mali_osk_notification + * + * User space notification framework + * + * Communication with user space of asynchronous events is performed through a + * synchronous call to the \ref u_k_api. + * + * Since the events are asynchronous, the events have to be queued until a + * synchronous U/K API call can be made by user-space. A U/K API call might also + * be received before any event has happened. Therefore the notifications the + * different subsystems wants to send to user space has to be queued for later + * reception, or a U/K API call has to be blocked until an event has occured. + * + * Typical uses of notifications are after running of jobs on the hardware or + * when changes to the system is detected that needs to be relayed to user + * space. + * + * After an event has occured user space has to be notified using some kind of + * message. The notification framework supports sending messages to waiting + * threads or queueing of messages until a U/K API call is made. + * + * The notification queue is a FIFO. There are no restrictions on the numbers + * of readers or writers in the queue. + * + * A message contains what user space needs to identifiy how to handle an + * event. This includes a type field and a possible type specific payload. + * + * A notification to user space is represented by a + * \ref _mali_osk_notification_t object. A sender gets hold of such an object + * using _mali_osk_notification_create(). The buffer given by the + * _mali_osk_notification_t::result_buffer field in the object is used to store + * any type specific data. The other fields are internal to the queue system + * and should not be touched. + * + * @{ */ + +/** @brief Create a notification object + * + * Returns a notification object which can be added to the queue of + * notifications pending for user space transfer. + * + * The implementation will initialize all members of the + * \ref _mali_osk_notification_t object. In particular, the + * _mali_osk_notification_t::result_buffer member will be initialized to point + * to \a size bytes of storage, and that storage will be suitably aligned for + * storage of any structure. That is, the created buffer meets the same + * requirements as _mali_osk_malloc(). + * + * The notification object must be deleted when not in use. Use + * _mali_osk_notification_delete() for deleting it. + * + * @note You \b must \b not call _mali_osk_free() on a \ref _mali_osk_notification_t, + * object, or on a _mali_osk_notification_t::result_buffer. You must only use + * _mali_osk_notification_delete() to free the resources assocaited with a + * \ref _mali_osk_notification_t object. + * + * @param type The notification type + * @param size The size of the type specific buffer to send + * @return Pointer to a notification object with a suitable buffer, or NULL on error. + */ +_mali_osk_notification_t *_mali_osk_notification_create(u32 type, u32 size); + +/** @brief Delete a notification object + * + * This must be called to reclaim the resources of a notification object. This + * includes: + * - The _mali_osk_notification_t::result_buffer + * - The \ref _mali_osk_notification_t itself. + * + * A notification object \b must \b not be used after it has been deleted by + * _mali_osk_notification_delete(). + * + * In addition, the notification object may not be deleted while it is in a + * queue. That is, if it has been placed on a queue with + * _mali_osk_notification_queue_send(), then it must not be deleted until + * it has been received by a call to _mali_osk_notification_queue_receive(). + * Otherwise, the queue may be corrupted. + * + * @param object the notification object to delete. + */ +void _mali_osk_notification_delete(_mali_osk_notification_t *object); + +/** @brief Create a notification queue + * + * Creates a notification queue which can be used to queue messages for user + * delivery and get queued messages from + * + * The queue is a FIFO, and has no restrictions on the numbers of readers or + * writers. + * + * When the queue is no longer in use, it must be terminated with + * \ref _mali_osk_notification_queue_term(). Failure to do so will result in a + * memory leak. + * + * @return Pointer to a new notification queue or NULL on error. + */ +_mali_osk_notification_queue_t *_mali_osk_notification_queue_init(void); + +/** @brief Destroy a notification queue + * + * Destroys a notification queue and frees associated resources from the queue. + * + * A notification queue \b must \b not be destroyed in the following cases: + * - while there are \ref _mali_osk_notification_t objects in the queue. + * - while there are writers currently acting upon the queue. That is, while + * a thread is currently calling \ref _mali_osk_notification_queue_send() on + * the queue, or while a thread may call + * \ref _mali_osk_notification_queue_send() on the queue in the future. + * - while there are readers currently waiting upon the queue. That is, while + * a thread is currently calling \ref _mali_osk_notification_queue_receive() on + * the queue, or while a thread may call + * \ref _mali_osk_notification_queue_receive() on the queue in the future. + * + * Therefore, all \ref _mali_osk_notification_t objects must be flushed and + * deleted by the code that makes use of the notification queues, since only + * they know the structure of the _mali_osk_notification_t::result_buffer + * (even if it may only be a flat sturcture). + * + * @note Since the queue is a FIFO, the code using notification queues may + * create its own 'flush' type of notification, to assist in flushing the + * queue. + * + * Once the queue has been destroyed, it must not be used again. + * + * @param queue The queue to destroy + */ +void _mali_osk_notification_queue_term(_mali_osk_notification_queue_t *queue); + +/** @brief Schedule notification for delivery + * + * When a \ref _mali_osk_notification_t object has been created successfully + * and set up, it may be added to the queue of objects waiting for user space + * transfer. + * + * The sending will not block if the queue is full. + * + * A \ref _mali_osk_notification_t object \b must \b not be put on two different + * queues at the same time, or enqueued twice onto a single queue before + * reception. However, it is acceptable for it to be requeued \em after reception + * from a call to _mali_osk_notification_queue_receive(), even onto the same queue. + * + * Again, requeuing must also not enqueue onto two different queues at the same + * time, or enqueue onto the same queue twice before reception. + * + * @param queue The notification queue to add this notification to + * @param object The entry to add + */ +void _mali_osk_notification_queue_send(_mali_osk_notification_queue_t *queue, _mali_osk_notification_t *object); + +/** @brief Receive a notification from a queue + * + * Receives a single notification from the given queue. + * + * If no notifciations are ready the thread will sleep until one becomes ready. + * Therefore, notifications may not be received into an + * IRQ or 'atomic' context (that is, a context where sleeping is disallowed). + * + * @param queue The queue to receive from + * @param result Pointer to storage of a pointer of type + * \ref _mali_osk_notification_t*. \a result will be written to such that the + * expression \a (*result) will evaluate to a pointer to a valid + * \ref _mali_osk_notification_t object, or NULL if none were received. + * @return _MALI_OSK_ERR_OK on success. _MALI_OSK_ERR_RESTARTSYSCALL if the sleep was interrupted. + */ +_mali_osk_errcode_t _mali_osk_notification_queue_receive(_mali_osk_notification_queue_t *queue, _mali_osk_notification_t **result); + +/** @brief Dequeues a notification from a queue + * + * Receives a single notification from the given queue. + * + * If no notifciations are ready the function call will return an error code. + * + * @param queue The queue to receive from + * @param result Pointer to storage of a pointer of type + * \ref _mali_osk_notification_t*. \a result will be written to such that the + * expression \a (*result) will evaluate to a pointer to a valid + * \ref _mali_osk_notification_t object, or NULL if none were received. + * @return _MALI_OSK_ERR_OK on success, _MALI_OSK_ERR_ITEM_NOT_FOUND if queue was empty. + */ +_mali_osk_errcode_t _mali_osk_notification_queue_dequeue(_mali_osk_notification_queue_t *queue, _mali_osk_notification_t **result); + +/** @} */ /* end group _mali_osk_notification */ + + +/** @addtogroup _mali_osk_timer + * + * Timers use the OS's representation of time, which are 'ticks'. This is to + * prevent aliasing problems between the internal timer time, and the time + * asked for. + * + * @{ */ + +/** @brief Initialize a timer + * + * Allocates resources for a new timer, and initializes them. This does not + * start the timer. + * + * @return a pointer to the allocated timer object, or NULL on failure. + */ +_mali_osk_timer_t *_mali_osk_timer_init(void); + +/** @brief Start a timer + * + * It is an error to start a timer without setting the callback via + * _mali_osk_timer_setcallback(). + * + * It is an error to use this to start an already started timer. + * + * The timer will expire in \a ticks_to_expire ticks, at which point, the + * callback function will be invoked with the callback-specific data, + * as registered by _mali_osk_timer_setcallback(). + * + * @param tim the timer to start + * @param ticks_to_expire the amount of time in ticks for the timer to run + * before triggering. + */ +void _mali_osk_timer_add(_mali_osk_timer_t *tim, unsigned long ticks_to_expire); + +/** @brief Modify a timer + * + * Set the relative time at which a timer will expire, and start it if it is + * stopped. If \a ticks_to_expire 0 the timer fires immediately. + * + * It is an error to modify a timer without setting the callback via + * _mali_osk_timer_setcallback(). + * + * The timer will expire at \a ticks_to_expire from the time of the call, at + * which point, the callback function will be invoked with the + * callback-specific data, as set by _mali_osk_timer_setcallback(). + * + * @param tim the timer to modify, and start if necessary + * @param ticks_to_expire the \em absolute time in ticks at which this timer + * should trigger. + * + */ +void _mali_osk_timer_mod(_mali_osk_timer_t *tim, unsigned long ticks_to_expire); + +/** @brief Stop a timer, and block on its completion. + * + * Stop the timer. When the function returns, it is guaranteed that the timer's + * callback will not be running on any CPU core. + * + * Since stoping the timer blocks on compeletion of the callback, the callback + * may not obtain any mutexes that the caller holds. Otherwise, a deadlock will + * occur. + * + * @note While the callback itself is guaranteed to not be running, work + * enqueued on the work-queue by the timer (with + * \ref _mali_osk_wq_schedule_work()) may still run. The timer callback and + * work handler must take this into account. + * + * It is legal to stop an already stopped timer. + * + * @param tim the timer to stop. + * + */ +void _mali_osk_timer_del(_mali_osk_timer_t *tim); + +/** @brief Stop a timer. + * + * Stop the timer. When the function returns, the timer's callback may still be + * running on any CPU core. + * + * It is legal to stop an already stopped timer. + * + * @param tim the timer to stop. + */ +void _mali_osk_timer_del_async(_mali_osk_timer_t *tim); + +/** @brief Check if timer is pending. + * + * Check if timer is active. + * + * @param tim the timer to check + * @return MALI_TRUE if time is active, MALI_FALSE if it is not active + */ +mali_bool _mali_osk_timer_pending(_mali_osk_timer_t *tim); + +/** @brief Set a timer's callback parameters. + * + * This must be called at least once before a timer is started/modified. + * + * After a timer has been stopped or expires, the callback remains set. This + * means that restarting the timer will call the same function with the same + * parameters on expiry. + * + * @param tim the timer to set callback on. + * @param callback Function to call when timer expires + * @param data Function-specific data to supply to the function on expiry. + */ +void _mali_osk_timer_setcallback(_mali_osk_timer_t *tim, _mali_osk_timer_callback_t callback, void *data); + +/** @brief Terminate a timer, and deallocate resources. + * + * The timer must first be stopped by calling _mali_osk_timer_del(). + * + * It is a programming error for _mali_osk_timer_term() to be called on: + * - timer that is currently running + * - a timer that is currently executing its callback. + * + * @param tim the timer to deallocate. + */ +void _mali_osk_timer_term(_mali_osk_timer_t *tim); +/** @} */ /* end group _mali_osk_timer */ + + +/** @defgroup _mali_osk_time OSK Time functions + * + * \ref _mali_osk_time use the OS's representation of time, which are + * 'ticks'. This is to prevent aliasing problems between the internal timer + * time, and the time asked for. + * + * OS tick time is measured as a u32. The time stored in a u32 may either be + * an absolute time, or a time delta between two events. Whilst it is valid to + * use math opeartors to \em change the tick value represented as a u32, it + * is often only meaningful to do such operations on time deltas, rather than + * on absolute time. However, it is meaningful to add/subtract time deltas to + * absolute times. + * + * Conversion between tick time and milliseconds (ms) may not be loss-less, + * and are \em implementation \em depenedant. + * + * Code use OS time must take this into account, since: + * - a small OS time may (or may not) be rounded + * - a large time may (or may not) overflow + * + * @{ */ + +/** @brief Return whether ticka occurs after or at the same time as tickb + * + * Systems where ticks can wrap must handle that. + * + * @param ticka ticka + * @param tickb tickb + * @return MALI_TRUE if ticka represents a time that occurs at or after tickb. + */ +mali_bool _mali_osk_time_after_eq(unsigned long ticka, unsigned long tickb); + +/** @brief Convert milliseconds to OS 'ticks' + * + * @param ms time interval in milliseconds + * @return the corresponding time interval in OS ticks. + */ +unsigned long _mali_osk_time_mstoticks(u32 ms); + +/** @brief Convert OS 'ticks' to milliseconds + * + * @param ticks time interval in OS ticks. + * @return the corresponding time interval in milliseconds + */ +u32 _mali_osk_time_tickstoms(unsigned long ticks); + + +/** @brief Get the current time in OS 'ticks'. + * @return the current time in OS 'ticks'. + */ +unsigned long _mali_osk_time_tickcount(void); + +/** @brief Cause a microsecond delay + * + * The delay will have microsecond resolution, and is necessary for correct + * operation of the driver. At worst, the delay will be \b at least \a usecs + * microseconds, and so may be (significantly) more. + * + * This function may be implemented as a busy-wait, which is the most sensible + * implementation. On OSs where there are situations in which a thread must not + * sleep, this is definitely implemented as a busy-wait. + * + * @param usecs the number of microseconds to wait for. + */ +void _mali_osk_time_ubusydelay(u32 usecs); + +/** @brief Return time in nano seconds, since any given reference. + * + * @return Time in nano seconds + */ +u64 _mali_osk_time_get_ns(void); + +/** @brief Return time in nano seconds, since boot time. + * + * @return Time in nano seconds + */ +u64 _mali_osk_boot_time_get_ns(void); + +/** @} */ /* end group _mali_osk_time */ + +/** @defgroup _mali_osk_math OSK Math + * @{ */ + +/** @brief Count Leading Zeros (Little-endian) + * + * @note This function must be implemented to support the reference + * implementation of _mali_osk_find_first_zero_bit, as defined in + * mali_osk_bitops.h. + * + * @param val 32-bit words to count leading zeros on + * @return the number of leading zeros. + */ +u32 _mali_osk_clz(u32 val); + +/** @brief find last (most-significant) bit set + * + * @param val 32-bit words to count last bit set on + * @return last bit set. + */ +u32 _mali_osk_fls(u32 val); + +/** @} */ /* end group _mali_osk_math */ + +/** @addtogroup _mali_osk_wait_queue OSK Wait Queue functionality + * @{ */ + +/** @brief Initialize an empty Wait Queue */ +_mali_osk_wait_queue_t *_mali_osk_wait_queue_init(void); + +/** @brief Sleep if condition is false + * + * @param queue the queue to use + * @param condition function pointer to a boolean function + * @param data data parameter for condition function + * + * Put thread to sleep if the given \a condition function returns false. When + * being asked to wake up again, the condition will be re-checked and the + * thread only woken up if the condition is now true. + */ +void _mali_osk_wait_queue_wait_event(_mali_osk_wait_queue_t *queue, mali_bool(*condition)(void *), void *data); + +/** @brief Sleep if condition is false + * + * @param queue the queue to use + * @param condition function pointer to a boolean function + * @param data data parameter for condition function + * @param timeout timeout in ms + * + * Put thread to sleep if the given \a condition function returns false. When + * being asked to wake up again, the condition will be re-checked and the + * thread only woken up if the condition is now true. Will return if time + * exceeds timeout. + */ +void _mali_osk_wait_queue_wait_event_timeout(_mali_osk_wait_queue_t *queue, mali_bool(*condition)(void *), void *data, u32 timeout); + +/** @brief Wake up all threads in wait queue if their respective conditions are + * true + * + * @param queue the queue whose threads should be woken up + * + * Wake up all threads in wait queue \a queue whose condition is now true. + */ +void _mali_osk_wait_queue_wake_up(_mali_osk_wait_queue_t *queue); + +/** @brief terminate a wait queue + * + * @param queue the queue to terminate. + */ +void _mali_osk_wait_queue_term(_mali_osk_wait_queue_t *queue); +/** @} */ /* end group _mali_osk_wait_queue */ + + +/** @addtogroup _mali_osk_miscellaneous + * @{ */ + +/** @brief Output a device driver debug message. + * + * The interpretation of \a fmt is the same as the \c format parameter in + * _mali_osu_vsnprintf(). + * + * @param fmt a _mali_osu_vsnprintf() style format string + * @param ... a variable-number of parameters suitable for \a fmt + */ +void _mali_osk_dbgmsg(const char *fmt, ...); + +/** @brief Print fmt into buf. + * + * The interpretation of \a fmt is the same as the \c format parameter in + * _mali_osu_vsnprintf(). + * + * @param buf a pointer to the result buffer + * @param size the total number of bytes allowed to write to \a buf + * @param fmt a _mali_osu_vsnprintf() style format string + * @param ... a variable-number of parameters suitable for \a fmt + * @return The number of bytes written to \a buf + */ +u32 _mali_osk_snprintf(char *buf, u32 size, const char *fmt, ...); + +/** @brief Print fmt into print_ctx. + * + * The interpretation of \a fmt is the same as the \c format parameter in + * _mali_osu_vsnprintf(). + * + * @param print_ctx a pointer to the result file buffer + * @param fmt a _mali_osu_vsnprintf() style format string + * @param ... a variable-number of parameters suitable for \a fmt + */ +void _mali_osk_ctxprintf(_mali_osk_print_ctx *print_ctx, const char *fmt, ...); + +/** @brief Abnormal process abort. + * + * Terminates the caller-process if this function is called. + * + * This function will be called from Debug assert-macros in mali_kernel_common.h. + * + * This function will never return - because to continue from a Debug assert + * could cause even more problems, and hinder debugging of the initial problem. + * + * This function is only used in Debug builds, and is not used in Release builds. + */ +void _mali_osk_abort(void); + +/** @brief Sets breakpoint at point where function is called. + * + * This function will be called from Debug assert-macros in mali_kernel_common.h, + * to assist in debugging. If debugging at this level is not required, then this + * function may be implemented as a stub. + * + * This function is only used in Debug builds, and is not used in Release builds. + */ +void _mali_osk_break(void); + +/** @brief Return an identificator for calling process. + * + * @return Identificator for calling process. + */ +u32 _mali_osk_get_pid(void); + +/** @brief Return an name for calling process. + * + * @return name for calling process. + */ +char *_mali_osk_get_comm(void); + +/** @brief Return an identificator for calling thread. + * + * @return Identificator for calling thread. + */ +u32 _mali_osk_get_tid(void); + + +/** @brief Take a reference to the power manager system for the Mali device (synchronously). + * + * When function returns successfully, Mali is ON. + * + * @note Call \a _mali_osk_pm_dev_ref_put() to release this reference. + */ +_mali_osk_errcode_t _mali_osk_pm_dev_ref_get_sync(void); + +/** @brief Take a reference to the external power manager system for the Mali device (asynchronously). + * + * Mali might not yet be on after this function as returned. + * Please use \a _mali_osk_pm_dev_barrier() or \a _mali_osk_pm_dev_ref_get_sync() + * to wait for Mali to be powered on. + * + * @note Call \a _mali_osk_pm_dev_ref_dec() to release this reference. + */ +_mali_osk_errcode_t _mali_osk_pm_dev_ref_get_async(void); + +/** @brief Release the reference to the external power manger system for the Mali device. + * + * When reference count reach zero, the cores can be off. + * + * @note This must be used to release references taken with + * \a _mali_osk_pm_dev_ref_get_sync() or \a _mali_osk_pm_dev_ref_get_sync(). + */ +void _mali_osk_pm_dev_ref_put(void); + +/** @brief Block until pending PM operations are done + */ +void _mali_osk_pm_dev_barrier(void); + +/** @} */ /* end group _mali_osk_miscellaneous */ + +/** @defgroup _mali_osk_bitmap OSK Bitmap + * @{ */ + +/** @brief Allocate a unique number from the bitmap object. + * + * @param bitmap Initialized bitmap object. + * @return An unique existence in the bitmap object. + */ +u32 _mali_osk_bitmap_alloc(struct _mali_osk_bitmap *bitmap); + +/** @brief Free a interger to the bitmap object. + * + * @param bitmap Initialized bitmap object. + * @param obj An number allocated from bitmap object. + */ +void _mali_osk_bitmap_free(struct _mali_osk_bitmap *bitmap, u32 obj); + +/** @brief Allocate continuous number from the bitmap object. + * + * @param bitmap Initialized bitmap object. + * @return start number of the continuous number block. + */ +u32 _mali_osk_bitmap_alloc_range(struct _mali_osk_bitmap *bitmap, int cnt); + +/** @brief Free a block of continuous number block to the bitmap object. + * + * @param bitmap Initialized bitmap object. + * @param obj Start number. + * @param cnt The size of the continuous number block. + */ +void _mali_osk_bitmap_free_range(struct _mali_osk_bitmap *bitmap, u32 obj, int cnt); + +/** @brief Available count could be used to allocate in the given bitmap object. + * + */ +u32 _mali_osk_bitmap_avail(struct _mali_osk_bitmap *bitmap); + +/** @brief Initialize an bitmap object.. + * + * @param bitmap An poiter of uninitialized bitmap object. + * @param num Size of thei bitmap object and decide the memory size allocated. + * @param reserve start number used to allocate. + */ +int _mali_osk_bitmap_init(struct _mali_osk_bitmap *bitmap, u32 num, u32 reserve); + +/** @brief Free the given bitmap object. + * + * @param bitmap Initialized bitmap object. + */ +void _mali_osk_bitmap_term(struct _mali_osk_bitmap *bitmap); +/** @} */ /* end group _mali_osk_bitmap */ + +/** @} */ /* end group osuapi */ + +/** @} */ /* end group uddapi */ + + + +#ifdef __cplusplus +} +#endif + +/* Check standard inlines */ +#ifndef MALI_STATIC_INLINE +#error MALI_STATIC_INLINE not defined on your OS +#endif + +#ifndef MALI_NON_STATIC_INLINE +#error MALI_NON_STATIC_INLINE not defined on your OS +#endif + +#endif /* __MALI_OSK_H__ */ diff --git a/drivers/gpu/arm/utgard/common/mali_osk_bitops.h b/drivers/gpu/arm/utgard/common/mali_osk_bitops.h new file mode 100644 index 00000000000000..c1709f94c8837c --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_osk_bitops.h @@ -0,0 +1,162 @@ +/* + * Copyright (C) 2010, 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_bitops.h + * Implementation of the OS abstraction layer for the kernel device driver + */ + +#ifndef __MALI_OSK_BITOPS_H__ +#define __MALI_OSK_BITOPS_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +MALI_STATIC_INLINE void _mali_internal_clear_bit(u32 bit, u32 *addr) +{ + MALI_DEBUG_ASSERT(bit < 32); + MALI_DEBUG_ASSERT(NULL != addr); + + (*addr) &= ~(1 << bit); +} + +MALI_STATIC_INLINE void _mali_internal_set_bit(u32 bit, u32 *addr) +{ + MALI_DEBUG_ASSERT(bit < 32); + MALI_DEBUG_ASSERT(NULL != addr); + + (*addr) |= (1 << bit); +} + +MALI_STATIC_INLINE u32 _mali_internal_test_bit(u32 bit, u32 value) +{ + MALI_DEBUG_ASSERT(bit < 32); + return value & (1 << bit); +} + +MALI_STATIC_INLINE int _mali_internal_find_first_zero_bit(u32 value) +{ + u32 inverted; + u32 negated; + u32 isolated; + u32 leading_zeros; + + /* Begin with xxx...x0yyy...y, where ys are 1, number of ys is in range 0..31 */ + inverted = ~value; /* zzz...z1000...0 */ + /* Using count_trailing_zeros on inverted value - + * See ARM System Developers Guide for details of count_trailing_zeros */ + + /* Isolate the zero: it is preceeded by a run of 1s, so add 1 to it */ + negated = (u32) - inverted ; /* -a == ~a + 1 (mod 2^n) for n-bit numbers */ + /* negated = xxx...x1000...0 */ + + isolated = negated & inverted ; /* xxx...x1000...0 & zzz...z1000...0, zs are ~xs */ + /* And so the first zero bit is in the same position as the 1 == number of 1s that preceeded it + * Note that the output is zero if value was all 1s */ + + leading_zeros = _mali_osk_clz(isolated); + + return 31 - leading_zeros; +} + + +/** @defgroup _mali_osk_bitops OSK Non-atomic Bit-operations + * @{ */ + +/** + * These bit-operations do not work atomically, and so locks must be used if + * atomicity is required. + * + * Reference implementations for Little Endian are provided, and so it should + * not normally be necessary to re-implement these. Efficient bit-twiddling + * techniques are used where possible, implemented in portable C. + * + * Note that these reference implementations rely on _mali_osk_clz() being + * implemented. + */ + +/** @brief Clear a bit in a sequence of 32-bit words + * @param nr bit number to clear, starting from the (Little-endian) least + * significant bit + * @param addr starting point for counting. + */ +MALI_STATIC_INLINE void _mali_osk_clear_nonatomic_bit(u32 nr, u32 *addr) +{ + addr += nr >> 5; /* find the correct word */ + nr = nr & ((1 << 5) - 1); /* The bit number within the word */ + + _mali_internal_clear_bit(nr, addr); +} + +/** @brief Set a bit in a sequence of 32-bit words + * @param nr bit number to set, starting from the (Little-endian) least + * significant bit + * @param addr starting point for counting. + */ +MALI_STATIC_INLINE void _mali_osk_set_nonatomic_bit(u32 nr, u32 *addr) +{ + addr += nr >> 5; /* find the correct word */ + nr = nr & ((1 << 5) - 1); /* The bit number within the word */ + + _mali_internal_set_bit(nr, addr); +} + +/** @brief Test a bit in a sequence of 32-bit words + * @param nr bit number to test, starting from the (Little-endian) least + * significant bit + * @param addr starting point for counting. + * @return zero if bit was clear, non-zero if set. Do not rely on the return + * value being related to the actual word under test. + */ +MALI_STATIC_INLINE u32 _mali_osk_test_bit(u32 nr, u32 *addr) +{ + addr += nr >> 5; /* find the correct word */ + nr = nr & ((1 << 5) - 1); /* The bit number within the word */ + + return _mali_internal_test_bit(nr, *addr); +} + +/* Return maxbit if not found */ +/** @brief Find the first zero bit in a sequence of 32-bit words + * @param addr starting point for search. + * @param maxbit the maximum number of bits to search + * @return the number of the first zero bit found, or maxbit if none were found + * in the specified range. + */ +MALI_STATIC_INLINE u32 _mali_osk_find_first_zero_bit(const u32 *addr, u32 maxbit) +{ + u32 total; + + for (total = 0; total < maxbit; total += 32, ++addr) { + int result; + result = _mali_internal_find_first_zero_bit(*addr); + + /* non-negative signifies the bit was found */ + if (result >= 0) { + total += (u32)result; + break; + } + } + + /* Now check if we reached maxbit or above */ + if (total >= maxbit) { + total = maxbit; + } + + return total; /* either the found bit nr, or maxbit if not found */ +} +/** @} */ /* end group _mali_osk_bitops */ + +#ifdef __cplusplus +} +#endif + +#endif /* __MALI_OSK_BITOPS_H__ */ diff --git a/drivers/gpu/arm/utgard/common/mali_osk_list.h b/drivers/gpu/arm/utgard/common/mali_osk_list.h new file mode 100644 index 00000000000000..22d22446d7a0be --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_osk_list.h @@ -0,0 +1,273 @@ +/* + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_list.h + * Implementation of the OS abstraction layer for the kernel device driver + */ + +#ifndef __MALI_OSK_LIST_H__ +#define __MALI_OSK_LIST_H__ + +#include "mali_osk.h" +#include "mali_kernel_common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +MALI_STATIC_INLINE void __mali_osk_list_add(_mali_osk_list_t *new_entry, _mali_osk_list_t *prev, _mali_osk_list_t *next) +{ + next->prev = new_entry; + new_entry->next = next; + new_entry->prev = prev; + prev->next = new_entry; +} + +MALI_STATIC_INLINE void __mali_osk_list_del(_mali_osk_list_t *prev, _mali_osk_list_t *next) +{ + next->prev = prev; + prev->next = next; +} + +/** @addtogroup _mali_osk_list OSK Doubly-Linked Circular Lists + * @{ */ + +/** Reference implementations of Doubly-linked Circular Lists are provided. + * There is often no need to re-implement these. + * + * @note The implementation may differ subtly from any lists the OS provides. + * For this reason, these lists should not be mixed with OS-specific lists + * inside the OSK/UKK implementation. */ + +/** @brief Initialize a list to be a head of an empty list + * @param exp the list to initialize. */ +#define _MALI_OSK_INIT_LIST_HEAD(exp) _mali_osk_list_init(exp) + +/** @brief Define a list variable, which is uninitialized. + * @param exp the name of the variable that the list will be defined as. */ +#define _MALI_OSK_LIST_HEAD(exp) _mali_osk_list_t exp + +/** @brief Define a list variable, which is initialized. + * @param exp the name of the variable that the list will be defined as. */ +#define _MALI_OSK_LIST_HEAD_STATIC_INIT(exp) _mali_osk_list_t exp = { &exp, &exp } + +/** @brief Initialize a list element. + * + * All list elements must be initialized before use. + * + * Do not use on any list element that is present in a list without using + * _mali_osk_list_del first, otherwise this will break the list. + * + * @param list the list element to initialize + */ +MALI_STATIC_INLINE void _mali_osk_list_init(_mali_osk_list_t *list) +{ + list->next = list; + list->prev = list; +} + +/** @brief Insert a single list element after an entry in a list + * + * As an example, if this is inserted to the head of a list, then this becomes + * the first element of the list. + * + * Do not use to move list elements from one list to another, as it will break + * the originating list. + * + * + * @param newlist the list element to insert + * @param list the list in which to insert. The new element will be the next + * entry in this list + */ +MALI_STATIC_INLINE void _mali_osk_list_add(_mali_osk_list_t *new_entry, _mali_osk_list_t *list) +{ + __mali_osk_list_add(new_entry, list, list->next); +} + +/** @brief Insert a single list element before an entry in a list + * + * As an example, if this is inserted to the head of a list, then this becomes + * the last element of the list. + * + * Do not use to move list elements from one list to another, as it will break + * the originating list. + * + * @param newlist the list element to insert + * @param list the list in which to insert. The new element will be the previous + * entry in this list + */ +MALI_STATIC_INLINE void _mali_osk_list_addtail(_mali_osk_list_t *new_entry, _mali_osk_list_t *list) +{ + __mali_osk_list_add(new_entry, list->prev, list); +} + +/** @brief Remove a single element from a list + * + * The element will no longer be present in the list. The removed list element + * will be uninitialized, and so should not be traversed. It must be + * initialized before further use. + * + * @param list the list element to remove. + */ +MALI_STATIC_INLINE void _mali_osk_list_del(_mali_osk_list_t *list) +{ + __mali_osk_list_del(list->prev, list->next); +} + +/** @brief Remove a single element from a list, and re-initialize it + * + * The element will no longer be present in the list. The removed list element + * will initialized, and so can be used as normal. + * + * @param list the list element to remove and initialize. + */ +MALI_STATIC_INLINE void _mali_osk_list_delinit(_mali_osk_list_t *list) +{ + __mali_osk_list_del(list->prev, list->next); + _mali_osk_list_init(list); +} + +/** @brief Determine whether a list is empty. + * + * An empty list is one that contains a single element that points to itself. + * + * @param list the list to check. + * @return non-zero if the list is empty, and zero otherwise. + */ +MALI_STATIC_INLINE mali_bool _mali_osk_list_empty(_mali_osk_list_t *list) +{ + return list->next == list; +} + +/** @brief Move a list element from one list to another. + * + * The list element must be initialized. + * + * As an example, moving a list item to the head of a new list causes this item + * to be the first element in the new list. + * + * @param move the list element to move + * @param list the new list into which the element will be inserted, as the next + * element in the list. + */ +MALI_STATIC_INLINE void _mali_osk_list_move(_mali_osk_list_t *move_entry, _mali_osk_list_t *list) +{ + __mali_osk_list_del(move_entry->prev, move_entry->next); + _mali_osk_list_add(move_entry, list); +} + +/** @brief Move an entire list + * + * The list element must be initialized. + * + * Allows you to move a list from one list head to another list head + * + * @param old_list The existing list head + * @param new_list The new list head (must be an empty list) + */ +MALI_STATIC_INLINE void _mali_osk_list_move_list(_mali_osk_list_t *old_list, _mali_osk_list_t *new_list) +{ + MALI_DEBUG_ASSERT(_mali_osk_list_empty(new_list)); + if (!_mali_osk_list_empty(old_list)) { + new_list->next = old_list->next; + new_list->prev = old_list->prev; + new_list->next->prev = new_list; + new_list->prev->next = new_list; + old_list->next = old_list; + old_list->prev = old_list; + } +} + +/** @brief Find the containing structure of a list + * + * When traversing a list, this is used to recover the containing structure, + * given that is contains a _mali_osk_list_t member. + * + * Each list must be of structures of one type, and must link the same members + * together, otherwise it will not be possible to correctly recover the + * sturctures that the lists link. + * + * @note no type or memory checking occurs to ensure that a structure does in + * fact exist for the list entry, and that it is being recovered with respect + * to the correct list member. + * + * @param ptr the pointer to the _mali_osk_list_t member in this structure + * @param type the type of the structure that contains the member + * @param member the member of the structure that ptr points to. + * @return a pointer to a \a type object which contains the _mali_osk_list_t + * \a member, as pointed to by the _mali_osk_list_t \a *ptr. + */ +#define _MALI_OSK_LIST_ENTRY(ptr, type, member) \ + _MALI_OSK_CONTAINER_OF(ptr, type, member) + +/** @brief Enumerate a list safely + * + * With this macro, lists can be enumerated in a 'safe' manner. That is, + * entries can be deleted from the list without causing an error during + * enumeration. To achieve this, a 'temporary' pointer is required, which must + * be provided to the macro. + * + * Use it like a 'for()', 'while()' or 'do()' construct, and so it must be + * followed by a statement or compound-statement which will be executed for + * each list entry. + * + * Upon loop completion, providing that an early out was not taken in the + * loop body, then it is guaranteed that ptr->member == list, even if the loop + * body never executed. + * + * @param ptr a pointer to an object of type 'type', which points to the + * structure that contains the currently enumerated list entry. + * @param tmp a pointer to an object of type 'type', which must not be used + * inside the list-execution statement. + * @param list a pointer to a _mali_osk_list_t, from which enumeration will + * begin + * @param type the type of the structure that contains the _mali_osk_list_t + * member that is part of the list to be enumerated. + * @param member the _mali_osk_list_t member of the structure that is part of + * the list to be enumerated. + */ +#define _MALI_OSK_LIST_FOREACHENTRY(ptr, tmp, list, type, member) \ + for (ptr = _MALI_OSK_LIST_ENTRY((list)->next, type, member), \ + tmp = _MALI_OSK_LIST_ENTRY(ptr->member.next, type, member); \ + &ptr->member != (list); \ + ptr = tmp, \ + tmp = _MALI_OSK_LIST_ENTRY(tmp->member.next, type, member)) + +/** @brief Enumerate a list in reverse order safely + * + * This macro is identical to @ref _MALI_OSK_LIST_FOREACHENTRY, except that + * entries are enumerated in reverse order. + * + * @param ptr a pointer to an object of type 'type', which points to the + * structure that contains the currently enumerated list entry. + * @param tmp a pointer to an object of type 'type', which must not be used + * inside the list-execution statement. + * @param list a pointer to a _mali_osk_list_t, from which enumeration will + * begin + * @param type the type of the structure that contains the _mali_osk_list_t + * member that is part of the list to be enumerated. + * @param member the _mali_osk_list_t member of the structure that is part of + * the list to be enumerated. + */ +#define _MALI_OSK_LIST_FOREACHENTRY_REVERSE(ptr, tmp, list, type, member) \ + for (ptr = _MALI_OSK_LIST_ENTRY((list)->prev, type, member), \ + tmp = _MALI_OSK_LIST_ENTRY(ptr->member.prev, type, member); \ + &ptr->member != (list); \ + ptr = tmp, \ + tmp = _MALI_OSK_LIST_ENTRY(tmp->member.prev, type, member)) + +/** @} */ /* end group _mali_osk_list */ + +#ifdef __cplusplus +} +#endif + +#endif /* __MALI_OSK_LIST_H__ */ diff --git a/drivers/gpu/arm/utgard/common/mali_osk_mali.h b/drivers/gpu/arm/utgard/common/mali_osk_mali.h new file mode 100644 index 00000000000000..b27fb7dd36edaa --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_osk_mali.h @@ -0,0 +1,97 @@ +/* + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_mali.h + * Defines the OS abstraction layer which is specific for the Mali kernel device driver (OSK) + */ + +#ifndef __MALI_OSK_MALI_H__ +#define __MALI_OSK_MALI_H__ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** @addtogroup _mali_osk_miscellaneous + * @{ */ + +/** @brief Struct with device specific configuration data + */ +typedef struct mali_gpu_device_data _mali_osk_device_data; + +#if defined(CONFIG_MALI_DT) && !defined(CONFIG_MALI_PLAT_SPECIFIC_DT) +/** @brief Initialize those device resources when we use device tree + * + * @return _MALI_OSK_ERR_OK on success, otherwise failure. + */ +_mali_osk_errcode_t _mali_osk_resource_initialize(void); +#endif + +/** @brief Find Mali GPU HW resource + * + * @param addr Address of Mali GPU resource to find + * @param res Storage for resource information if resource is found. + * @return _MALI_OSK_ERR_OK on success, _MALI_OSK_ERR_ITEM_NOT_FOUND if resource is not found + */ +_mali_osk_errcode_t _mali_osk_resource_find(u32 addr, _mali_osk_resource_t *res); + + +/** @brief Find Mali GPU HW base address + * + * @return 0 if resources are found, otherwise the Mali GPU component with lowest address. + */ +uintptr_t _mali_osk_resource_base_address(void); + +/** @brief Find the specific GPU resource. + * + * @return value + * 0x400 if Mali 400 specific GPU resource identified + * 0x450 if Mali 450 specific GPU resource identified + * 0x470 if Mali 470 specific GPU resource identified + * + */ +u32 _mali_osk_identify_gpu_resource(void); + +/** @brief Retrieve the Mali GPU specific data + * + * @return _MALI_OSK_ERR_OK on success, otherwise failure. + */ +_mali_osk_errcode_t _mali_osk_device_data_get(_mali_osk_device_data *data); + +/** @brief Find the pmu domain config from device data. + * + * @param domain_config_array used to store pmu domain config found in device data. + * @param array_size is the size of array domain_config_array. + */ +void _mali_osk_device_data_pmu_config_get(u16 *domain_config_array, int array_size); + +/** @brief Get Mali PMU switch delay + * + *@return pmu switch delay if it is configured + */ +u32 _mali_osk_get_pmu_switch_delay(void); + +/** @brief Determines if Mali GPU has been configured with shared interrupts. + * + * @return MALI_TRUE if shared interrupts, MALI_FALSE if not. + */ +mali_bool _mali_osk_shared_interrupts(void); + +/** @} */ /* end group _mali_osk_miscellaneous */ + +#ifdef __cplusplus +} +#endif + +#endif /* __MALI_OSK_MALI_H__ */ diff --git a/drivers/gpu/arm/utgard/common/mali_osk_profiling.h b/drivers/gpu/arm/utgard/common/mali_osk_profiling.h new file mode 100644 index 00000000000000..10f4dc552b03d4 --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_osk_profiling.h @@ -0,0 +1,146 @@ +/* + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_OSK_PROFILING_H__ +#define __MALI_OSK_PROFILING_H__ + +#if defined(CONFIG_MALI400_PROFILING) && defined (CONFIG_TRACEPOINTS) + +#include "mali_linux_trace.h" +#include "mali_profiling_events.h" +#include "mali_profiling_gator_api.h" + +#define MALI_PROFILING_MAX_BUFFER_ENTRIES 1048576 + +#define MALI_PROFILING_NO_HW_COUNTER = ((u32)-1) + +/** @defgroup _mali_osk_profiling External profiling connectivity + * @{ */ + +/** + * Initialize the profiling module. + * @return _MALI_OSK_ERR_OK on success, otherwise failure. + */ +_mali_osk_errcode_t _mali_osk_profiling_init(mali_bool auto_start); + +/* + * Terminate the profiling module. + */ +void _mali_osk_profiling_term(void); + +/** + * Stop the profile sampling operation. + */ +void _mali_osk_profiling_stop_sampling(u32 pid); + +/** + * Start recording profiling data + * + * The specified limit will determine how large the capture buffer is. + * MALI_PROFILING_MAX_BUFFER_ENTRIES determines the maximum size allowed by the device driver. + * + * @param limit The desired maximum number of events to record on input, the actual maximum on output. + * @return _MALI_OSK_ERR_OK on success, otherwise failure. + */ +_mali_osk_errcode_t _mali_osk_profiling_start(u32 *limit); + +/** + * Add an profiling event + * + * @param event_id The event identificator. + * @param data0 First data parameter, depending on event_id specified. + * @param data1 Second data parameter, depending on event_id specified. + * @param data2 Third data parameter, depending on event_id specified. + * @param data3 Fourth data parameter, depending on event_id specified. + * @param data4 Fifth data parameter, depending on event_id specified. + */ +void _mali_osk_profiling_add_event(u32 event_id, u32 data0, u32 data1, u32 data2, u32 data3, u32 data4); + +/** + * Report a hardware counter event. + * + * @param counter_id The ID of the counter. + * @param value The value of the counter. + */ + +/* Call Linux tracepoint directly */ +#define _mali_osk_profiling_report_hw_counter(counter_id, value) trace_mali_hw_counter(counter_id, value) + +/** + * Report SW counters + * + * @param counters array of counter values + */ +void _mali_osk_profiling_report_sw_counters(u32 *counters); + +void _mali_osk_profiling_record_global_counters(int counter_id, u32 value); + +/** + * Stop recording profiling data + * + * @param count Returns the number of recorded events. + * @return _MALI_OSK_ERR_OK on success, otherwise failure. + */ +_mali_osk_errcode_t _mali_osk_profiling_stop(u32 *count); + +/** + * Retrieves the number of events that can be retrieved + * + * @return The number of recorded events that can be retrieved. + */ +u32 _mali_osk_profiling_get_count(void); + +/** + * Retrieve an event + * + * @param index Event index (start with 0 and continue until this function fails to retrieve all events) + * @param timestamp The timestamp for the retrieved event will be stored here. + * @param event_id The event ID for the retrieved event will be stored here. + * @param data The 5 data values for the retrieved event will be stored here. + * @return _MALI_OSK_ERR_OK on success, otherwise failure. + */ +_mali_osk_errcode_t _mali_osk_profiling_get_event(u32 index, u64 *timestamp, u32 *event_id, u32 data[5]); + +/** + * Clear the recorded buffer. + * + * This is needed in order to start another recording. + * + * @return _MALI_OSK_ERR_OK on success, otherwise failure. + */ +_mali_osk_errcode_t _mali_osk_profiling_clear(void); + +/** + * Checks if a recording of profiling data is in progress + * + * @return MALI_TRUE if recording of profiling data is in progress, MALI_FALSE if not + */ +mali_bool _mali_osk_profiling_is_recording(void); + +/** + * Checks if profiling data is available for retrival + * + * @return MALI_TRUE if profiling data is avaiable, MALI_FALSE if not + */ +mali_bool _mali_osk_profiling_have_recording(void); + +/** @} */ /* end group _mali_osk_profiling */ + +#else /* defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_TRACEPOINTS) */ + +/* Dummy add_event, for when profiling is disabled. */ + +#define _mali_osk_profiling_add_event(event_id, data0, data1, data2, data3, data4) + +#endif /* defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_TRACEPOINTS) */ + +#endif /* __MALI_OSK_PROFILING_H__ */ + + diff --git a/drivers/gpu/arm/utgard/common/mali_osk_types.h b/drivers/gpu/arm/utgard/common/mali_osk_types.h new file mode 100644 index 00000000000000..b65ad29e16c02f --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_osk_types.h @@ -0,0 +1,471 @@ +/* + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_types.h + * Defines types of the OS abstraction layer for the kernel device driver (OSK) + */ + +#ifndef __MALI_OSK_TYPES_H__ +#define __MALI_OSK_TYPES_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @addtogroup uddapi Unified Device Driver (UDD) APIs + * + * @{ + */ + +/** + * @addtogroup oskapi UDD OS Abstraction for Kernel-side (OSK) APIs + * + * @{ + */ + +/** @defgroup _mali_osk_miscellaneous OSK Miscellaneous functions, constants and types + * @{ */ + +/* Define integer types used by OSK. Note: these currently clash with Linux so we only define them if not defined already */ +#ifndef __KERNEL__ +typedef unsigned char u8; +typedef signed char s8; +typedef unsigned short u16; +typedef signed short s16; +typedef unsigned int u32; +typedef signed int s32; +typedef unsigned long long u64; +#define BITS_PER_LONG (sizeof(long)*8) +#else +/* Ensure Linux types u32, etc. are defined */ +#include +#endif + +/** @brief Mali Boolean type which uses MALI_TRUE and MALI_FALSE + */ +typedef unsigned long mali_bool; + +#ifndef MALI_TRUE +#define MALI_TRUE ((mali_bool)1) +#endif + +#ifndef MALI_FALSE +#define MALI_FALSE ((mali_bool)0) +#endif + +#define MALI_HW_CORE_NO_COUNTER ((u32)-1) + + +#define MALI_S32_MAX 0x7fffffff + +/** + * @brief OSK Error codes + * + * Each OS may use its own set of error codes, and may require that the + * User/Kernel interface take certain error code. This means that the common + * error codes need to be sufficiently rich to pass the correct error code + * thorugh from the OSK to U/K layer, across all OSs. + * + * The result is that some error codes will appear redundant on some OSs. + * Under all OSs, the OSK layer must translate native OS error codes to + * _mali_osk_errcode_t codes. Similarly, the U/K layer must translate from + * _mali_osk_errcode_t codes to native OS error codes. + */ +typedef enum { + _MALI_OSK_ERR_OK = 0, /**< Success. */ + _MALI_OSK_ERR_FAULT = -1, /**< General non-success */ + _MALI_OSK_ERR_INVALID_FUNC = -2, /**< Invalid function requested through User/Kernel interface (e.g. bad IOCTL number) */ + _MALI_OSK_ERR_INVALID_ARGS = -3, /**< Invalid arguments passed through User/Kernel interface */ + _MALI_OSK_ERR_NOMEM = -4, /**< Insufficient memory */ + _MALI_OSK_ERR_TIMEOUT = -5, /**< Timeout occurred */ + _MALI_OSK_ERR_RESTARTSYSCALL = -6, /**< Special: On certain OSs, must report when an interruptable mutex is interrupted. Ignore otherwise. */ + _MALI_OSK_ERR_ITEM_NOT_FOUND = -7, /**< Table Lookup failed */ + _MALI_OSK_ERR_BUSY = -8, /**< Device/operation is busy. Try again later */ + _MALI_OSK_ERR_UNSUPPORTED = -9, /**< Optional part of the interface used, and is unsupported */ +} _mali_osk_errcode_t; + +/** @} */ /* end group _mali_osk_miscellaneous */ + +/** @defgroup _mali_osk_wq OSK work queues + * @{ */ + +/** @brief Private type for work objects */ +typedef struct _mali_osk_wq_work_s _mali_osk_wq_work_t; +typedef struct _mali_osk_wq_delayed_work_s _mali_osk_wq_delayed_work_t; + +/** @brief Work queue handler function + * + * This function type is called when the work is scheduled by the work queue, + * e.g. as an IRQ bottom-half handler. + * + * Refer to \ref _mali_osk_wq_schedule_work() for more information on the + * work-queue and work handlers. + * + * @param arg resource-specific data + */ +typedef void (*_mali_osk_wq_work_handler_t)(void *arg); + +/* @} */ /* end group _mali_osk_wq */ + +/** @defgroup _mali_osk_irq OSK IRQ handling + * @{ */ + +/** @brief Private type for IRQ handling objects */ +typedef struct _mali_osk_irq_t_struct _mali_osk_irq_t; + +/** @brief Optional function to trigger an irq from a resource + * + * This function is implemented by the common layer to allow probing of a resource's IRQ. + * @param arg resource-specific data */ +typedef void (*_mali_osk_irq_trigger_t)(void *arg); + +/** @brief Optional function to acknowledge an irq from a resource + * + * This function is implemented by the common layer to allow probing of a resource's IRQ. + * @param arg resource-specific data + * @return _MALI_OSK_ERR_OK if the IRQ was successful, or a suitable _mali_osk_errcode_t on failure. */ +typedef _mali_osk_errcode_t (*_mali_osk_irq_ack_t)(void *arg); + +/** @brief IRQ 'upper-half' handler callback. + * + * This function is implemented by the common layer to do the initial handling of a + * resource's IRQ. This maps on to the concept of an ISR that does the minimum + * work necessary before handing off to an IST. + * + * The communication of the resource-specific data from the ISR to the IST is + * handled by the OSK implementation. + * + * On most systems, the IRQ upper-half handler executes in IRQ context. + * Therefore, the system may have restrictions about what can be done in this + * context + * + * If an IRQ upper-half handler requires more work to be done than can be + * acheived in an IRQ context, then it may defer the work with + * _mali_osk_wq_schedule_work(). Refer to \ref _mali_osk_wq_create_work() for + * more information. + * + * @param arg resource-specific data + * @return _MALI_OSK_ERR_OK if the IRQ was correctly handled, or a suitable + * _mali_osk_errcode_t otherwise. + */ +typedef _mali_osk_errcode_t (*_mali_osk_irq_uhandler_t)(void *arg); + + +/** @} */ /* end group _mali_osk_irq */ + + +/** @defgroup _mali_osk_atomic OSK Atomic counters + * @{ */ + +/** @brief Public type of atomic counters + * + * This is public for allocation on stack. On systems that support it, this is just a single 32-bit value. + * On others, it could be encapsulating an object stored elsewhere. + * + * Regardless of implementation, the \ref _mali_osk_atomic functions \b must be used + * for all accesses to the variable's value, even if atomicity is not required. + * Do not access u.val or u.obj directly. + */ +typedef struct { + union { + u32 val; + void *obj; + } u; +} _mali_osk_atomic_t; +/** @} */ /* end group _mali_osk_atomic */ + + +/** @defgroup _mali_osk_lock OSK Mutual Exclusion Locks + * @{ */ + + +/** @brief OSK Mutual Exclusion Lock ordered list + * + * This lists the various types of locks in the system and is used to check + * that locks are taken in the correct order. + * + * - Holding more than one lock of the same order at the same time is not + * allowed. + * - Taking a lock of a lower order than the highest-order lock currently held + * is not allowed. + * + */ +typedef enum { + /* || Locks || */ + /* || must be || */ + /* _||_ taken in _||_ */ + /* \ / this \ / */ + /* \/ order! \/ */ + + _MALI_OSK_LOCK_ORDER_FIRST = 0, + + _MALI_OSK_LOCK_ORDER_SESSIONS, + _MALI_OSK_LOCK_ORDER_MEM_SESSION, + _MALI_OSK_LOCK_ORDER_MEM_INFO, + _MALI_OSK_LOCK_ORDER_MEM_PT_CACHE, + _MALI_OSK_LOCK_ORDER_DESCRIPTOR_MAP, + _MALI_OSK_LOCK_ORDER_PM_EXECUTION, + _MALI_OSK_LOCK_ORDER_EXECUTOR, + _MALI_OSK_LOCK_ORDER_TIMELINE_SYSTEM, + _MALI_OSK_LOCK_ORDER_SCHEDULER, + _MALI_OSK_LOCK_ORDER_SCHEDULER_DEFERRED, + _MALI_OSK_LOCK_ORDER_PROFILING, + _MALI_OSK_LOCK_ORDER_L2, + _MALI_OSK_LOCK_ORDER_L2_COMMAND, + _MALI_OSK_LOCK_ORDER_UTILIZATION, + _MALI_OSK_LOCK_ORDER_SESSION_PENDING_JOBS, + _MALI_OSK_LOCK_ORDER_PM_STATE, + + _MALI_OSK_LOCK_ORDER_LAST, +} _mali_osk_lock_order_t; + + +/** @brief OSK Mutual Exclusion Lock flags type + * + * - Any lock can use the order parameter. + */ +typedef enum { + _MALI_OSK_LOCKFLAG_UNORDERED = 0x1, /**< Indicate that the order of this lock should not be checked */ + _MALI_OSK_LOCKFLAG_ORDERED = 0x2, + /** @enum _mali_osk_lock_flags_t + * + * Flags from 0x10000--0x80000000 are RESERVED for User-mode */ + +} _mali_osk_lock_flags_t; + +/** @brief Mutual Exclusion Lock Mode Optimization hint + * + * The lock mode is used to implement the read/write locking of locks when we call + * functions _mali_osk_mutex_rw_init/wait/signal/term/. In this case, the RO mode can + * be used to allow multiple concurrent readers, but no writers. The RW mode is used for + * writers, and so will wait for all readers to release the lock (if any present). + * Further readers and writers will wait until the writer releases the lock. + * + * The mode is purely an optimization hint: for example, it is permissible for + * all locks to behave in RW mode, regardless of that supplied. + * + * It is an error to attempt to use locks in anything other that RW mode when + * call functions _mali_osk_mutex_rw_wait/signal(). + * + */ +typedef enum { + _MALI_OSK_LOCKMODE_UNDEF = -1, /**< Undefined lock mode. For internal use only */ + _MALI_OSK_LOCKMODE_RW = 0x0, /**< Read-write mode, default. All readers and writers are mutually-exclusive */ + _MALI_OSK_LOCKMODE_RO, /**< Read-only mode, to support multiple concurrent readers, but mutual exclusion in the presence of writers. */ + /** @enum _mali_osk_lock_mode_t + * + * Lock modes 0x40--0x7F are RESERVED for User-mode */ +} _mali_osk_lock_mode_t; + +/** @brief Private types for Mutual Exclusion lock objects */ +typedef struct _mali_osk_lock_debug_s _mali_osk_lock_debug_t; +typedef struct _mali_osk_spinlock_s _mali_osk_spinlock_t; +typedef struct _mali_osk_spinlock_irq_s _mali_osk_spinlock_irq_t; +typedef struct _mali_osk_mutex_s _mali_osk_mutex_t; +typedef struct _mali_osk_mutex_rw_s _mali_osk_mutex_rw_t; + +/** @} */ /* end group _mali_osk_lock */ + +/** @defgroup _mali_osk_low_level_memory OSK Low-level Memory Operations + * @{ */ + +/** + * @brief Private data type for use in IO accesses to/from devices. + * + * This represents some range that is accessible from the device. Examples + * include: + * - Device Registers, which could be readable and/or writeable. + * - Memory that the device has access to, for storing configuration structures. + * + * Access to this range must be made through the _mali_osk_mem_ioread32() and + * _mali_osk_mem_iowrite32() functions. + */ +typedef struct _mali_io_address *mali_io_address; + +/** @defgroup _MALI_OSK_CPU_PAGE CPU Physical page size macros. + * + * The order of the page size is supplied for + * ease of use by algorithms that might require it, since it is easier to know + * it ahead of time rather than calculating it. + * + * The Mali Page Mask macro masks off the lower bits of a physical address to + * give the start address of the page for that physical address. + * + * @note The Mali device driver code is designed for systems with 4KB page size. + * Changing these macros will not make the entire Mali device driver work with + * page sizes other than 4KB. + * + * @note The CPU Physical Page Size has been assumed to be the same as the Mali + * Physical Page Size. + * + * @{ + */ + +/** CPU Page Order, as log to base 2 of the Page size. @see _MALI_OSK_CPU_PAGE_SIZE */ +#define _MALI_OSK_CPU_PAGE_ORDER ((u32)12) +/** CPU Page Size, in bytes. */ +#define _MALI_OSK_CPU_PAGE_SIZE (((u32)1) << (_MALI_OSK_CPU_PAGE_ORDER)) +/** CPU Page Mask, which masks off the offset within a page */ +#define _MALI_OSK_CPU_PAGE_MASK (~((((u32)1) << (_MALI_OSK_CPU_PAGE_ORDER)) - ((u32)1))) +/** @} */ /* end of group _MALI_OSK_CPU_PAGE */ + +/** @defgroup _MALI_OSK_MALI_PAGE Mali Physical Page size macros + * + * Mali Physical page size macros. The order of the page size is supplied for + * ease of use by algorithms that might require it, since it is easier to know + * it ahead of time rather than calculating it. + * + * The Mali Page Mask macro masks off the lower bits of a physical address to + * give the start address of the page for that physical address. + * + * @note The Mali device driver code is designed for systems with 4KB page size. + * Changing these macros will not make the entire Mali device driver work with + * page sizes other than 4KB. + * + * @note The Mali Physical Page Size has been assumed to be the same as the CPU + * Physical Page Size. + * + * @{ + */ + +/** Mali Page Order, as log to base 2 of the Page size. @see _MALI_OSK_MALI_PAGE_SIZE */ +#define _MALI_OSK_MALI_PAGE_ORDER PAGE_SHIFT +/** Mali Page Size, in bytes. */ +#define _MALI_OSK_MALI_PAGE_SIZE PAGE_SIZE +/** Mali Page Mask, which masks off the offset within a page */ +#define _MALI_OSK_MALI_PAGE_MASK PAGE_MASK +/** @} */ /* end of group _MALI_OSK_MALI_PAGE*/ + +/** @brief flags for mapping a user-accessible memory range + * + * Where a function with prefix '_mali_osk_mem_mapregion' accepts flags as one + * of the function parameters, it will use one of these. These allow per-page + * control over mappings. Compare with the mali_memory_allocation_flag type, + * which acts over an entire range + * + * These may be OR'd together with bitwise OR (|), but must be cast back into + * the type after OR'ing. + */ +typedef enum { + _MALI_OSK_MEM_MAPREGION_FLAG_OS_ALLOCATED_PHYSADDR = 0x1, /**< Physical address is OS Allocated */ +} _mali_osk_mem_mapregion_flags_t; +/** @} */ /* end group _mali_osk_low_level_memory */ + +/** @defgroup _mali_osk_notification OSK Notification Queues + * @{ */ + +/** @brief Private type for notification queue objects */ +typedef struct _mali_osk_notification_queue_t_struct _mali_osk_notification_queue_t; + +/** @brief Public notification data object type */ +typedef struct _mali_osk_notification_t_struct { + u32 notification_type; /**< The notification type */ + u32 result_buffer_size; /**< Size of the result buffer to copy to user space */ + void *result_buffer; /**< Buffer containing any type specific data */ +} _mali_osk_notification_t; + +/** @} */ /* end group _mali_osk_notification */ + + +/** @defgroup _mali_osk_timer OSK Timer Callbacks + * @{ */ + +/** @brief Function to call when a timer expires + * + * When a timer expires, this function is called. Note that on many systems, + * a timer callback will be executed in IRQ context. Therefore, restrictions + * may apply on what can be done inside the timer callback. + * + * If a timer requires more work to be done than can be acheived in an IRQ + * context, then it may defer the work with a work-queue. For example, it may + * use \ref _mali_osk_wq_schedule_work() to make use of a bottom-half handler + * to carry out the remaining work. + * + * Stopping the timer with \ref _mali_osk_timer_del() blocks on compeletion of + * the callback. Therefore, the callback may not obtain any mutexes also held + * by any callers of _mali_osk_timer_del(). Otherwise, a deadlock may occur. + * + * @param arg Function-specific data */ +typedef void (*_mali_osk_timer_callback_t)(void *arg); + +/** @brief Private type for Timer Callback Objects */ +typedef struct _mali_osk_timer_t_struct _mali_osk_timer_t; +/** @} */ /* end group _mali_osk_timer */ + + +/** @addtogroup _mali_osk_list OSK Doubly-Linked Circular Lists + * @{ */ + +/** @brief Public List objects. + * + * To use, add a _mali_osk_list_t member to the structure that may become part + * of a list. When traversing the _mali_osk_list_t objects, use the + * _MALI_OSK_CONTAINER_OF() macro to recover the structure from its + *_mali_osk_list_t member + * + * Each structure may have multiple _mali_osk_list_t members, so that the + * structure is part of multiple lists. When traversing lists, ensure that the + * correct _mali_osk_list_t member is used, because type-checking will be + * lost by the compiler. + */ +typedef struct _mali_osk_list_s { + struct _mali_osk_list_s *next; + struct _mali_osk_list_s *prev; +} _mali_osk_list_t; +/** @} */ /* end group _mali_osk_list */ + +/** @addtogroup _mali_osk_miscellaneous + * @{ */ + +/** @brief resource description struct + * + * Platform independent representation of a Mali HW resource + */ +typedef struct _mali_osk_resource { + const char *description; /**< short description of the resource */ + uintptr_t base; /**< Physical base address of the resource, as seen by Mali resources. */ + const char *irq_name; /**< Name of irq belong to this resource */ + u32 irq; /**< IRQ number delivered to the CPU, or -1 to tell the driver to probe for it (if possible) */ +} _mali_osk_resource_t; +/** @} */ /* end group _mali_osk_miscellaneous */ + +/** @defgroup _mali_osk_wait_queue OSK Wait Queue functionality + * @{ */ +/** @brief Private type for wait queue objects */ +typedef struct _mali_osk_wait_queue_t_struct _mali_osk_wait_queue_t; +/** @} */ /* end group _mali_osk_wait_queue */ + +/** @} */ /* end group osuapi */ + +/** @} */ /* end group uddapi */ + +/** @brief Mali print ctx type which uses seq_file + */ +typedef struct seq_file _mali_osk_print_ctx; + +#define _MALI_OSK_BITMAP_INVALIDATE_INDEX -1 + +typedef struct _mali_osk_bitmap { + u32 reserve; + u32 last; + u32 max; + u32 avail; + _mali_osk_spinlock_t *lock; + unsigned long *table; +} _mali_osk_bitmap_t; + + +#ifdef __cplusplus +} +#endif + +#endif /* __MALI_OSK_TYPES_H__ */ diff --git a/drivers/gpu/arm/utgard/common/mali_pm.c b/drivers/gpu/arm/utgard/common/mali_pm.c new file mode 100644 index 00000000000000..dbd94d31074155 --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_pm.c @@ -0,0 +1,1362 @@ +/* + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_pm.h" +#include "mali_kernel_common.h" +#include "mali_osk.h" +#include "mali_osk_mali.h" +#include "mali_scheduler.h" +#include "mali_group.h" +#include "mali_pm_domain.h" +#include "mali_pmu.h" + +#include "mali_executor.h" +#include "mali_control_timer.h" + +#if defined(DEBUG) +u32 num_pm_runtime_resume = 0; +u32 num_pm_updates = 0; +u32 num_pm_updates_up = 0; +u32 num_pm_updates_down = 0; +#endif + +#define MALI_PM_DOMAIN_DUMMY_MASK (1 << MALI_DOMAIN_INDEX_DUMMY) + +/* lock protecting power state (including pm_domains) */ +static _mali_osk_spinlock_irq_t *pm_lock_state = NULL; + +/* the wanted domain mask (protected by pm_lock_state) */ +static u32 pd_mask_wanted = 0; + +/* used to deferring the actual power changes */ +static _mali_osk_wq_work_t *pm_work = NULL; + +/* lock protecting power change execution */ +static _mali_osk_mutex_t *pm_lock_exec = NULL; + +/* PMU domains which are actually powered on (protected by pm_lock_exec) */ +static u32 pmu_mask_current = 0; + +/* + * domains which marked as powered on (protected by pm_lock_exec) + * This can be different from pmu_mask_current right after GPU power on + * if the PMU domains default to powered up. + */ +static u32 pd_mask_current = 0; + +static u16 domain_config[MALI_MAX_NUMBER_OF_DOMAINS] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1 << MALI_DOMAIN_INDEX_DUMMY +}; + +/* The relative core power cost */ +#define MALI_GP_COST 3 +#define MALI_PP_COST 6 +#define MALI_L2_COST 1 + +/* + *We have MALI_MAX_NUMBER_OF_PP_PHYSICAL_CORES + 1 rows in this matrix + *because we mush store the mask of different pp cores: 0, 1, 2, 3, 4, 5, 6, 7, 8. + */ +static int mali_pm_domain_power_cost_result[MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS + 1][MALI_MAX_NUMBER_OF_DOMAINS]; +/* + * Keep track of runtime PM state, so that we know + * how to resume during OS resume. + */ +#ifdef CONFIG_PM_RUNTIME +static mali_bool mali_pm_runtime_active = MALI_FALSE; +#else +/* when kernel don't enable PM_RUNTIME, set the flag always true, + * for GPU will not power off by runtime */ +static mali_bool mali_pm_runtime_active = MALI_TRUE; +#endif + +static void mali_pm_state_lock(void); +static void mali_pm_state_unlock(void); +static _mali_osk_errcode_t mali_pm_create_pm_domains(void); +static void mali_pm_set_pmu_domain_config(void); +static u32 mali_pm_get_registered_cores_mask(void); +static void mali_pm_update_sync_internal(void); +static mali_bool mali_pm_common_suspend(void); +static void mali_pm_update_work(void *data); +#if defined(DEBUG) +const char *mali_pm_mask_to_string(u32 mask); +const char *mali_pm_group_stats_to_string(void); +#endif + +_mali_osk_errcode_t mali_pm_initialize(void) +{ + _mali_osk_errcode_t err; + struct mali_pmu_core *pmu; + + pm_lock_state = _mali_osk_spinlock_irq_init(_MALI_OSK_LOCKFLAG_ORDERED, + _MALI_OSK_LOCK_ORDER_PM_STATE); + if (NULL == pm_lock_state) { + mali_pm_terminate(); + return _MALI_OSK_ERR_FAULT; + } + + pm_lock_exec = _mali_osk_mutex_init(_MALI_OSK_LOCKFLAG_ORDERED, + _MALI_OSK_LOCK_ORDER_PM_STATE); + if (NULL == pm_lock_exec) { + mali_pm_terminate(); + return _MALI_OSK_ERR_FAULT; + } + + pm_work = _mali_osk_wq_create_work(mali_pm_update_work, NULL); + if (NULL == pm_work) { + mali_pm_terminate(); + return _MALI_OSK_ERR_FAULT; + } + + pmu = mali_pmu_get_global_pmu_core(); + if (NULL != pmu) { + /* + * We have a Mali PMU, set the correct domain + * configuration (default or custom) + */ + + u32 registered_cores_mask; + + mali_pm_set_pmu_domain_config(); + + registered_cores_mask = mali_pm_get_registered_cores_mask(); + mali_pmu_set_registered_cores_mask(pmu, registered_cores_mask); + + MALI_DEBUG_ASSERT(0 == pd_mask_wanted); + } + + /* Create all power domains needed (at least one dummy domain) */ + err = mali_pm_create_pm_domains(); + if (_MALI_OSK_ERR_OK != err) { + mali_pm_terminate(); + return err; + } + + return _MALI_OSK_ERR_OK; +} + +void mali_pm_terminate(void) +{ + if (NULL != pm_work) { + _mali_osk_wq_delete_work(pm_work); + pm_work = NULL; + } + + mali_pm_domain_terminate(); + + if (NULL != pm_lock_exec) { + _mali_osk_mutex_term(pm_lock_exec); + pm_lock_exec = NULL; + } + + if (NULL != pm_lock_state) { + _mali_osk_spinlock_irq_term(pm_lock_state); + pm_lock_state = NULL; + } +} + +struct mali_pm_domain *mali_pm_register_l2_cache(u32 domain_index, + struct mali_l2_cache_core *l2_cache) +{ + struct mali_pm_domain *domain; + + domain = mali_pm_domain_get_from_mask(domain_config[domain_index]); + if (NULL == domain) { + MALI_DEBUG_ASSERT(0 == domain_config[domain_index]); + domain = mali_pm_domain_get_from_index( + MALI_DOMAIN_INDEX_DUMMY); + domain_config[domain_index] = MALI_PM_DOMAIN_DUMMY_MASK; + } else { + MALI_DEBUG_ASSERT(0 != domain_config[domain_index]); + } + + MALI_DEBUG_ASSERT(NULL != domain); + + mali_pm_domain_add_l2_cache(domain, l2_cache); + + return domain; /* return the actual domain this was registered in */ +} + +struct mali_pm_domain *mali_pm_register_group(u32 domain_index, + struct mali_group *group) +{ + struct mali_pm_domain *domain; + + domain = mali_pm_domain_get_from_mask(domain_config[domain_index]); + if (NULL == domain) { + MALI_DEBUG_ASSERT(0 == domain_config[domain_index]); + domain = mali_pm_domain_get_from_index( + MALI_DOMAIN_INDEX_DUMMY); + domain_config[domain_index] = MALI_PM_DOMAIN_DUMMY_MASK; + } else { + MALI_DEBUG_ASSERT(0 != domain_config[domain_index]); + } + + MALI_DEBUG_ASSERT(NULL != domain); + + mali_pm_domain_add_group(domain, group); + + return domain; /* return the actual domain this was registered in */ +} + +mali_bool mali_pm_get_domain_refs(struct mali_pm_domain **domains, + struct mali_group **groups, + u32 num_domains) +{ + mali_bool ret = MALI_TRUE; /* Assume all is powered on instantly */ + u32 i; + + mali_pm_state_lock(); + + for (i = 0; i < num_domains; i++) { + MALI_DEBUG_ASSERT_POINTER(domains[i]); + pd_mask_wanted |= mali_pm_domain_ref_get(domains[i]); + if (MALI_FALSE == mali_pm_domain_power_is_on(domains[i])) { + /* + * Tell caller that the corresponding group + * was not already powered on. + */ + ret = MALI_FALSE; + } else { + /* + * There is a time gap between we power on the domain and + * set the power state of the corresponding groups to be on. + */ + if (NULL != groups[i] && + MALI_FALSE == mali_group_power_is_on(groups[i])) { + ret = MALI_FALSE; + } + } + } + + MALI_DEBUG_PRINT(3, ("PM: wanted domain mask = 0x%08X (get refs)\n", pd_mask_wanted)); + + mali_pm_state_unlock(); + + return ret; +} + +mali_bool mali_pm_put_domain_refs(struct mali_pm_domain **domains, + u32 num_domains) +{ + u32 mask = 0; + mali_bool ret; + u32 i; + + mali_pm_state_lock(); + + for (i = 0; i < num_domains; i++) { + MALI_DEBUG_ASSERT_POINTER(domains[i]); + mask |= mali_pm_domain_ref_put(domains[i]); + } + + if (0 == mask) { + /* return false, all domains should still stay on */ + ret = MALI_FALSE; + } else { + /* Assert that we are dealing with a change */ + MALI_DEBUG_ASSERT((pd_mask_wanted & mask) == mask); + + /* Update our desired domain mask */ + pd_mask_wanted &= ~mask; + + /* return true; one or more domains can now be powered down */ + ret = MALI_TRUE; + } + + MALI_DEBUG_PRINT(3, ("PM: wanted domain mask = 0x%08X (put refs)\n", pd_mask_wanted)); + + mali_pm_state_unlock(); + + return ret; +} + +void mali_pm_init_begin(void) +{ + struct mali_pmu_core *pmu = mali_pmu_get_global_pmu_core(); + + _mali_osk_pm_dev_ref_get_sync(); + + /* Ensure all PMU domains are on */ + if (NULL != pmu) { + mali_pmu_power_up_all(pmu); + } +} + +void mali_pm_init_end(void) +{ + struct mali_pmu_core *pmu = mali_pmu_get_global_pmu_core(); + + /* Ensure all PMU domains are off */ + if (NULL != pmu) { + mali_pmu_power_down_all(pmu); + } + + _mali_osk_pm_dev_ref_put(); +} + +void mali_pm_update_sync(void) +{ + mali_pm_exec_lock(); + + if (MALI_TRUE == mali_pm_runtime_active) { + /* + * Only update if GPU is powered on. + * Deactivation of the last group will result in both a + * deferred runtime PM suspend operation and + * deferred execution of this function. + * mali_pm_runtime_active will be false if runtime PM + * executed first and thus the GPU is now fully powered off. + */ + mali_pm_update_sync_internal(); + } + + mali_pm_exec_unlock(); +} + +void mali_pm_update_async(void) +{ + _mali_osk_wq_schedule_work(pm_work); +} + +void mali_pm_os_suspend(mali_bool os_suspend) +{ + int ret; + + MALI_DEBUG_PRINT(3, ("Mali PM: OS suspend\n")); + + /* Suspend execution of all jobs, and go to inactive state */ + mali_executor_suspend(); + + if (os_suspend) { + mali_control_timer_suspend(MALI_TRUE); + } + + mali_pm_exec_lock(); + + ret = mali_pm_common_suspend(); + + MALI_DEBUG_ASSERT(MALI_TRUE == ret); + MALI_IGNORE(ret); + + mali_pm_exec_unlock(); +} + +void mali_pm_os_resume(void) +{ + struct mali_pmu_core *pmu = mali_pmu_get_global_pmu_core(); + + MALI_DEBUG_PRINT(3, ("Mali PM: OS resume\n")); + + mali_pm_exec_lock(); + +#if defined(DEBUG) + mali_pm_state_lock(); + + /* Assert that things are as we left them in os_suspend(). */ + MALI_DEBUG_ASSERT(0 == pd_mask_wanted); + MALI_DEBUG_ASSERT(0 == pd_mask_current); + MALI_DEBUG_ASSERT(0 == pmu_mask_current); + + MALI_DEBUG_ASSERT(MALI_TRUE == mali_pm_domain_all_unused()); + + mali_pm_state_unlock(); +#endif + + if (MALI_TRUE == mali_pm_runtime_active) { + /* Runtime PM was active, so reset PMU */ + if (NULL != pmu) { + mali_pmu_reset(pmu); + pmu_mask_current = mali_pmu_get_mask(pmu); + + MALI_DEBUG_PRINT(3, ("Mali PM: OS resume 0x%x \n", pmu_mask_current)); + } + + mali_pm_update_sync_internal(); + } + + mali_pm_exec_unlock(); + + /* Start executing jobs again */ + mali_executor_resume(); +} + +mali_bool mali_pm_runtime_suspend(void) +{ + mali_bool ret; + + MALI_DEBUG_PRINT(3, ("Mali PM: Runtime suspend\n")); + + mali_pm_exec_lock(); + + /* + * Put SW state directly into "off" state, and do not bother to power + * down each power domain, because entire GPU will be powered off + * when we return. + * For runtime PM suspend, in contrast to OS suspend, there is a race + * between this function and the mali_pm_update_sync_internal(), which + * is fine... + */ + ret = mali_pm_common_suspend(); + if (MALI_TRUE == ret) { + mali_pm_runtime_active = MALI_FALSE; + } else { + /* + * Process the "power up" instead, + * which could have been "lost" + */ + mali_pm_update_sync_internal(); + } + + mali_pm_exec_unlock(); + + return ret; +} + +void mali_pm_runtime_resume(void) +{ + struct mali_pmu_core *pmu = mali_pmu_get_global_pmu_core(); + + mali_pm_exec_lock(); + + mali_pm_runtime_active = MALI_TRUE; + +#if defined(DEBUG) + ++num_pm_runtime_resume; + + mali_pm_state_lock(); + + /* + * Assert that things are as we left them in runtime_suspend(), + * except for pd_mask_wanted which normally will be the reason we + * got here (job queued => domains wanted) + */ + MALI_DEBUG_ASSERT(0 == pd_mask_current); + MALI_DEBUG_ASSERT(0 == pmu_mask_current); + + mali_pm_state_unlock(); +#endif + + if (NULL != pmu) { + mali_pmu_reset(pmu); + pmu_mask_current = mali_pmu_get_mask(pmu); + MALI_DEBUG_PRINT(3, ("Mali PM: Runtime resume 0x%x \n", pmu_mask_current)); + } + + /* + * Normally we are resumed because a job has just been queued. + * pd_mask_wanted should thus be != 0. + * It is however possible for others to take a Mali Runtime PM ref + * without having a job queued. + * We should however always call mali_pm_update_sync_internal(), + * because this will take care of any potential mismatch between + * pmu_mask_current and pd_mask_current. + */ + mali_pm_update_sync_internal(); + + mali_pm_exec_unlock(); +} + +#if MALI_STATE_TRACKING +u32 mali_pm_dump_state_domain(struct mali_pm_domain *domain, + char *buf, u32 size) +{ + int n = 0; + + n += _mali_osk_snprintf(buf + n, size - n, + "\tPower domain: id %u\n", + mali_pm_domain_get_id(domain)); + + n += _mali_osk_snprintf(buf + n, size - n, + "\t\tMask: 0x%04x\n", + mali_pm_domain_get_mask(domain)); + + n += _mali_osk_snprintf(buf + n, size - n, + "\t\tUse count: %u\n", + mali_pm_domain_get_use_count(domain)); + + n += _mali_osk_snprintf(buf + n, size - n, + "\t\tCurrent power state: %s\n", + (mali_pm_domain_get_mask(domain) & pd_mask_current) ? + "On" : "Off"); + + n += _mali_osk_snprintf(buf + n, size - n, + "\t\tWanted power state: %s\n", + (mali_pm_domain_get_mask(domain) & pd_mask_wanted) ? + "On" : "Off"); + + return n; +} +#endif + +static void mali_pm_state_lock(void) +{ + _mali_osk_spinlock_irq_lock(pm_lock_state); +} + +static void mali_pm_state_unlock(void) +{ + _mali_osk_spinlock_irq_unlock(pm_lock_state); +} + +void mali_pm_exec_lock(void) +{ + _mali_osk_mutex_wait(pm_lock_exec); +} + +void mali_pm_exec_unlock(void) +{ + _mali_osk_mutex_signal(pm_lock_exec); +} + +static void mali_pm_domain_power_up(u32 power_up_mask, + struct mali_group *groups_up[MALI_MAX_NUMBER_OF_GROUPS], + u32 *num_groups_up, + struct mali_l2_cache_core *l2_up[MALI_MAX_NUMBER_OF_L2_CACHE_CORES], + u32 *num_l2_up) +{ + u32 domain_bit; + u32 notify_mask = power_up_mask; + + MALI_DEBUG_ASSERT(0 != power_up_mask); + MALI_DEBUG_ASSERT_POINTER(groups_up); + MALI_DEBUG_ASSERT_POINTER(num_groups_up); + MALI_DEBUG_ASSERT(0 == *num_groups_up); + MALI_DEBUG_ASSERT_POINTER(l2_up); + MALI_DEBUG_ASSERT_POINTER(num_l2_up); + MALI_DEBUG_ASSERT(0 == *num_l2_up); + + MALI_DEBUG_ASSERT_LOCK_HELD(pm_lock_exec); + MALI_DEBUG_ASSERT_LOCK_HELD(pm_lock_state); + + MALI_DEBUG_PRINT(5, + ("PM update: Powering up domains: . [%s]\n", + mali_pm_mask_to_string(power_up_mask))); + + pd_mask_current |= power_up_mask; + + domain_bit = _mali_osk_fls(notify_mask); + while (0 != domain_bit) { + u32 domain_id = domain_bit - 1; + struct mali_pm_domain *domain = + mali_pm_domain_get_from_index( + domain_id); + struct mali_l2_cache_core *l2_cache; + struct mali_l2_cache_core *l2_cache_tmp; + struct mali_group *group; + struct mali_group *group_tmp; + + /* Mark domain as powered up */ + mali_pm_domain_set_power_on(domain, MALI_TRUE); + + /* + * Make a note of the L2 and/or group(s) to notify + * (need to release the PM state lock before doing so) + */ + + _MALI_OSK_LIST_FOREACHENTRY(l2_cache, + l2_cache_tmp, + mali_pm_domain_get_l2_cache_list( + domain), + struct mali_l2_cache_core, + pm_domain_list) { + MALI_DEBUG_ASSERT(*num_l2_up < + MALI_MAX_NUMBER_OF_L2_CACHE_CORES); + l2_up[*num_l2_up] = l2_cache; + (*num_l2_up)++; + } + + _MALI_OSK_LIST_FOREACHENTRY(group, + group_tmp, + mali_pm_domain_get_group_list(domain), + struct mali_group, + pm_domain_list) { + MALI_DEBUG_ASSERT(*num_groups_up < + MALI_MAX_NUMBER_OF_GROUPS); + groups_up[*num_groups_up] = group; + + (*num_groups_up)++; + } + + /* Remove current bit and find next */ + notify_mask &= ~(1 << (domain_id)); + domain_bit = _mali_osk_fls(notify_mask); + } +} +static void mali_pm_domain_power_down(u32 power_down_mask, + struct mali_group *groups_down[MALI_MAX_NUMBER_OF_GROUPS], + u32 *num_groups_down, + struct mali_l2_cache_core *l2_down[MALI_MAX_NUMBER_OF_L2_CACHE_CORES], + u32 *num_l2_down) +{ + u32 domain_bit; + u32 notify_mask = power_down_mask; + + MALI_DEBUG_ASSERT(0 != power_down_mask); + MALI_DEBUG_ASSERT_POINTER(groups_down); + MALI_DEBUG_ASSERT_POINTER(num_groups_down); + MALI_DEBUG_ASSERT(0 == *num_groups_down); + MALI_DEBUG_ASSERT_POINTER(l2_down); + MALI_DEBUG_ASSERT_POINTER(num_l2_down); + MALI_DEBUG_ASSERT(0 == *num_l2_down); + + MALI_DEBUG_ASSERT_LOCK_HELD(pm_lock_exec); + MALI_DEBUG_ASSERT_LOCK_HELD(pm_lock_state); + + MALI_DEBUG_PRINT(5, + ("PM update: Powering down domains: [%s]\n", + mali_pm_mask_to_string(power_down_mask))); + + pd_mask_current &= ~power_down_mask; + + domain_bit = _mali_osk_fls(notify_mask); + while (0 != domain_bit) { + u32 domain_id = domain_bit - 1; + struct mali_pm_domain *domain = + mali_pm_domain_get_from_index(domain_id); + struct mali_l2_cache_core *l2_cache; + struct mali_l2_cache_core *l2_cache_tmp; + struct mali_group *group; + struct mali_group *group_tmp; + + /* Mark domain as powered down */ + mali_pm_domain_set_power_on(domain, MALI_FALSE); + + /* + * Make a note of the L2s and/or groups to notify + * (need to release the PM state lock before doing so) + */ + + _MALI_OSK_LIST_FOREACHENTRY(l2_cache, + l2_cache_tmp, + mali_pm_domain_get_l2_cache_list(domain), + struct mali_l2_cache_core, + pm_domain_list) { + MALI_DEBUG_ASSERT(*num_l2_down < + MALI_MAX_NUMBER_OF_L2_CACHE_CORES); + l2_down[*num_l2_down] = l2_cache; + (*num_l2_down)++; + } + + _MALI_OSK_LIST_FOREACHENTRY(group, + group_tmp, + mali_pm_domain_get_group_list(domain), + struct mali_group, + pm_domain_list) { + MALI_DEBUG_ASSERT(*num_groups_down < + MALI_MAX_NUMBER_OF_GROUPS); + groups_down[*num_groups_down] = group; + (*num_groups_down)++; + } + + /* Remove current bit and find next */ + notify_mask &= ~(1 << (domain_id)); + domain_bit = _mali_osk_fls(notify_mask); + } +} + +/* + * Execute pending power domain changes + * pm_lock_exec lock must be taken by caller. + */ +static void mali_pm_update_sync_internal(void) +{ + /* + * This should only be called in non-atomic context + * (normally as deferred work) + * + * Look at the pending power domain changes, and execute these. + * Make sure group and schedulers are notified about changes. + */ + + struct mali_pmu_core *pmu = mali_pmu_get_global_pmu_core(); + + u32 power_down_mask; + u32 power_up_mask; + + MALI_DEBUG_ASSERT_LOCK_HELD(pm_lock_exec); + +#if defined(DEBUG) + ++num_pm_updates; +#endif + + /* Hold PM state lock while we look at (and obey) the wanted state */ + mali_pm_state_lock(); + + MALI_DEBUG_PRINT(5, ("PM update pre: Wanted domain mask: .. [%s]\n", + mali_pm_mask_to_string(pd_mask_wanted))); + MALI_DEBUG_PRINT(5, ("PM update pre: Current domain mask: . [%s]\n", + mali_pm_mask_to_string(pd_mask_current))); + MALI_DEBUG_PRINT(5, ("PM update pre: Current PMU mask: .... [%s]\n", + mali_pm_mask_to_string(pmu_mask_current))); + MALI_DEBUG_PRINT(5, ("PM update pre: Group power stats: ... <%s>\n", + mali_pm_group_stats_to_string())); + + /* Figure out which cores we need to power on */ + power_up_mask = pd_mask_wanted & + (pd_mask_wanted ^ pd_mask_current); + + if (0 != power_up_mask) { + u32 power_up_mask_pmu; + struct mali_group *groups_up[MALI_MAX_NUMBER_OF_GROUPS]; + u32 num_groups_up = 0; + struct mali_l2_cache_core * + l2_up[MALI_MAX_NUMBER_OF_L2_CACHE_CORES]; + u32 num_l2_up = 0; + u32 i; + +#if defined(DEBUG) + ++num_pm_updates_up; +#endif + + /* + * Make sure dummy/global domain is always included when + * powering up, since this is controlled by runtime PM, + * and device power is on at this stage. + */ + power_up_mask |= MALI_PM_DOMAIN_DUMMY_MASK; + + /* Power up only real PMU domains */ + power_up_mask_pmu = power_up_mask & ~MALI_PM_DOMAIN_DUMMY_MASK; + + /* But not those that happen to be powered on already */ + power_up_mask_pmu &= (power_up_mask ^ pmu_mask_current) & + power_up_mask; + + if (0 != power_up_mask_pmu) { + MALI_DEBUG_ASSERT(NULL != pmu); + pmu_mask_current |= power_up_mask_pmu; + mali_pmu_power_up(pmu, power_up_mask_pmu); + } + + /* + * Put the domains themselves in power up state. + * We get the groups and L2s to notify in return. + */ + mali_pm_domain_power_up(power_up_mask, + groups_up, &num_groups_up, + l2_up, &num_l2_up); + + /* Need to unlock PM state lock before notifying L2 + groups */ + mali_pm_state_unlock(); + + /* Notify each L2 cache that we have be powered up */ + for (i = 0; i < num_l2_up; i++) { + mali_l2_cache_power_up(l2_up[i]); + } + + /* + * Tell execution module about all the groups we have + * powered up. Groups will be notified as a result of this. + */ + mali_executor_group_power_up(groups_up, num_groups_up); + + /* Lock state again before checking for power down */ + mali_pm_state_lock(); + } + + /* Figure out which cores we need to power off */ + power_down_mask = pd_mask_current & + (pd_mask_wanted ^ pd_mask_current); + + /* + * Never power down the dummy/global domain here. This is to be done + * from a suspend request (since this domain is only physicall powered + * down at that point) + */ + power_down_mask &= ~MALI_PM_DOMAIN_DUMMY_MASK; + + if (0 != power_down_mask) { + u32 power_down_mask_pmu; + struct mali_group *groups_down[MALI_MAX_NUMBER_OF_GROUPS]; + u32 num_groups_down = 0; + struct mali_l2_cache_core * + l2_down[MALI_MAX_NUMBER_OF_L2_CACHE_CORES]; + u32 num_l2_down = 0; + u32 i; + +#if defined(DEBUG) + ++num_pm_updates_down; +#endif + + /* + * Put the domains themselves in power down state. + * We get the groups and L2s to notify in return. + */ + mali_pm_domain_power_down(power_down_mask, + groups_down, &num_groups_down, + l2_down, &num_l2_down); + + /* Need to unlock PM state lock before notifying L2 + groups */ + mali_pm_state_unlock(); + + /* + * Tell execution module about all the groups we will be + * powering down. Groups will be notified as a result of this. + */ + if (0 < num_groups_down) { + mali_executor_group_power_down(groups_down, num_groups_down); + } + + /* Notify each L2 cache that we will be powering down */ + for (i = 0; i < num_l2_down; i++) { + mali_l2_cache_power_down(l2_down[i]); + } + + /* + * Power down only PMU domains which should not stay on + * Some domains might for instance currently be incorrectly + * powered up if default domain power state is all on. + */ + power_down_mask_pmu = pmu_mask_current & (~pd_mask_current); + + if (0 != power_down_mask_pmu) { + MALI_DEBUG_ASSERT(NULL != pmu); + pmu_mask_current &= ~power_down_mask_pmu; + mali_pmu_power_down(pmu, power_down_mask_pmu); + + } + } else { + /* + * Power down only PMU domains which should not stay on + * Some domains might for instance currently be incorrectly + * powered up if default domain power state is all on. + */ + u32 power_down_mask_pmu; + + /* No need for state lock since we'll only update PMU */ + mali_pm_state_unlock(); + + power_down_mask_pmu = pmu_mask_current & (~pd_mask_current); + + if (0 != power_down_mask_pmu) { + MALI_DEBUG_ASSERT(NULL != pmu); + pmu_mask_current &= ~power_down_mask_pmu; + mali_pmu_power_down(pmu, power_down_mask_pmu); + } + } + + MALI_DEBUG_PRINT(5, ("PM update post: Current domain mask: . [%s]\n", + mali_pm_mask_to_string(pd_mask_current))); + MALI_DEBUG_PRINT(5, ("PM update post: Current PMU mask: .... [%s]\n", + mali_pm_mask_to_string(pmu_mask_current))); + MALI_DEBUG_PRINT(5, ("PM update post: Group power stats: ... <%s>\n", + mali_pm_group_stats_to_string())); +} + +static mali_bool mali_pm_common_suspend(void) +{ + mali_pm_state_lock(); + + if (0 != pd_mask_wanted) { + MALI_DEBUG_PRINT(5, ("PM: Aborting suspend operation\n\n\n")); + mali_pm_state_unlock(); + return MALI_FALSE; + } + + MALI_DEBUG_PRINT(5, ("PM suspend pre: Wanted domain mask: .. [%s]\n", + mali_pm_mask_to_string(pd_mask_wanted))); + MALI_DEBUG_PRINT(5, ("PM suspend pre: Current domain mask: . [%s]\n", + mali_pm_mask_to_string(pd_mask_current))); + MALI_DEBUG_PRINT(5, ("PM suspend pre: Current PMU mask: .... [%s]\n", + mali_pm_mask_to_string(pmu_mask_current))); + MALI_DEBUG_PRINT(5, ("PM suspend pre: Group power stats: ... <%s>\n", + mali_pm_group_stats_to_string())); + + if (0 != pd_mask_current) { + /* + * We have still some domains powered on. + * It is for instance very normal that at least the + * dummy/global domain is marked as powered on at this point. + * (because it is physically powered on until this function + * returns) + */ + + struct mali_group *groups_down[MALI_MAX_NUMBER_OF_GROUPS]; + u32 num_groups_down = 0; + struct mali_l2_cache_core * + l2_down[MALI_MAX_NUMBER_OF_L2_CACHE_CORES]; + u32 num_l2_down = 0; + u32 i; + + /* + * Put the domains themselves in power down state. + * We get the groups and L2s to notify in return. + */ + mali_pm_domain_power_down(pd_mask_current, + groups_down, + &num_groups_down, + l2_down, + &num_l2_down); + + MALI_DEBUG_ASSERT(0 == pd_mask_current); + MALI_DEBUG_ASSERT(MALI_TRUE == mali_pm_domain_all_unused()); + + /* Need to unlock PM state lock before notifying L2 + groups */ + mali_pm_state_unlock(); + + /* + * Tell execution module about all the groups we will be + * powering down. Groups will be notified as a result of this. + */ + if (0 < num_groups_down) { + mali_executor_group_power_down(groups_down, num_groups_down); + } + + /* Notify each L2 cache that we will be powering down */ + for (i = 0; i < num_l2_down; i++) { + mali_l2_cache_power_down(l2_down[i]); + } + + pmu_mask_current = 0; + } else { + MALI_DEBUG_ASSERT(0 == pmu_mask_current); + + MALI_DEBUG_ASSERT(MALI_TRUE == mali_pm_domain_all_unused()); + + mali_pm_state_unlock(); + } + + MALI_DEBUG_PRINT(5, ("PM suspend post: Current domain mask: [%s]\n", + mali_pm_mask_to_string(pd_mask_current))); + MALI_DEBUG_PRINT(5, ("PM suspend post: Current PMU mask: ... [%s]\n", + mali_pm_mask_to_string(pmu_mask_current))); + MALI_DEBUG_PRINT(5, ("PM suspend post: Group power stats: .. <%s>\n", + mali_pm_group_stats_to_string())); + + return MALI_TRUE; +} + +static void mali_pm_update_work(void *data) +{ + MALI_IGNORE(data); + mali_pm_update_sync(); +} + +static _mali_osk_errcode_t mali_pm_create_pm_domains(void) +{ + int i; + + /* Create all domains (including dummy domain) */ + for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) { + if (0x0 == domain_config[i]) continue; + + if (NULL == mali_pm_domain_create(domain_config[i])) { + return _MALI_OSK_ERR_NOMEM; + } + } + + return _MALI_OSK_ERR_OK; +} + +static void mali_pm_set_default_pm_domain_config(void) +{ + MALI_DEBUG_ASSERT(0 != _mali_osk_resource_base_address()); + + /* GP core */ + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( + MALI_OFFSET_GP, NULL)) { + domain_config[MALI_DOMAIN_INDEX_GP] = 0x01; + } + + /* PP0 - PP3 core */ + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( + MALI_OFFSET_PP0, NULL)) { + if (mali_is_mali400()) { + domain_config[MALI_DOMAIN_INDEX_PP0] = 0x01 << 2; + } else if (mali_is_mali450()) { + domain_config[MALI_DOMAIN_INDEX_PP0] = 0x01 << 1; + } else if (mali_is_mali470()) { + domain_config[MALI_DOMAIN_INDEX_PP0] = 0x01 << 0; + } + } + + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( + MALI_OFFSET_PP1, NULL)) { + if (mali_is_mali400()) { + domain_config[MALI_DOMAIN_INDEX_PP1] = 0x01 << 3; + } else if (mali_is_mali450()) { + domain_config[MALI_DOMAIN_INDEX_PP1] = 0x01 << 2; + } else if (mali_is_mali470()) { + domain_config[MALI_DOMAIN_INDEX_PP1] = 0x01 << 1; + } + } + + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( + MALI_OFFSET_PP2, NULL)) { + if (mali_is_mali400()) { + domain_config[MALI_DOMAIN_INDEX_PP2] = 0x01 << 4; + } else if (mali_is_mali450()) { + domain_config[MALI_DOMAIN_INDEX_PP2] = 0x01 << 2; + } else if (mali_is_mali470()) { + domain_config[MALI_DOMAIN_INDEX_PP2] = 0x01 << 1; + } + } + + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( + MALI_OFFSET_PP3, NULL)) { + if (mali_is_mali400()) { + domain_config[MALI_DOMAIN_INDEX_PP3] = 0x01 << 5; + } else if (mali_is_mali450()) { + domain_config[MALI_DOMAIN_INDEX_PP3] = 0x01 << 2; + } else if (mali_is_mali470()) { + domain_config[MALI_DOMAIN_INDEX_PP3] = 0x01 << 1; + } + } + + /* PP4 - PP7 */ + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( + MALI_OFFSET_PP4, NULL)) { + domain_config[MALI_DOMAIN_INDEX_PP4] = 0x01 << 3; + } + + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( + MALI_OFFSET_PP5, NULL)) { + domain_config[MALI_DOMAIN_INDEX_PP5] = 0x01 << 3; + } + + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( + MALI_OFFSET_PP6, NULL)) { + domain_config[MALI_DOMAIN_INDEX_PP6] = 0x01 << 3; + } + + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( + MALI_OFFSET_PP7, NULL)) { + domain_config[MALI_DOMAIN_INDEX_PP7] = 0x01 << 3; + } + + /* L2gp/L2PP0/L2PP4 */ + if (mali_is_mali400()) { + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( + MALI400_OFFSET_L2_CACHE0, NULL)) { + domain_config[MALI_DOMAIN_INDEX_L20] = 0x01 << 1; + } + } else if (mali_is_mali450()) { + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( + MALI450_OFFSET_L2_CACHE0, NULL)) { + domain_config[MALI_DOMAIN_INDEX_L20] = 0x01 << 0; + } + + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( + MALI450_OFFSET_L2_CACHE1, NULL)) { + domain_config[MALI_DOMAIN_INDEX_L21] = 0x01 << 1; + } + + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( + MALI450_OFFSET_L2_CACHE2, NULL)) { + domain_config[MALI_DOMAIN_INDEX_L22] = 0x01 << 3; + } + } else if (mali_is_mali470()) { + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( + MALI470_OFFSET_L2_CACHE1, NULL)) { + domain_config[MALI_DOMAIN_INDEX_L21] = 0x01 << 0; + } + } +} + +static u32 mali_pm_get_registered_cores_mask(void) +{ + int i = 0; + u32 mask = 0; + + for (i = 0; i < MALI_DOMAIN_INDEX_DUMMY; i++) { + mask |= domain_config[i]; + } + + return mask; +} + +static void mali_pm_set_pmu_domain_config(void) +{ + int i = 0; + + _mali_osk_device_data_pmu_config_get(domain_config, MALI_MAX_NUMBER_OF_DOMAINS - 1); + + for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS - 1; i++) { + if (0 != domain_config[i]) { + MALI_DEBUG_PRINT(2, ("Using customer pmu config:\n")); + break; + } + } + + if (MALI_MAX_NUMBER_OF_DOMAINS - 1 == i) { + MALI_DEBUG_PRINT(2, ("Using hw detect pmu config:\n")); + mali_pm_set_default_pm_domain_config(); + } + + for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS - 1; i++) { + if (domain_config[i]) { + MALI_DEBUG_PRINT(2, ("domain_config[%d] = 0x%x \n", i, domain_config[i])); + } + } + /* Can't override dummy domain mask */ + domain_config[MALI_DOMAIN_INDEX_DUMMY] = + 1 << MALI_DOMAIN_INDEX_DUMMY; +} + +#if defined(DEBUG) +const char *mali_pm_mask_to_string(u32 mask) +{ + static char bit_str[MALI_MAX_NUMBER_OF_DOMAINS + 1]; + int bit; + int str_pos = 0; + + /* Must be protected by lock since we use shared string buffer */ + if (NULL != pm_lock_exec) { + MALI_DEBUG_ASSERT_LOCK_HELD(pm_lock_exec); + } + + for (bit = MALI_MAX_NUMBER_OF_DOMAINS - 1; bit >= 0; bit--) { + if (mask & (1 << bit)) { + bit_str[str_pos] = 'X'; + } else { + bit_str[str_pos] = '-'; + } + str_pos++; + } + + bit_str[MALI_MAX_NUMBER_OF_DOMAINS] = '\0'; + + return bit_str; +} + +const char *mali_pm_group_stats_to_string(void) +{ + static char bit_str[MALI_MAX_NUMBER_OF_GROUPS + 1]; + u32 num_groups = mali_group_get_glob_num_groups(); + u32 i; + + /* Must be protected by lock since we use shared string buffer */ + if (NULL != pm_lock_exec) { + MALI_DEBUG_ASSERT_LOCK_HELD(pm_lock_exec); + } + + for (i = 0; i < num_groups && i < MALI_MAX_NUMBER_OF_GROUPS; i++) { + struct mali_group *group; + + group = mali_group_get_glob_group(i); + + if (MALI_TRUE == mali_group_power_is_on(group)) { + bit_str[i] = 'X'; + } else { + bit_str[i] = '-'; + } + } + + bit_str[i] = '\0'; + + return bit_str; +} +#endif + +/* + * num_pp is the number of PP cores which will be powered on given this mask + * cost is the total power cost of cores which will be powered on given this mask + */ +static void mali_pm_stat_from_mask(u32 mask, u32 *num_pp, u32 *cost) +{ + u32 i; + + /* loop through all cores */ + for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) { + if (!(domain_config[i] & mask)) { + continue; + } + + switch (i) { + case MALI_DOMAIN_INDEX_GP: + *cost += MALI_GP_COST; + + break; + case MALI_DOMAIN_INDEX_PP0: /* Fall through */ + case MALI_DOMAIN_INDEX_PP1: /* Fall through */ + case MALI_DOMAIN_INDEX_PP2: /* Fall through */ + case MALI_DOMAIN_INDEX_PP3: + if (mali_is_mali400()) { + if ((domain_config[MALI_DOMAIN_INDEX_L20] & mask) + || (domain_config[MALI_DOMAIN_INDEX_DUMMY] + == domain_config[MALI_DOMAIN_INDEX_L20])) { + *num_pp += 1; + } + } else { + if ((domain_config[MALI_DOMAIN_INDEX_L21] & mask) + || (domain_config[MALI_DOMAIN_INDEX_DUMMY] + == domain_config[MALI_DOMAIN_INDEX_L21])) { + *num_pp += 1; + } + } + + *cost += MALI_PP_COST; + break; + case MALI_DOMAIN_INDEX_PP4: /* Fall through */ + case MALI_DOMAIN_INDEX_PP5: /* Fall through */ + case MALI_DOMAIN_INDEX_PP6: /* Fall through */ + case MALI_DOMAIN_INDEX_PP7: + MALI_DEBUG_ASSERT(mali_is_mali450()); + + if ((domain_config[MALI_DOMAIN_INDEX_L22] & mask) + || (domain_config[MALI_DOMAIN_INDEX_DUMMY] + == domain_config[MALI_DOMAIN_INDEX_L22])) { + *num_pp += 1; + } + + *cost += MALI_PP_COST; + break; + case MALI_DOMAIN_INDEX_L20: /* Fall through */ + case MALI_DOMAIN_INDEX_L21: /* Fall through */ + case MALI_DOMAIN_INDEX_L22: + *cost += MALI_L2_COST; + + break; + } + } +} + +void mali_pm_power_cost_setup(void) +{ + /* + * Two parallel arrays which store the best domain mask and its cost + * The index is the number of PP cores, E.g. Index 0 is for 1 PP option, + * might have mask 0x2 and with cost of 1, lower cost is better + */ + u32 best_mask[MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS] = { 0 }; + u32 best_cost[MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS] = { 0 }; + /* Array cores_in_domain is used to store the total pp cores in each pm domain. */ + u32 cores_in_domain[MALI_MAX_NUMBER_OF_DOMAINS] = { 0 }; + /* Domain_count is used to represent the max domain we have.*/ + u32 max_domain_mask = 0; + u32 max_domain_id = 0; + u32 always_on_pp_cores = 0; + + u32 num_pp, cost, mask; + u32 i, j , k; + + /* Initialize statistics */ + for (i = 0; i < MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS; i++) { + best_mask[i] = 0; + best_cost[i] = 0xFFFFFFFF; /* lower cost is better */ + } + + for (i = 0; i < MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS + 1; i++) { + for (j = 0; j < MALI_MAX_NUMBER_OF_DOMAINS; j++) { + mali_pm_domain_power_cost_result[i][j] = 0; + } + } + + /* Caculate number of pp cores of a given domain config. */ + for (i = MALI_DOMAIN_INDEX_PP0; i <= MALI_DOMAIN_INDEX_PP7; i++) { + if (0 < domain_config[i]) { + /* Get the max domain mask value used to caculate power cost + * and we don't count in always on pp cores. */ + if (MALI_PM_DOMAIN_DUMMY_MASK != domain_config[i] + && max_domain_mask < domain_config[i]) { + max_domain_mask = domain_config[i]; + } + + if (MALI_PM_DOMAIN_DUMMY_MASK == domain_config[i]) { + always_on_pp_cores++; + } + } + } + max_domain_id = _mali_osk_fls(max_domain_mask); + + /* + * Try all combinations of power domains and check how many PP cores + * they have and their power cost. + */ + for (mask = 0; mask < (1 << max_domain_id); mask++) { + num_pp = 0; + cost = 0; + + mali_pm_stat_from_mask(mask, &num_pp, &cost); + + /* This mask is usable for all MP1 up to num_pp PP cores, check statistics for all */ + for (i = 0; i < num_pp; i++) { + if (best_cost[i] >= cost) { + best_cost[i] = cost; + best_mask[i] = mask; + } + } + } + + /* + * If we want to enable x pp cores, if x is less than number of always_on pp cores, + * all of pp cores we will enable must be always_on pp cores. + */ + for (i = 0; i < mali_executor_get_num_cores_total(); i++) { + if (i < always_on_pp_cores) { + mali_pm_domain_power_cost_result[i + 1][MALI_MAX_NUMBER_OF_DOMAINS - 1] + = i + 1; + } else { + mali_pm_domain_power_cost_result[i + 1][MALI_MAX_NUMBER_OF_DOMAINS - 1] + = always_on_pp_cores; + } + } + + /* In this loop, variable i represent for the number of non-always on pp cores we want to enabled. */ + for (i = 0; i < (mali_executor_get_num_cores_total() - always_on_pp_cores); i++) { + if (best_mask[i] == 0) { + /* This MP variant is not available */ + continue; + } + + for (j = 0; j < MALI_MAX_NUMBER_OF_DOMAINS; j++) { + cores_in_domain[j] = 0; + } + + for (j = MALI_DOMAIN_INDEX_PP0; j <= MALI_DOMAIN_INDEX_PP7; j++) { + if (0 < domain_config[j] + && (MALI_PM_DOMAIN_DUMMY_MASK != domain_config[i])) { + cores_in_domain[_mali_osk_fls(domain_config[j]) - 1]++; + } + } + + /* In this loop, j represent for the number we have already enabled.*/ + for (j = 0; j <= i;) { + /* j used to visit all of domain to get the number of pp cores remained in it. */ + for (k = 0; k < max_domain_id; k++) { + /* If domain k in best_mask[i] is enabled and this domain has extra pp cores, + * we know we must pick at least one pp core from this domain. + * And then we move to next enabled pm domain. */ + if ((best_mask[i] & (0x1 << k)) && (0 < cores_in_domain[k])) { + cores_in_domain[k]--; + mali_pm_domain_power_cost_result[always_on_pp_cores + i + 1][k]++; + j++; + if (j > i) { + break; + } + } + } + } + } +} + +/* + * When we are doing core scaling, + * this function is called to return the best mask to + * achieve the best pp group power cost. + */ +void mali_pm_get_best_power_cost_mask(int num_requested, int *dst) +{ + MALI_DEBUG_ASSERT((mali_executor_get_num_cores_total() >= num_requested) && (0 <= num_requested)); + + _mali_osk_memcpy(dst, mali_pm_domain_power_cost_result[num_requested], MALI_MAX_NUMBER_OF_DOMAINS * sizeof(int)); +} + +u32 mali_pm_get_current_mask(void) +{ + return pd_mask_current; +} + +u32 mali_pm_get_wanted_mask(void) +{ + return pd_mask_wanted; +} diff --git a/drivers/gpu/arm/utgard/common/mali_pm.h b/drivers/gpu/arm/utgard/common/mali_pm.h new file mode 100644 index 00000000000000..d72c732e698d65 --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_pm.h @@ -0,0 +1,91 @@ +/* + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_PM_H__ +#define __MALI_PM_H__ + +#include "mali_osk.h" +#include "mali_pm_domain.h" + +#define MALI_DOMAIN_INDEX_GP 0 +#define MALI_DOMAIN_INDEX_PP0 1 +#define MALI_DOMAIN_INDEX_PP1 2 +#define MALI_DOMAIN_INDEX_PP2 3 +#define MALI_DOMAIN_INDEX_PP3 4 +#define MALI_DOMAIN_INDEX_PP4 5 +#define MALI_DOMAIN_INDEX_PP5 6 +#define MALI_DOMAIN_INDEX_PP6 7 +#define MALI_DOMAIN_INDEX_PP7 8 +#define MALI_DOMAIN_INDEX_L20 9 +#define MALI_DOMAIN_INDEX_L21 10 +#define MALI_DOMAIN_INDEX_L22 11 +/* + * The dummy domain is used when there is no physical power domain + * (e.g. no PMU or always on cores) + */ +#define MALI_DOMAIN_INDEX_DUMMY 12 +#define MALI_MAX_NUMBER_OF_DOMAINS 13 + +/** + * Initialize the Mali PM module + * + * PM module covers Mali PM core, PM domains and Mali PMU + */ +_mali_osk_errcode_t mali_pm_initialize(void); + +/** + * Terminate the Mali PM module + */ +void mali_pm_terminate(void); + +void mali_pm_exec_lock(void); +void mali_pm_exec_unlock(void); + + +struct mali_pm_domain *mali_pm_register_l2_cache(u32 domain_index, + struct mali_l2_cache_core *l2_cache); +struct mali_pm_domain *mali_pm_register_group(u32 domain_index, + struct mali_group *group); + +mali_bool mali_pm_get_domain_refs(struct mali_pm_domain **domains, + struct mali_group **groups, + u32 num_domains); +mali_bool mali_pm_put_domain_refs(struct mali_pm_domain **domains, + u32 num_domains); + +void mali_pm_init_begin(void); +void mali_pm_init_end(void); + +void mali_pm_update_sync(void); +void mali_pm_update_async(void); + +/* Callback functions for system power management */ +void mali_pm_os_suspend(mali_bool os_suspend); +void mali_pm_os_resume(void); + +mali_bool mali_pm_runtime_suspend(void); +void mali_pm_runtime_resume(void); + +#if MALI_STATE_TRACKING +u32 mali_pm_dump_state_domain(struct mali_pm_domain *domain, + char *buf, u32 size); +#endif + +void mali_pm_power_cost_setup(void); + +void mali_pm_get_best_power_cost_mask(int num_requested, int *dst); + +#if defined(DEBUG) +const char *mali_pm_mask_to_string(u32 mask); +#endif + +u32 mali_pm_get_current_mask(void); +u32 mali_pm_get_wanted_mask(void); +#endif /* __MALI_PM_H__ */ diff --git a/drivers/gpu/arm/utgard/common/mali_pm_domain.c b/drivers/gpu/arm/utgard/common/mali_pm_domain.c new file mode 100644 index 00000000000000..dbf985e6d37b0a --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_pm_domain.c @@ -0,0 +1,209 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_kernel_common.h" +#include "mali_osk.h" +#include "mali_pm_domain.h" +#include "mali_pmu.h" +#include "mali_group.h" +#include "mali_pm.h" + +static struct mali_pm_domain *mali_pm_domains[MALI_MAX_NUMBER_OF_DOMAINS] = +{ NULL, }; + +void mali_pm_domain_initialize(void) +{ + /* Domains will be initialized/created on demand */ +} + +void mali_pm_domain_terminate(void) +{ + int i; + + /* Delete all domains that has been created */ + for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) { + mali_pm_domain_delete(mali_pm_domains[i]); + mali_pm_domains[i] = NULL; + } +} + +struct mali_pm_domain *mali_pm_domain_create(u32 pmu_mask) +{ + struct mali_pm_domain *domain = NULL; + u32 domain_id = 0; + + domain = mali_pm_domain_get_from_mask(pmu_mask); + if (NULL != domain) return domain; + + MALI_DEBUG_PRINT(2, + ("Mali PM domain: Creating Mali PM domain (mask=0x%08X)\n", + pmu_mask)); + + domain = (struct mali_pm_domain *)_mali_osk_malloc( + sizeof(struct mali_pm_domain)); + if (NULL != domain) { + domain->power_is_on = MALI_FALSE; + domain->pmu_mask = pmu_mask; + domain->use_count = 0; + _mali_osk_list_init(&domain->group_list); + _mali_osk_list_init(&domain->l2_cache_list); + + domain_id = _mali_osk_fls(pmu_mask) - 1; + /* Verify the domain_id */ + MALI_DEBUG_ASSERT(MALI_MAX_NUMBER_OF_DOMAINS > domain_id); + /* Verify that pmu_mask only one bit is set */ + MALI_DEBUG_ASSERT((1 << domain_id) == pmu_mask); + mali_pm_domains[domain_id] = domain; + + return domain; + } else { + MALI_DEBUG_PRINT_ERROR(("Unable to create PM domain\n")); + } + + return NULL; +} + +void mali_pm_domain_delete(struct mali_pm_domain *domain) +{ + if (NULL == domain) { + return; + } + + _mali_osk_list_delinit(&domain->group_list); + _mali_osk_list_delinit(&domain->l2_cache_list); + + _mali_osk_free(domain); +} + +void mali_pm_domain_add_group(struct mali_pm_domain *domain, + struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(domain); + MALI_DEBUG_ASSERT_POINTER(group); + + /* + * Use addtail because virtual group is created last and it needs + * to be at the end of the list (in order to be activated after + * all children. + */ + _mali_osk_list_addtail(&group->pm_domain_list, &domain->group_list); +} + +void mali_pm_domain_add_l2_cache(struct mali_pm_domain *domain, + struct mali_l2_cache_core *l2_cache) +{ + MALI_DEBUG_ASSERT_POINTER(domain); + MALI_DEBUG_ASSERT_POINTER(l2_cache); + _mali_osk_list_add(&l2_cache->pm_domain_list, &domain->l2_cache_list); +} + +struct mali_pm_domain *mali_pm_domain_get_from_mask(u32 mask) +{ + u32 id = 0; + + if (0 == mask) { + return NULL; + } + + id = _mali_osk_fls(mask) - 1; + + MALI_DEBUG_ASSERT(MALI_MAX_NUMBER_OF_DOMAINS > id); + /* Verify that pmu_mask only one bit is set */ + MALI_DEBUG_ASSERT((1 << id) == mask); + + return mali_pm_domains[id]; +} + +struct mali_pm_domain *mali_pm_domain_get_from_index(u32 id) +{ + MALI_DEBUG_ASSERT(MALI_MAX_NUMBER_OF_DOMAINS > id); + + return mali_pm_domains[id]; +} + +u32 mali_pm_domain_ref_get(struct mali_pm_domain *domain) +{ + MALI_DEBUG_ASSERT_POINTER(domain); + + if (0 == domain->use_count) { + _mali_osk_pm_dev_ref_get_async(); + } + + ++domain->use_count; + MALI_DEBUG_PRINT(4, ("PM domain %p: ref_get, use_count => %u\n", domain, domain->use_count)); + + /* Return our mask so caller can check this against wanted mask */ + return domain->pmu_mask; +} + +u32 mali_pm_domain_ref_put(struct mali_pm_domain *domain) +{ + MALI_DEBUG_ASSERT_POINTER(domain); + + --domain->use_count; + MALI_DEBUG_PRINT(4, ("PM domain %p: ref_put, use_count => %u\n", domain, domain->use_count)); + + if (0 == domain->use_count) { + _mali_osk_pm_dev_ref_put(); + } + + /* + * Return the PMU mask which now could be be powered down + * (the bit for this domain). + * This is the responsibility of the caller (mali_pm) + */ + return (0 == domain->use_count ? domain->pmu_mask : 0); +} + +#if MALI_STATE_TRACKING +u32 mali_pm_domain_get_id(struct mali_pm_domain *domain) +{ + u32 id = 0; + + MALI_DEBUG_ASSERT_POINTER(domain); + MALI_DEBUG_ASSERT(0 != domain->pmu_mask); + + id = _mali_osk_fls(domain->pmu_mask) - 1; + + MALI_DEBUG_ASSERT(MALI_MAX_NUMBER_OF_DOMAINS > id); + /* Verify that pmu_mask only one bit is set */ + MALI_DEBUG_ASSERT((1 << id) == domain->pmu_mask); + /* Verify that we have stored the domain at right id/index */ + MALI_DEBUG_ASSERT(domain == mali_pm_domains[id]); + + return id; +} +#endif + +#if defined(DEBUG) +mali_bool mali_pm_domain_all_unused(void) +{ + int i; + + for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) { + if (NULL == mali_pm_domains[i]) { + /* Nothing to check */ + continue; + } + + if (MALI_TRUE == mali_pm_domains[i]->power_is_on) { + /* Not ready for suspend! */ + return MALI_FALSE; + } + + if (0 != mali_pm_domains[i]->use_count) { + /* Not ready for suspend! */ + return MALI_FALSE; + } + } + + return MALI_TRUE; +} +#endif diff --git a/drivers/gpu/arm/utgard/common/mali_pm_domain.h b/drivers/gpu/arm/utgard/common/mali_pm_domain.h new file mode 100644 index 00000000000000..aceb3449359af6 --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_pm_domain.h @@ -0,0 +1,104 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_PM_DOMAIN_H__ +#define __MALI_PM_DOMAIN_H__ + +#include "mali_kernel_common.h" +#include "mali_osk.h" + +#include "mali_l2_cache.h" +#include "mali_group.h" +#include "mali_pmu.h" + +/* Instances are protected by PM state lock */ +struct mali_pm_domain { + mali_bool power_is_on; + s32 use_count; + u32 pmu_mask; + + /* Zero or more groups can belong to this domain */ + _mali_osk_list_t group_list; + + /* Zero or more L2 caches can belong to this domain */ + _mali_osk_list_t l2_cache_list; +}; + + +void mali_pm_domain_initialize(void); +void mali_pm_domain_terminate(void); + +struct mali_pm_domain *mali_pm_domain_create(u32 pmu_mask); +void mali_pm_domain_delete(struct mali_pm_domain *domain); + +void mali_pm_domain_add_l2_cache( + struct mali_pm_domain *domain, + struct mali_l2_cache_core *l2_cache); +void mali_pm_domain_add_group(struct mali_pm_domain *domain, + struct mali_group *group); + +struct mali_pm_domain *mali_pm_domain_get_from_mask(u32 mask); +struct mali_pm_domain *mali_pm_domain_get_from_index(u32 id); + +/* Ref counting */ +u32 mali_pm_domain_ref_get(struct mali_pm_domain *domain); +u32 mali_pm_domain_ref_put(struct mali_pm_domain *domain); + +MALI_STATIC_INLINE _mali_osk_list_t *mali_pm_domain_get_group_list( + struct mali_pm_domain *domain) +{ + MALI_DEBUG_ASSERT_POINTER(domain); + return &domain->group_list; +} + +MALI_STATIC_INLINE _mali_osk_list_t *mali_pm_domain_get_l2_cache_list( + struct mali_pm_domain *domain) +{ + MALI_DEBUG_ASSERT_POINTER(domain); + return &domain->l2_cache_list; +} + +MALI_STATIC_INLINE mali_bool mali_pm_domain_power_is_on( + struct mali_pm_domain *domain) +{ + MALI_DEBUG_ASSERT_POINTER(domain); + return domain->power_is_on; +} + +MALI_STATIC_INLINE void mali_pm_domain_set_power_on( + struct mali_pm_domain *domain, + mali_bool power_is_on) +{ + MALI_DEBUG_ASSERT_POINTER(domain); + domain->power_is_on = power_is_on; +} + +MALI_STATIC_INLINE u32 mali_pm_domain_get_use_count( + struct mali_pm_domain *domain) +{ + MALI_DEBUG_ASSERT_POINTER(domain); + return domain->use_count; +} + +#if MALI_STATE_TRACKING +u32 mali_pm_domain_get_id(struct mali_pm_domain *domain); + +MALI_STATIC_INLINE u32 mali_pm_domain_get_mask(struct mali_pm_domain *domain) +{ + MALI_DEBUG_ASSERT_POINTER(domain); + return domain->pmu_mask; +} +#endif + +#if defined(DEBUG) +mali_bool mali_pm_domain_all_unused(void); +#endif + +#endif /* __MALI_PM_DOMAIN_H__ */ diff --git a/drivers/gpu/arm/utgard/common/mali_pmu.c b/drivers/gpu/arm/utgard/common/mali_pmu.c new file mode 100644 index 00000000000000..2a3008a6dd83ec --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_pmu.c @@ -0,0 +1,270 @@ +/* + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_pmu.c + * Mali driver functions for Mali 400 PMU hardware + */ +#include "mali_hw_core.h" +#include "mali_pmu.h" +#include "mali_pp.h" +#include "mali_kernel_common.h" +#include "mali_osk.h" +#include "mali_pm.h" +#include "mali_osk_mali.h" + +struct mali_pmu_core *mali_global_pmu_core = NULL; + +static _mali_osk_errcode_t mali_pmu_wait_for_command_finish( + struct mali_pmu_core *pmu); + +struct mali_pmu_core *mali_pmu_create(_mali_osk_resource_t *resource) +{ + struct mali_pmu_core *pmu; + + MALI_DEBUG_ASSERT(NULL == mali_global_pmu_core); + MALI_DEBUG_PRINT(2, ("Mali PMU: Creating Mali PMU core\n")); + + pmu = (struct mali_pmu_core *)_mali_osk_malloc( + sizeof(struct mali_pmu_core)); + if (NULL != pmu) { + pmu->registered_cores_mask = 0; /* to be set later */ + + if (_MALI_OSK_ERR_OK == mali_hw_core_create(&pmu->hw_core, + resource, PMU_REGISTER_ADDRESS_SPACE_SIZE)) { + + pmu->switch_delay = _mali_osk_get_pmu_switch_delay(); + + mali_global_pmu_core = pmu; + + return pmu; + } + _mali_osk_free(pmu); + } + + return NULL; +} + +void mali_pmu_delete(struct mali_pmu_core *pmu) +{ + MALI_DEBUG_ASSERT_POINTER(pmu); + MALI_DEBUG_ASSERT(pmu == mali_global_pmu_core); + + MALI_DEBUG_PRINT(2, ("Mali PMU: Deleting Mali PMU core\n")); + + mali_global_pmu_core = NULL; + + mali_hw_core_delete(&pmu->hw_core); + _mali_osk_free(pmu); +} + +void mali_pmu_set_registered_cores_mask(struct mali_pmu_core *pmu, u32 mask) +{ + pmu->registered_cores_mask = mask; +} + +void mali_pmu_reset(struct mali_pmu_core *pmu) +{ + MALI_DEBUG_ASSERT_POINTER(pmu); + MALI_DEBUG_ASSERT(pmu->registered_cores_mask != 0); + + /* Setup the desired defaults */ + mali_hw_core_register_write_relaxed(&pmu->hw_core, + PMU_REG_ADDR_MGMT_INT_MASK, 0); + mali_hw_core_register_write_relaxed(&pmu->hw_core, + PMU_REG_ADDR_MGMT_SW_DELAY, pmu->switch_delay); +} + +void mali_pmu_power_up_all(struct mali_pmu_core *pmu) +{ + u32 stat; + + MALI_DEBUG_ASSERT_POINTER(pmu); + MALI_DEBUG_ASSERT(pmu->registered_cores_mask != 0); + + mali_pm_exec_lock(); + + mali_pmu_reset(pmu); + + /* Now simply power up the domains which are marked as powered down */ + stat = mali_hw_core_register_read(&pmu->hw_core, + PMU_REG_ADDR_MGMT_STATUS); + mali_pmu_power_up(pmu, stat); + + mali_pm_exec_unlock(); +} + +void mali_pmu_power_down_all(struct mali_pmu_core *pmu) +{ + u32 stat; + + MALI_DEBUG_ASSERT_POINTER(pmu); + MALI_DEBUG_ASSERT(pmu->registered_cores_mask != 0); + + mali_pm_exec_lock(); + + /* Now simply power down the domains which are marked as powered up */ + stat = mali_hw_core_register_read(&pmu->hw_core, + PMU_REG_ADDR_MGMT_STATUS); + mali_pmu_power_down(pmu, (~stat) & pmu->registered_cores_mask); + + mali_pm_exec_unlock(); +} + +_mali_osk_errcode_t mali_pmu_power_down(struct mali_pmu_core *pmu, u32 mask) +{ + u32 stat; + _mali_osk_errcode_t err; + + MALI_DEBUG_ASSERT_POINTER(pmu); + MALI_DEBUG_ASSERT(pmu->registered_cores_mask != 0); + MALI_DEBUG_ASSERT(mask <= pmu->registered_cores_mask); + MALI_DEBUG_ASSERT(0 == (mali_hw_core_register_read(&pmu->hw_core, + PMU_REG_ADDR_MGMT_INT_RAWSTAT) & + PMU_REG_VAL_IRQ)); + + MALI_DEBUG_PRINT(3, + ("PMU power down: ...................... [%s]\n", + mali_pm_mask_to_string(mask))); + + stat = mali_hw_core_register_read(&pmu->hw_core, + PMU_REG_ADDR_MGMT_STATUS); + + /* + * Assert that we are not powering down domains which are already + * powered down. + */ + MALI_DEBUG_ASSERT(0 == (stat & mask)); + + mask &= ~(0x1 << MALI_DOMAIN_INDEX_DUMMY); + + if (0 == mask || 0 == ((~stat) & mask)) return _MALI_OSK_ERR_OK; + + mali_hw_core_register_write(&pmu->hw_core, + PMU_REG_ADDR_MGMT_POWER_DOWN, mask); + + /* + * Do not wait for interrupt on Mali-300/400 if all domains are + * powered off by our power down command, because the HW will simply + * not generate an interrupt in this case. + */ + if (mali_is_mali450() || mali_is_mali470() || pmu->registered_cores_mask != (mask | stat)) { + err = mali_pmu_wait_for_command_finish(pmu); + if (_MALI_OSK_ERR_OK != err) { + return err; + } + } else { + mali_hw_core_register_write(&pmu->hw_core, + PMU_REG_ADDR_MGMT_INT_CLEAR, PMU_REG_VAL_IRQ); + } + +#if defined(DEBUG) + /* Verify power status of domains after power down */ + stat = mali_hw_core_register_read(&pmu->hw_core, + PMU_REG_ADDR_MGMT_STATUS); + MALI_DEBUG_ASSERT(mask == (stat & mask)); +#endif + + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t mali_pmu_power_up(struct mali_pmu_core *pmu, u32 mask) +{ + u32 stat; + _mali_osk_errcode_t err; +#if !defined(CONFIG_MALI_PMU_PARALLEL_POWER_UP) + u32 current_domain; +#endif + + MALI_DEBUG_ASSERT_POINTER(pmu); + MALI_DEBUG_ASSERT(pmu->registered_cores_mask != 0); + MALI_DEBUG_ASSERT(mask <= pmu->registered_cores_mask); + MALI_DEBUG_ASSERT(0 == (mali_hw_core_register_read(&pmu->hw_core, + PMU_REG_ADDR_MGMT_INT_RAWSTAT) & + PMU_REG_VAL_IRQ)); + + MALI_DEBUG_PRINT(3, + ("PMU power up: ........................ [%s]\n", + mali_pm_mask_to_string(mask))); + + stat = mali_hw_core_register_read(&pmu->hw_core, + PMU_REG_ADDR_MGMT_STATUS); + stat &= pmu->registered_cores_mask; + + mask &= ~(0x1 << MALI_DOMAIN_INDEX_DUMMY); + if (0 == mask || 0 == (stat & mask)) return _MALI_OSK_ERR_OK; + + /* + * Assert that we are only powering up domains which are currently + * powered down. + */ + MALI_DEBUG_ASSERT(mask == (stat & mask)); + +#if defined(CONFIG_MALI_PMU_PARALLEL_POWER_UP) + mali_hw_core_register_write(&pmu->hw_core, + PMU_REG_ADDR_MGMT_POWER_UP, mask); + + err = mali_pmu_wait_for_command_finish(pmu); + if (_MALI_OSK_ERR_OK != err) { + return err; + } +#else + for (current_domain = 1; + current_domain <= pmu->registered_cores_mask; + current_domain <<= 1) { + if (current_domain & mask & stat) { + mali_hw_core_register_write(&pmu->hw_core, + PMU_REG_ADDR_MGMT_POWER_UP, + current_domain); + + err = mali_pmu_wait_for_command_finish(pmu); + if (_MALI_OSK_ERR_OK != err) { + return err; + } + } + } +#endif + +#if defined(DEBUG) + /* Verify power status of domains after power up */ + stat = mali_hw_core_register_read(&pmu->hw_core, + PMU_REG_ADDR_MGMT_STATUS); + MALI_DEBUG_ASSERT(0 == (stat & mask)); +#endif /* defined(DEBUG) */ + + return _MALI_OSK_ERR_OK; +} + +static _mali_osk_errcode_t mali_pmu_wait_for_command_finish( + struct mali_pmu_core *pmu) +{ + u32 rawstat; + u32 timeout = MALI_REG_POLL_COUNT_SLOW; + + MALI_DEBUG_ASSERT(pmu); + + /* Wait for the command to complete */ + do { + rawstat = mali_hw_core_register_read(&pmu->hw_core, + PMU_REG_ADDR_MGMT_INT_RAWSTAT); + --timeout; + } while (0 == (rawstat & PMU_REG_VAL_IRQ) && 0 < timeout); + + MALI_DEBUG_ASSERT(0 < timeout); + + if (0 == timeout) { + return _MALI_OSK_ERR_TIMEOUT; + } + + mali_hw_core_register_write(&pmu->hw_core, + PMU_REG_ADDR_MGMT_INT_CLEAR, PMU_REG_VAL_IRQ); + + return _MALI_OSK_ERR_OK; +} diff --git a/drivers/gpu/arm/utgard/common/mali_pmu.h b/drivers/gpu/arm/utgard/common/mali_pmu.h new file mode 100644 index 00000000000000..5ca78795f53503 --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_pmu.h @@ -0,0 +1,123 @@ +/* + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_platform.h + * Platform specific Mali driver functions + */ + +#ifndef __MALI_PMU_H__ +#define __MALI_PMU_H__ + +#include "mali_osk.h" +#include "mali_kernel_common.h" +#include "mali_hw_core.h" + +/** @brief MALI inbuilt PMU hardware info and PMU hardware has knowledge of cores power mask + */ +struct mali_pmu_core { + struct mali_hw_core hw_core; + u32 registered_cores_mask; + u32 switch_delay; +}; + +/** @brief Register layout for hardware PMU + */ +typedef enum { + PMU_REG_ADDR_MGMT_POWER_UP = 0x00, /*< Power up register */ + PMU_REG_ADDR_MGMT_POWER_DOWN = 0x04, /*< Power down register */ + PMU_REG_ADDR_MGMT_STATUS = 0x08, /*< Core sleep status register */ + PMU_REG_ADDR_MGMT_INT_MASK = 0x0C, /*< Interrupt mask register */ + PMU_REG_ADDR_MGMT_INT_RAWSTAT = 0x10, /*< Interrupt raw status register */ + PMU_REG_ADDR_MGMT_INT_CLEAR = 0x18, /*< Interrupt clear register */ + PMU_REG_ADDR_MGMT_SW_DELAY = 0x1C, /*< Switch delay register */ + PMU_REGISTER_ADDRESS_SPACE_SIZE = 0x28, /*< Size of register space */ +} pmu_reg_addr_mgmt_addr; + +#define PMU_REG_VAL_IRQ 1 + +extern struct mali_pmu_core *mali_global_pmu_core; + +/** @brief Initialisation of MALI PMU + * + * This is called from entry point of the driver in order to create and intialize the PMU resource + * + * @param resource it will be a pointer to a PMU resource + * @param number_of_pp_cores Number of found PP resources in configuration + * @param number_of_l2_caches Number of found L2 cache resources in configuration + * @return The created PMU object, or NULL in case of failure. + */ +struct mali_pmu_core *mali_pmu_create(_mali_osk_resource_t *resource); + +/** @brief It deallocates the PMU resource + * + * This is called on the exit of the driver to terminate the PMU resource + * + * @param pmu Pointer to PMU core object to delete + */ +void mali_pmu_delete(struct mali_pmu_core *pmu); + +/** @brief Set registered cores mask + * + * @param pmu Pointer to PMU core object + * @param mask All available/valid domain bits + */ +void mali_pmu_set_registered_cores_mask(struct mali_pmu_core *pmu, u32 mask); + +/** @brief Retrieves the Mali PMU core object (if any) + * + * @return The Mali PMU object, or NULL if no PMU exists. + */ +MALI_STATIC_INLINE struct mali_pmu_core *mali_pmu_get_global_pmu_core(void) +{ + return mali_global_pmu_core; +} + +/** @brief Reset PMU core + * + * @param pmu Pointer to PMU core object to reset + */ +void mali_pmu_reset(struct mali_pmu_core *pmu); + +void mali_pmu_power_up_all(struct mali_pmu_core *pmu); + +void mali_pmu_power_down_all(struct mali_pmu_core *pmu); + +/** @brief Returns a mask of the currently powered up domains + * + * @param pmu Pointer to PMU core object + */ +MALI_STATIC_INLINE u32 mali_pmu_get_mask(struct mali_pmu_core *pmu) +{ + u32 stat = mali_hw_core_register_read(&pmu->hw_core, PMU_REG_ADDR_MGMT_STATUS); + return ((~stat) & pmu->registered_cores_mask); +} + +/** @brief MALI GPU power down using MALI in-built PMU + * + * Called to power down the specified cores. + * + * @param pmu Pointer to PMU core object to power down + * @param mask Mask specifying which power domains to power down + * @return _MALI_OSK_ERR_OK on success otherwise, a suitable _mali_osk_errcode_t error. + */ +_mali_osk_errcode_t mali_pmu_power_down(struct mali_pmu_core *pmu, u32 mask); + +/** @brief MALI GPU power up using MALI in-built PMU + * + * Called to power up the specified cores. + * + * @param pmu Pointer to PMU core object to power up + * @param mask Mask specifying which power domains to power up + * @return _MALI_OSK_ERR_OK on success otherwise, a suitable _mali_osk_errcode_t error. + */ +_mali_osk_errcode_t mali_pmu_power_up(struct mali_pmu_core *pmu, u32 mask); + +#endif /* __MALI_PMU_H__ */ diff --git a/drivers/gpu/arm/utgard/common/mali_pp.c b/drivers/gpu/arm/utgard/common/mali_pp.c new file mode 100644 index 00000000000000..68bfd50bf9ae02 --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_pp.c @@ -0,0 +1,501 @@ +/* + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_pp_job.h" +#include "mali_pp.h" +#include "mali_hw_core.h" +#include "mali_group.h" +#include "regs/mali_200_regs.h" +#include "mali_kernel_common.h" +#include "mali_kernel_core.h" +#if defined(CONFIG_MALI400_PROFILING) +#include "mali_osk_profiling.h" +#endif + +/* Number of frame registers on Mali-200 */ +#define MALI_PP_MALI200_NUM_FRAME_REGISTERS ((0x04C/4)+1) +/* Number of frame registers on Mali-300 and later */ +#define MALI_PP_MALI400_NUM_FRAME_REGISTERS ((0x058/4)+1) + +static struct mali_pp_core *mali_global_pp_cores[MALI_MAX_NUMBER_OF_PP_CORES] = { NULL }; +static u32 mali_global_num_pp_cores = 0; + +/* Interrupt handlers */ +static void mali_pp_irq_probe_trigger(void *data); +static _mali_osk_errcode_t mali_pp_irq_probe_ack(void *data); + +struct mali_pp_core *mali_pp_create(const _mali_osk_resource_t *resource, struct mali_group *group, mali_bool is_virtual, u32 bcast_id) +{ + struct mali_pp_core *core = NULL; + + MALI_DEBUG_PRINT(2, ("Mali PP: Creating Mali PP core: %s\n", resource->description)); + MALI_DEBUG_PRINT(2, ("Mali PP: Base address of PP core: 0x%x\n", resource->base)); + + if (mali_global_num_pp_cores >= MALI_MAX_NUMBER_OF_PP_CORES) { + MALI_PRINT_ERROR(("Mali PP: Too many PP core objects created\n")); + return NULL; + } + + core = _mali_osk_calloc(1, sizeof(struct mali_pp_core)); + if (NULL != core) { + core->core_id = mali_global_num_pp_cores; + core->bcast_id = bcast_id; + + if (_MALI_OSK_ERR_OK == mali_hw_core_create(&core->hw_core, resource, MALI200_REG_SIZEOF_REGISTER_BANK)) { + _mali_osk_errcode_t ret; + + if (!is_virtual) { + ret = mali_pp_reset(core); + } else { + ret = _MALI_OSK_ERR_OK; + } + + if (_MALI_OSK_ERR_OK == ret) { + ret = mali_group_add_pp_core(group, core); + if (_MALI_OSK_ERR_OK == ret) { + /* Setup IRQ handlers (which will do IRQ probing if needed) */ + MALI_DEBUG_ASSERT(!is_virtual || -1 != resource->irq); + + core->irq = _mali_osk_irq_init(resource->irq, + mali_group_upper_half_pp, + group, + mali_pp_irq_probe_trigger, + mali_pp_irq_probe_ack, + core, + resource->description); + if (NULL != core->irq) { + mali_global_pp_cores[mali_global_num_pp_cores] = core; + mali_global_num_pp_cores++; + + return core; + } else { + MALI_PRINT_ERROR(("Mali PP: Failed to setup interrupt handlers for PP core %s\n", core->hw_core.description)); + } + mali_group_remove_pp_core(group); + } else { + MALI_PRINT_ERROR(("Mali PP: Failed to add core %s to group\n", core->hw_core.description)); + } + } + mali_hw_core_delete(&core->hw_core); + } + + _mali_osk_free(core); + } else { + MALI_PRINT_ERROR(("Mali PP: Failed to allocate memory for PP core\n")); + } + + return NULL; +} + +void mali_pp_delete(struct mali_pp_core *core) +{ + u32 i; + + MALI_DEBUG_ASSERT_POINTER(core); + + _mali_osk_irq_term(core->irq); + mali_hw_core_delete(&core->hw_core); + + /* Remove core from global list */ + for (i = 0; i < mali_global_num_pp_cores; i++) { + if (mali_global_pp_cores[i] == core) { + mali_global_pp_cores[i] = NULL; + mali_global_num_pp_cores--; + + if (i != mali_global_num_pp_cores) { + /* We removed a PP core from the middle of the array -- move the last + * PP core to the current position to close the gap */ + mali_global_pp_cores[i] = mali_global_pp_cores[mali_global_num_pp_cores]; + mali_global_pp_cores[mali_global_num_pp_cores] = NULL; + } + + break; + } + } + + _mali_osk_free(core); +} + +void mali_pp_stop_bus(struct mali_pp_core *core) +{ + MALI_DEBUG_ASSERT_POINTER(core); + /* Will only send the stop bus command, and not wait for it to complete */ + mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_CTRL_MGMT, MALI200_REG_VAL_CTRL_MGMT_STOP_BUS); +} + +_mali_osk_errcode_t mali_pp_stop_bus_wait(struct mali_pp_core *core) +{ + int i; + + MALI_DEBUG_ASSERT_POINTER(core); + + /* Send the stop bus command. */ + mali_pp_stop_bus(core); + + /* Wait for bus to be stopped */ + for (i = 0; i < MALI_REG_POLL_COUNT_FAST; i++) { + if (mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_STATUS) & MALI200_REG_VAL_STATUS_BUS_STOPPED) + break; + } + + if (MALI_REG_POLL_COUNT_FAST == i) { + MALI_PRINT_ERROR(("Mali PP: Failed to stop bus on %s. Status: 0x%08x\n", core->hw_core.description, mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_STATUS))); + return _MALI_OSK_ERR_FAULT; + } + return _MALI_OSK_ERR_OK; +} + +/* Frame register reset values. + * Taken from the Mali400 TRM, 3.6. Pixel processor control register summary */ +static const u32 mali_frame_registers_reset_values[_MALI_PP_MAX_FRAME_REGISTERS] = { + 0x0, /* Renderer List Address Register */ + 0x0, /* Renderer State Word Base Address Register */ + 0x0, /* Renderer Vertex Base Register */ + 0x2, /* Feature Enable Register */ + 0x0, /* Z Clear Value Register */ + 0x0, /* Stencil Clear Value Register */ + 0x0, /* ABGR Clear Value 0 Register */ + 0x0, /* ABGR Clear Value 1 Register */ + 0x0, /* ABGR Clear Value 2 Register */ + 0x0, /* ABGR Clear Value 3 Register */ + 0x0, /* Bounding Box Left Right Register */ + 0x0, /* Bounding Box Bottom Register */ + 0x0, /* FS Stack Address Register */ + 0x0, /* FS Stack Size and Initial Value Register */ + 0x0, /* Reserved */ + 0x0, /* Reserved */ + 0x0, /* Origin Offset X Register */ + 0x0, /* Origin Offset Y Register */ + 0x75, /* Subpixel Specifier Register */ + 0x0, /* Tiebreak mode Register */ + 0x0, /* Polygon List Format Register */ + 0x0, /* Scaling Register */ + 0x0 /* Tilebuffer configuration Register */ +}; + +/* WBx register reset values */ +static const u32 mali_wb_registers_reset_values[_MALI_PP_MAX_WB_REGISTERS] = { + 0x0, /* WBx Source Select Register */ + 0x0, /* WBx Target Address Register */ + 0x0, /* WBx Target Pixel Format Register */ + 0x0, /* WBx Target AA Format Register */ + 0x0, /* WBx Target Layout */ + 0x0, /* WBx Target Scanline Length */ + 0x0, /* WBx Target Flags Register */ + 0x0, /* WBx MRT Enable Register */ + 0x0, /* WBx MRT Offset Register */ + 0x0, /* WBx Global Test Enable Register */ + 0x0, /* WBx Global Test Reference Value Register */ + 0x0 /* WBx Global Test Compare Function Register */ +}; + +/* Performance Counter 0 Enable Register reset value */ +static const u32 mali_perf_cnt_enable_reset_value = 0; + +_mali_osk_errcode_t mali_pp_hard_reset(struct mali_pp_core *core) +{ + /* Bus must be stopped before calling this function */ + const u32 reset_wait_target_register = MALI200_REG_ADDR_MGMT_PERF_CNT_0_LIMIT; + const u32 reset_invalid_value = 0xC0FFE000; + const u32 reset_check_value = 0xC01A0000; + int i; + + MALI_DEBUG_ASSERT_POINTER(core); + MALI_DEBUG_PRINT(2, ("Mali PP: Hard reset of core %s\n", core->hw_core.description)); + + /* Set register to a bogus value. The register will be used to detect when reset is complete */ + mali_hw_core_register_write_relaxed(&core->hw_core, reset_wait_target_register, reset_invalid_value); + mali_hw_core_register_write_relaxed(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_MASK, MALI200_REG_VAL_IRQ_MASK_NONE); + + /* Force core to reset */ + mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_CTRL_MGMT, MALI200_REG_VAL_CTRL_MGMT_FORCE_RESET); + + /* Wait for reset to be complete */ + for (i = 0; i < MALI_REG_POLL_COUNT_FAST; i++) { + mali_hw_core_register_write(&core->hw_core, reset_wait_target_register, reset_check_value); + if (reset_check_value == mali_hw_core_register_read(&core->hw_core, reset_wait_target_register)) { + break; + } + } + + if (MALI_REG_POLL_COUNT_FAST == i) { + MALI_PRINT_ERROR(("Mali PP: The hard reset loop didn't work, unable to recover\n")); + } + + mali_hw_core_register_write(&core->hw_core, reset_wait_target_register, 0x00000000); /* set it back to the default */ + /* Re-enable interrupts */ + mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_CLEAR, MALI200_REG_VAL_IRQ_MASK_ALL); + mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_MASK, MALI200_REG_VAL_IRQ_MASK_USED); + + return _MALI_OSK_ERR_OK; +} + +void mali_pp_reset_async(struct mali_pp_core *core) +{ + MALI_DEBUG_ASSERT_POINTER(core); + + MALI_DEBUG_PRINT(4, ("Mali PP: Reset of core %s\n", core->hw_core.description)); + + mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_MASK, 0); /* disable the IRQs */ + mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_RAWSTAT, MALI200_REG_VAL_IRQ_MASK_ALL); + mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_CTRL_MGMT, MALI400PP_REG_VAL_CTRL_MGMT_SOFT_RESET); +} + +_mali_osk_errcode_t mali_pp_reset_wait(struct mali_pp_core *core) +{ + int i; + u32 rawstat = 0; + + for (i = 0; i < MALI_REG_POLL_COUNT_FAST; i++) { + u32 status = mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_STATUS); + if (!(status & MALI200_REG_VAL_STATUS_RENDERING_ACTIVE)) { + rawstat = mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_RAWSTAT); + if (rawstat == MALI400PP_REG_VAL_IRQ_RESET_COMPLETED) { + break; + } + } + } + + if (i == MALI_REG_POLL_COUNT_FAST) { + MALI_PRINT_ERROR(("Mali PP: Failed to reset core %s, rawstat: 0x%08x\n", + core->hw_core.description, rawstat)); + return _MALI_OSK_ERR_FAULT; + } + + /* Re-enable interrupts */ + mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_CLEAR, MALI200_REG_VAL_IRQ_MASK_ALL); + mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_MASK, MALI200_REG_VAL_IRQ_MASK_USED); + + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t mali_pp_reset(struct mali_pp_core *core) +{ + mali_pp_reset_async(core); + return mali_pp_reset_wait(core); +} + +void mali_pp_job_start(struct mali_pp_core *core, struct mali_pp_job *job, u32 sub_job, mali_bool restart_virtual) +{ + u32 relative_address; + u32 start_index; + u32 nr_of_regs; + u32 *frame_registers = mali_pp_job_get_frame_registers(job); + u32 *wb0_registers = mali_pp_job_get_wb0_registers(job); + u32 *wb1_registers = mali_pp_job_get_wb1_registers(job); + u32 *wb2_registers = mali_pp_job_get_wb2_registers(job); + u32 counter_src0 = mali_pp_job_get_perf_counter_src0(job, sub_job); + u32 counter_src1 = mali_pp_job_get_perf_counter_src1(job, sub_job); + + MALI_DEBUG_ASSERT_POINTER(core); + + /* Write frame registers */ + + /* + * There are two frame registers which are different for each sub job: + * 1. The Renderer List Address Register (MALI200_REG_ADDR_FRAME) + * 2. The FS Stack Address Register (MALI200_REG_ADDR_STACK) + */ + mali_hw_core_register_write_relaxed_conditional(&core->hw_core, MALI200_REG_ADDR_FRAME, mali_pp_job_get_addr_frame(job, sub_job), mali_frame_registers_reset_values[MALI200_REG_ADDR_FRAME / sizeof(u32)]); + + /* For virtual jobs, the stack address shouldn't be broadcast but written individually */ + if (!mali_pp_job_is_virtual(job) || restart_virtual) { + mali_hw_core_register_write_relaxed_conditional(&core->hw_core, MALI200_REG_ADDR_STACK, mali_pp_job_get_addr_stack(job, sub_job), mali_frame_registers_reset_values[MALI200_REG_ADDR_STACK / sizeof(u32)]); + } + + /* Write registers between MALI200_REG_ADDR_FRAME and MALI200_REG_ADDR_STACK */ + relative_address = MALI200_REG_ADDR_RSW; + start_index = MALI200_REG_ADDR_RSW / sizeof(u32); + nr_of_regs = (MALI200_REG_ADDR_STACK - MALI200_REG_ADDR_RSW) / sizeof(u32); + + mali_hw_core_register_write_array_relaxed_conditional(&core->hw_core, + relative_address, &frame_registers[start_index], + nr_of_regs, &mali_frame_registers_reset_values[start_index]); + + /* MALI200_REG_ADDR_STACK_SIZE */ + relative_address = MALI200_REG_ADDR_STACK_SIZE; + start_index = MALI200_REG_ADDR_STACK_SIZE / sizeof(u32); + + mali_hw_core_register_write_relaxed_conditional(&core->hw_core, + relative_address, frame_registers[start_index], + mali_frame_registers_reset_values[start_index]); + + /* Skip 2 reserved registers */ + + /* Write remaining registers */ + relative_address = MALI200_REG_ADDR_ORIGIN_OFFSET_X; + start_index = MALI200_REG_ADDR_ORIGIN_OFFSET_X / sizeof(u32); + nr_of_regs = MALI_PP_MALI400_NUM_FRAME_REGISTERS - MALI200_REG_ADDR_ORIGIN_OFFSET_X / sizeof(u32); + + mali_hw_core_register_write_array_relaxed_conditional(&core->hw_core, + relative_address, &frame_registers[start_index], + nr_of_regs, &mali_frame_registers_reset_values[start_index]); + + /* Write WBx registers */ + if (wb0_registers[0]) { /* M200_WB0_REG_SOURCE_SELECT register */ + mali_hw_core_register_write_array_relaxed_conditional(&core->hw_core, MALI200_REG_ADDR_WB0, wb0_registers, _MALI_PP_MAX_WB_REGISTERS, mali_wb_registers_reset_values); + } + + if (wb1_registers[0]) { /* M200_WB1_REG_SOURCE_SELECT register */ + mali_hw_core_register_write_array_relaxed_conditional(&core->hw_core, MALI200_REG_ADDR_WB1, wb1_registers, _MALI_PP_MAX_WB_REGISTERS, mali_wb_registers_reset_values); + } + + if (wb2_registers[0]) { /* M200_WB2_REG_SOURCE_SELECT register */ + mali_hw_core_register_write_array_relaxed_conditional(&core->hw_core, MALI200_REG_ADDR_WB2, wb2_registers, _MALI_PP_MAX_WB_REGISTERS, mali_wb_registers_reset_values); + } + + if (MALI_HW_CORE_NO_COUNTER != counter_src0) { + mali_hw_core_register_write_relaxed(&core->hw_core, MALI200_REG_ADDR_MGMT_PERF_CNT_0_SRC, counter_src0); + mali_hw_core_register_write_relaxed_conditional(&core->hw_core, MALI200_REG_ADDR_MGMT_PERF_CNT_0_ENABLE, MALI200_REG_VAL_PERF_CNT_ENABLE, mali_perf_cnt_enable_reset_value); + } + if (MALI_HW_CORE_NO_COUNTER != counter_src1) { + mali_hw_core_register_write_relaxed(&core->hw_core, MALI200_REG_ADDR_MGMT_PERF_CNT_1_SRC, counter_src1); + mali_hw_core_register_write_relaxed_conditional(&core->hw_core, MALI200_REG_ADDR_MGMT_PERF_CNT_1_ENABLE, MALI200_REG_VAL_PERF_CNT_ENABLE, mali_perf_cnt_enable_reset_value); + } + +#ifdef CONFIG_MALI400_HEATMAPS_ENABLED + if (job->uargs.perf_counter_flag & _MALI_PERFORMANCE_COUNTER_FLAG_HEATMAP_ENABLE) { + mali_hw_core_register_write_relaxed(&core->hw_core, MALI200_REG_ADDR_MGMT_PERFMON_CONTR, ((job->uargs.tilesx & 0x3FF) << 16) | 1); + mali_hw_core_register_write_relaxed(&core->hw_core, MALI200_REG_ADDR_MGMT_PERFMON_BASE, job->uargs.heatmap_mem & 0xFFFFFFF8); + } +#endif /* CONFIG_MALI400_HEATMAPS_ENABLED */ + + MALI_DEBUG_PRINT(3, ("Mali PP: Starting job 0x%08X part %u/%u on PP core %s\n", job, sub_job + 1, mali_pp_job_get_sub_job_count(job), core->hw_core.description)); + + /* Adding barrier to make sure all rester writes are finished */ + _mali_osk_write_mem_barrier(); + + /* This is the command that starts the core. + * + * Don't actually run the job if PROFILING_SKIP_PP_JOBS are set, just + * force core to assert the completion interrupt. + */ +#if !defined(PROFILING_SKIP_PP_JOBS) + mali_hw_core_register_write_relaxed(&core->hw_core, MALI200_REG_ADDR_MGMT_CTRL_MGMT, MALI200_REG_VAL_CTRL_MGMT_START_RENDERING); +#else + mali_hw_core_register_write_relaxed(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_RAWSTAT, MALI200_REG_VAL_IRQ_END_OF_FRAME); +#endif + + /* Adding barrier to make sure previous rester writes is finished */ + _mali_osk_write_mem_barrier(); +} + +u32 mali_pp_core_get_version(struct mali_pp_core *core) +{ + MALI_DEBUG_ASSERT_POINTER(core); + return mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_VERSION); +} + +struct mali_pp_core *mali_pp_get_global_pp_core(u32 index) +{ + if (mali_global_num_pp_cores > index) { + return mali_global_pp_cores[index]; + } + + return NULL; +} + +u32 mali_pp_get_glob_num_pp_cores(void) +{ + return mali_global_num_pp_cores; +} + +/* ------------- interrupt handling below ------------------ */ +static void mali_pp_irq_probe_trigger(void *data) +{ + struct mali_pp_core *core = (struct mali_pp_core *)data; + mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_MASK, MALI200_REG_VAL_IRQ_MASK_USED); + mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_RAWSTAT, MALI200_REG_VAL_IRQ_BUS_ERROR); + _mali_osk_mem_barrier(); +} + +static _mali_osk_errcode_t mali_pp_irq_probe_ack(void *data) +{ + struct mali_pp_core *core = (struct mali_pp_core *)data; + u32 irq_readout; + + irq_readout = mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_STATUS); + if (MALI200_REG_VAL_IRQ_BUS_ERROR & irq_readout) { + mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_CLEAR, MALI200_REG_VAL_IRQ_BUS_ERROR); + _mali_osk_mem_barrier(); + return _MALI_OSK_ERR_OK; + } + + return _MALI_OSK_ERR_FAULT; +} + + +#if 0 +static void mali_pp_print_registers(struct mali_pp_core *core) +{ + MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_VERSION = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_VERSION))); + MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_CURRENT_REND_LIST_ADDR = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_CURRENT_REND_LIST_ADDR))); + MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_STATUS = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_STATUS))); + MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_INT_RAWSTAT = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_RAWSTAT))); + MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_INT_MASK = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_MASK))); + MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_INT_STATUS = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_STATUS))); + MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_BUS_ERROR_STATUS = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_BUS_ERROR_STATUS))); + MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_PERF_CNT_0_ENABLE = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_PERF_CNT_0_ENABLE))); + MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_PERF_CNT_0_SRC = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_PERF_CNT_0_SRC))); + MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_PERF_CNT_0_VALUE = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_PERF_CNT_0_VALUE))); + MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_PERF_CNT_1_ENABLE = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_PERF_CNT_1_ENABLE))); + MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_PERF_CNT_1_SRC = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_PERF_CNT_1_SRC))); + MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_PERF_CNT_1_VALUE = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_PERF_CNT_1_VALUE))); +} +#endif + +#if 0 +void mali_pp_print_state(struct mali_pp_core *core) +{ + MALI_DEBUG_PRINT(2, ("Mali PP: State: 0x%08x\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_STATUS))); +} +#endif + +void mali_pp_update_performance_counters(struct mali_pp_core *parent, struct mali_pp_core *child, struct mali_pp_job *job, u32 subjob) +{ + u32 val0 = 0; + u32 val1 = 0; + u32 counter_src0 = mali_pp_job_get_perf_counter_src0(job, subjob); + u32 counter_src1 = mali_pp_job_get_perf_counter_src1(job, subjob); +#if defined(CONFIG_MALI400_PROFILING) + int counter_index = COUNTER_FP_0_C0 + (2 * child->core_id); +#endif + + if (MALI_HW_CORE_NO_COUNTER != counter_src0) { + val0 = mali_hw_core_register_read(&child->hw_core, MALI200_REG_ADDR_MGMT_PERF_CNT_0_VALUE); + mali_pp_job_set_perf_counter_value0(job, subjob, val0); + +#if defined(CONFIG_MALI400_PROFILING) + _mali_osk_profiling_report_hw_counter(counter_index, val0); + _mali_osk_profiling_record_global_counters(counter_index, val0); +#endif + } + + if (MALI_HW_CORE_NO_COUNTER != counter_src1) { + val1 = mali_hw_core_register_read(&child->hw_core, MALI200_REG_ADDR_MGMT_PERF_CNT_1_VALUE); + mali_pp_job_set_perf_counter_value1(job, subjob, val1); + +#if defined(CONFIG_MALI400_PROFILING) + _mali_osk_profiling_report_hw_counter(counter_index + 1, val1); + _mali_osk_profiling_record_global_counters(counter_index + 1, val1); +#endif + } +} + +#if MALI_STATE_TRACKING +u32 mali_pp_dump_state(struct mali_pp_core *core, char *buf, u32 size) +{ + int n = 0; + + n += _mali_osk_snprintf(buf + n, size - n, "\tPP #%d: %s\n", core->core_id, core->hw_core.description); + + return n; +} +#endif diff --git a/drivers/gpu/arm/utgard/common/mali_pp.h b/drivers/gpu/arm/utgard/common/mali_pp.h new file mode 100644 index 00000000000000..45712a30e831c5 --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_pp.h @@ -0,0 +1,137 @@ +/* + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_PP_H__ +#define __MALI_PP_H__ + +#include "mali_osk.h" +#include "mali_pp_job.h" +#include "mali_hw_core.h" + +struct mali_group; + +#define MALI_MAX_NUMBER_OF_PP_CORES 9 + +/** + * Definition of the PP core struct + * Used to track a PP core in the system. + */ +struct mali_pp_core { + struct mali_hw_core hw_core; /**< Common for all HW cores */ + _mali_osk_irq_t *irq; /**< IRQ handler */ + u32 core_id; /**< Unique core ID */ + u32 bcast_id; /**< The "flag" value used by the Mali-450 broadcast and DLBU unit */ +}; + +_mali_osk_errcode_t mali_pp_initialize(void); +void mali_pp_terminate(void); + +struct mali_pp_core *mali_pp_create(const _mali_osk_resource_t *resource, struct mali_group *group, mali_bool is_virtual, u32 bcast_id); +void mali_pp_delete(struct mali_pp_core *core); + +void mali_pp_stop_bus(struct mali_pp_core *core); +_mali_osk_errcode_t mali_pp_stop_bus_wait(struct mali_pp_core *core); +void mali_pp_reset_async(struct mali_pp_core *core); +_mali_osk_errcode_t mali_pp_reset_wait(struct mali_pp_core *core); +_mali_osk_errcode_t mali_pp_reset(struct mali_pp_core *core); +_mali_osk_errcode_t mali_pp_hard_reset(struct mali_pp_core *core); + +void mali_pp_job_start(struct mali_pp_core *core, struct mali_pp_job *job, u32 sub_job, mali_bool restart_virtual); + +u32 mali_pp_core_get_version(struct mali_pp_core *core); + +MALI_STATIC_INLINE u32 mali_pp_core_get_id(struct mali_pp_core *core) +{ + MALI_DEBUG_ASSERT_POINTER(core); + return core->core_id; +} + +MALI_STATIC_INLINE u32 mali_pp_core_get_bcast_id(struct mali_pp_core *core) +{ + MALI_DEBUG_ASSERT_POINTER(core); + return core->bcast_id; +} + +struct mali_pp_core *mali_pp_get_global_pp_core(u32 index); +u32 mali_pp_get_glob_num_pp_cores(void); + +/* Debug */ +u32 mali_pp_dump_state(struct mali_pp_core *core, char *buf, u32 size); + +/** + * Put instrumented HW counters from the core(s) to the job object (if enabled) + * + * parent and child is always the same, except for virtual jobs on Mali-450. + * In this case, the counters will be enabled on the virtual core (parent), + * but values need to be read from the child cores. + * + * @param parent The core used to see if the counters was enabled + * @param child The core to actually read the values from + * @job Job object to update with counter values (if enabled) + * @subjob Which subjob the counters are applicable for (core ID for virtual jobs) + */ +void mali_pp_update_performance_counters(struct mali_pp_core *parent, struct mali_pp_core *child, struct mali_pp_job *job, u32 subjob); + +MALI_STATIC_INLINE const char *mali_pp_core_description(struct mali_pp_core *core) +{ + return core->hw_core.description; +} + +MALI_STATIC_INLINE enum mali_interrupt_result mali_pp_get_interrupt_result(struct mali_pp_core *core) +{ + u32 rawstat_used = mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_RAWSTAT) & + MALI200_REG_VAL_IRQ_MASK_USED; + if (0 == rawstat_used) { + return MALI_INTERRUPT_RESULT_NONE; + } else if (MALI200_REG_VAL_IRQ_END_OF_FRAME == rawstat_used) { + return MALI_INTERRUPT_RESULT_SUCCESS; + } + return MALI_INTERRUPT_RESULT_ERROR; +} + +MALI_STATIC_INLINE u32 mali_pp_get_rawstat(struct mali_pp_core *core) +{ + MALI_DEBUG_ASSERT_POINTER(core); + return mali_hw_core_register_read(&core->hw_core, + MALI200_REG_ADDR_MGMT_INT_RAWSTAT); +} + + +MALI_STATIC_INLINE u32 mali_pp_is_active(struct mali_pp_core *core) +{ + u32 status = mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_STATUS); + return (status & MALI200_REG_VAL_STATUS_RENDERING_ACTIVE) ? MALI_TRUE : MALI_FALSE; +} + +MALI_STATIC_INLINE void mali_pp_mask_all_interrupts(struct mali_pp_core *core) +{ + mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_MASK, MALI200_REG_VAL_IRQ_MASK_NONE); +} + +MALI_STATIC_INLINE void mali_pp_enable_interrupts(struct mali_pp_core *core) +{ + mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_MASK, MALI200_REG_VAL_IRQ_MASK_USED); +} + +MALI_STATIC_INLINE void mali_pp_write_addr_renderer_list(struct mali_pp_core *core, + struct mali_pp_job *job, u32 subjob) +{ + u32 addr = mali_pp_job_get_addr_frame(job, subjob); + mali_hw_core_register_write_relaxed(&core->hw_core, MALI200_REG_ADDR_FRAME, addr); +} + + +MALI_STATIC_INLINE void mali_pp_write_addr_stack(struct mali_pp_core *core, struct mali_pp_job *job) +{ + u32 addr = mali_pp_job_get_addr_stack(job, core->core_id); + mali_hw_core_register_write_relaxed(&core->hw_core, MALI200_REG_ADDR_STACK, addr); +} + +#endif /* __MALI_PP_H__ */ diff --git a/drivers/gpu/arm/utgard/common/mali_pp_job.c b/drivers/gpu/arm/utgard/common/mali_pp_job.c new file mode 100644 index 00000000000000..5528360841af7f --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_pp_job.c @@ -0,0 +1,308 @@ +/* + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_pp.h" +#include "mali_pp_job.h" +#include "mali_osk.h" +#include "mali_osk_list.h" +#include "mali_kernel_common.h" +#include "mali_uk_types.h" +#include "mali_executor.h" +#if defined(CONFIG_DMA_SHARED_BUFFER) && !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) +#include "linux/mali_memory_dma_buf.h" +#endif +#include "mali_memory_swap_alloc.h" +#include "mali_scheduler.h" + +static u32 pp_counter_src0 = MALI_HW_CORE_NO_COUNTER; /**< Performance counter 0, MALI_HW_CORE_NO_COUNTER for disabled */ +static u32 pp_counter_src1 = MALI_HW_CORE_NO_COUNTER; /**< Performance counter 1, MALI_HW_CORE_NO_COUNTER for disabled */ +static _mali_osk_atomic_t pp_counter_per_sub_job_count; /**< Number of values in the two arrays which is != MALI_HW_CORE_NO_COUNTER */ +static u32 pp_counter_per_sub_job_src0[_MALI_PP_MAX_SUB_JOBS] = { MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER }; +static u32 pp_counter_per_sub_job_src1[_MALI_PP_MAX_SUB_JOBS] = { MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER }; + +void mali_pp_job_initialize(void) +{ + _mali_osk_atomic_init(&pp_counter_per_sub_job_count, 0); +} + +void mali_pp_job_terminate(void) +{ + _mali_osk_atomic_term(&pp_counter_per_sub_job_count); +} + +struct mali_pp_job *mali_pp_job_create(struct mali_session_data *session, + _mali_uk_pp_start_job_s __user *uargs, u32 id) +{ + struct mali_pp_job *job; + u32 perf_counter_flag; + + job = _mali_osk_calloc(1, sizeof(struct mali_pp_job)); + if (NULL != job) { + + _mali_osk_list_init(&job->list); + _mali_osk_list_init(&job->session_fb_lookup_list); + + if (0 != _mali_osk_copy_from_user(&job->uargs, uargs, sizeof(_mali_uk_pp_start_job_s))) { + goto fail; + } + + if (job->uargs.num_cores > _MALI_PP_MAX_SUB_JOBS) { + MALI_PRINT_ERROR(("Mali PP job: Too many sub jobs specified in job object\n")); + goto fail; + } + + if (!mali_pp_job_use_no_notification(job)) { + job->finished_notification = _mali_osk_notification_create(_MALI_NOTIFICATION_PP_FINISHED, sizeof(_mali_uk_pp_job_finished_s)); + if (NULL == job->finished_notification) goto fail; + } + + perf_counter_flag = mali_pp_job_get_perf_counter_flag(job); + + /* case when no counters came from user space + * so pass the debugfs / DS-5 provided global ones to the job object */ + if (!((perf_counter_flag & _MALI_PERFORMANCE_COUNTER_FLAG_SRC0_ENABLE) || + (perf_counter_flag & _MALI_PERFORMANCE_COUNTER_FLAG_SRC1_ENABLE))) { + u32 sub_job_count = _mali_osk_atomic_read(&pp_counter_per_sub_job_count); + + /* These counters apply for all virtual jobs, and where no per sub job counter is specified */ + job->uargs.perf_counter_src0 = pp_counter_src0; + job->uargs.perf_counter_src1 = pp_counter_src1; + + /* We only copy the per sub job array if it is enabled with at least one counter */ + if (0 < sub_job_count) { + job->perf_counter_per_sub_job_count = sub_job_count; + _mali_osk_memcpy(job->perf_counter_per_sub_job_src0, pp_counter_per_sub_job_src0, sizeof(pp_counter_per_sub_job_src0)); + _mali_osk_memcpy(job->perf_counter_per_sub_job_src1, pp_counter_per_sub_job_src1, sizeof(pp_counter_per_sub_job_src1)); + } + } + + job->session = session; + job->id = id; + + job->sub_jobs_num = job->uargs.num_cores ? job->uargs.num_cores : 1; + job->pid = _mali_osk_get_pid(); + job->tid = _mali_osk_get_tid(); + + _mali_osk_atomic_init(&job->sub_jobs_completed, 0); + _mali_osk_atomic_init(&job->sub_job_errors, 0); + job->swap_status = MALI_NO_SWAP_IN; + job->user_notification = MALI_FALSE; + job->num_pp_cores_in_virtual = 0; + + if (job->uargs.num_memory_cookies > session->allocation_mgr.mali_allocation_num) { + MALI_PRINT_ERROR(("Mali PP job: The number of memory cookies is invalid !\n")); + goto fail; + } + + if (job->uargs.num_memory_cookies > 0) { + u32 size; + u32 __user *memory_cookies = (u32 __user *)(uintptr_t)job->uargs.memory_cookies; + + size = sizeof(*memory_cookies) * (job->uargs.num_memory_cookies); + + job->memory_cookies = _mali_osk_malloc(size); + if (NULL == job->memory_cookies) { + MALI_PRINT_ERROR(("Mali PP job: Failed to allocate %d bytes of memory cookies!\n", size)); + goto fail; + } + + if (0 != _mali_osk_copy_from_user(job->memory_cookies, memory_cookies, size)) { + MALI_PRINT_ERROR(("Mali PP job: Failed to copy %d bytes of memory cookies from user!\n", size)); + goto fail; + } + } + + if (_MALI_OSK_ERR_OK != mali_pp_job_check(job)) { + /* Not a valid job. */ + goto fail; + } + + mali_timeline_tracker_init(&job->tracker, MALI_TIMELINE_TRACKER_PP, NULL, job); + mali_timeline_fence_copy_uk_fence(&(job->tracker.fence), &(job->uargs.fence)); + + mali_mem_swap_in_pages(job); + + return job; + } + +fail: + if (NULL != job) { + mali_pp_job_delete(job); + } + + return NULL; +} + +void mali_pp_job_delete(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT(_mali_osk_list_empty(&job->list)); + MALI_DEBUG_ASSERT(_mali_osk_list_empty(&job->session_fb_lookup_list)); + + if (NULL != job->memory_cookies) { +#if defined(CONFIG_DMA_SHARED_BUFFER) && !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) + /* Unmap buffers attached to job */ + mali_dma_buf_unmap_job(job); +#endif + if (MALI_NO_SWAP_IN != job->swap_status) { + mali_mem_swap_out_pages(job); + } + + _mali_osk_free(job->memory_cookies); + } + + if (job->user_notification) { + mali_scheduler_return_pp_job_to_user(job, + job->num_pp_cores_in_virtual); + } + + if (NULL != job->finished_notification) { + _mali_osk_notification_delete(job->finished_notification); + } + + _mali_osk_atomic_term(&job->sub_jobs_completed); + _mali_osk_atomic_term(&job->sub_job_errors); + + _mali_osk_free(job); +} + +void mali_pp_job_list_add(struct mali_pp_job *job, _mali_osk_list_t *list) +{ + struct mali_pp_job *iter; + struct mali_pp_job *tmp; + + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + + /* Find position in list/queue where job should be added. */ + _MALI_OSK_LIST_FOREACHENTRY_REVERSE(iter, tmp, list, + struct mali_pp_job, list) { + /* job should be started after iter if iter is in progress. */ + if (0 < iter->sub_jobs_started) { + break; + } + + /* + * job should be started after iter if it has a higher + * job id. A span is used to handle job id wrapping. + */ + if ((mali_pp_job_get_id(job) - + mali_pp_job_get_id(iter)) < + MALI_SCHEDULER_JOB_ID_SPAN) { + break; + } + } + + _mali_osk_list_add(&job->list, &iter->list); +} + + +u32 mali_pp_job_get_perf_counter_src0(struct mali_pp_job *job, u32 sub_job) +{ + /* Virtual jobs always use the global job counter (or if there are per sub job counters at all) */ + if (mali_pp_job_is_virtual(job) || 0 == job->perf_counter_per_sub_job_count) { + return job->uargs.perf_counter_src0; + } + + /* Use per sub job counter if enabled... */ + if (MALI_HW_CORE_NO_COUNTER != job->perf_counter_per_sub_job_src0[sub_job]) { + return job->perf_counter_per_sub_job_src0[sub_job]; + } + + /* ...else default to global job counter */ + return job->uargs.perf_counter_src0; +} + +u32 mali_pp_job_get_perf_counter_src1(struct mali_pp_job *job, u32 sub_job) +{ + /* Virtual jobs always use the global job counter (or if there are per sub job counters at all) */ + if (mali_pp_job_is_virtual(job) || 0 == job->perf_counter_per_sub_job_count) { + /* Virtual jobs always use the global job counter */ + return job->uargs.perf_counter_src1; + } + + /* Use per sub job counter if enabled... */ + if (MALI_HW_CORE_NO_COUNTER != job->perf_counter_per_sub_job_src1[sub_job]) { + return job->perf_counter_per_sub_job_src1[sub_job]; + } + + /* ...else default to global job counter */ + return job->uargs.perf_counter_src1; +} + +void mali_pp_job_set_pp_counter_global_src0(u32 counter) +{ + pp_counter_src0 = counter; +} + +void mali_pp_job_set_pp_counter_global_src1(u32 counter) +{ + pp_counter_src1 = counter; +} + +void mali_pp_job_set_pp_counter_sub_job_src0(u32 sub_job, u32 counter) +{ + MALI_DEBUG_ASSERT(sub_job < _MALI_PP_MAX_SUB_JOBS); + + if (MALI_HW_CORE_NO_COUNTER == pp_counter_per_sub_job_src0[sub_job]) { + /* increment count since existing counter was disabled */ + _mali_osk_atomic_inc(&pp_counter_per_sub_job_count); + } + + if (MALI_HW_CORE_NO_COUNTER == counter) { + /* decrement count since new counter is disabled */ + _mali_osk_atomic_dec(&pp_counter_per_sub_job_count); + } + + /* PS: A change from MALI_HW_CORE_NO_COUNTER to MALI_HW_CORE_NO_COUNTER will inc and dec, result will be 0 change */ + + pp_counter_per_sub_job_src0[sub_job] = counter; +} + +void mali_pp_job_set_pp_counter_sub_job_src1(u32 sub_job, u32 counter) +{ + MALI_DEBUG_ASSERT(sub_job < _MALI_PP_MAX_SUB_JOBS); + + if (MALI_HW_CORE_NO_COUNTER == pp_counter_per_sub_job_src1[sub_job]) { + /* increment count since existing counter was disabled */ + _mali_osk_atomic_inc(&pp_counter_per_sub_job_count); + } + + if (MALI_HW_CORE_NO_COUNTER == counter) { + /* decrement count since new counter is disabled */ + _mali_osk_atomic_dec(&pp_counter_per_sub_job_count); + } + + /* PS: A change from MALI_HW_CORE_NO_COUNTER to MALI_HW_CORE_NO_COUNTER will inc and dec, result will be 0 change */ + + pp_counter_per_sub_job_src1[sub_job] = counter; +} + +u32 mali_pp_job_get_pp_counter_global_src0(void) +{ + return pp_counter_src0; +} + +u32 mali_pp_job_get_pp_counter_global_src1(void) +{ + return pp_counter_src1; +} + +u32 mali_pp_job_get_pp_counter_sub_job_src0(u32 sub_job) +{ + MALI_DEBUG_ASSERT(sub_job < _MALI_PP_MAX_SUB_JOBS); + return pp_counter_per_sub_job_src0[sub_job]; +} + +u32 mali_pp_job_get_pp_counter_sub_job_src1(u32 sub_job) +{ + MALI_DEBUG_ASSERT(sub_job < _MALI_PP_MAX_SUB_JOBS); + return pp_counter_per_sub_job_src1[sub_job]; +} diff --git a/drivers/gpu/arm/utgard/common/mali_pp_job.h b/drivers/gpu/arm/utgard/common/mali_pp_job.h new file mode 100644 index 00000000000000..7b9d2efa3019cd --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_pp_job.h @@ -0,0 +1,574 @@ +/* + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_PP_JOB_H__ +#define __MALI_PP_JOB_H__ + +#include "mali_osk.h" +#include "mali_osk_list.h" +#include "mali_uk_types.h" +#include "mali_session.h" +#include "mali_kernel_common.h" +#include "regs/mali_200_regs.h" +#include "mali_kernel_core.h" +#include "mali_dlbu.h" +#include "mali_timeline.h" +#include "mali_scheduler.h" +#include "mali_executor.h" +#if defined(CONFIG_DMA_SHARED_BUFFER) && !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) +#include "linux/mali_memory_dma_buf.h" +#endif + +typedef enum pp_job_status { + MALI_NO_SWAP_IN, + MALI_SWAP_IN_FAIL, + MALI_SWAP_IN_SUCC, +} pp_job_status; + +/** + * This structure represents a PP job, including all sub jobs. + * + * The PP job object itself is not protected by any single lock, + * but relies on other locks instead (scheduler, executor and timeline lock). + * Think of the job object as moving between these sub systems through-out + * its lifetime. Different part of the PP job struct is used by different + * subsystems. Accessor functions ensure that correct lock is taken. + * Do NOT access any data members directly from outside this module! + */ +struct mali_pp_job { + /* + * These members are typically only set at creation, + * and only read later on. + * They do not require any lock protection. + */ + _mali_uk_pp_start_job_s uargs; /**< Arguments from user space */ + struct mali_session_data *session; /**< Session which submitted this job */ + u32 pid; /**< Process ID of submitting process */ + u32 tid; /**< Thread ID of submitting thread */ + u32 id; /**< Identifier for this job in kernel space (sequential numbering) */ + u32 cache_order; /**< Cache order used for L2 cache flushing (sequential numbering) */ + struct mali_timeline_tracker tracker; /**< Timeline tracker for this job */ + _mali_osk_notification_t *finished_notification; /**< Notification sent back to userspace on job complete */ + u32 perf_counter_per_sub_job_count; /**< Number of values in the two arrays which is != MALI_HW_CORE_NO_COUNTER */ + u32 perf_counter_per_sub_job_src0[_MALI_PP_MAX_SUB_JOBS]; /**< Per sub job counters src0 */ + u32 perf_counter_per_sub_job_src1[_MALI_PP_MAX_SUB_JOBS]; /**< Per sub job counters src1 */ + u32 sub_jobs_num; /**< Number of subjobs; set to 1 for Mali-450 if DLBU is used, otherwise equals number of PP cores */ + + pp_job_status swap_status; /**< Used to track each PP job swap status, if fail, we need to drop them in scheduler part */ + mali_bool user_notification; /**< When we deferred delete PP job, we need to judge if we need to send job finish notification to user space */ + u32 num_pp_cores_in_virtual; /**< How many PP cores we have when job finished */ + + /* + * These members are used by both scheduler and executor. + * They are "protected" by atomic operations. + */ + _mali_osk_atomic_t sub_jobs_completed; /**< Number of completed sub-jobs in this superjob */ + _mali_osk_atomic_t sub_job_errors; /**< Bitfield with errors (errors for each single sub-job is or'ed together) */ + + /* + * These members are used by scheduler, but only when no one else + * knows about this job object but the working function. + * No lock is thus needed for these. + */ + u32 *memory_cookies; /**< Memory cookies attached to job */ + + /* + * These members are used by the scheduler, + * protected by scheduler lock + */ + _mali_osk_list_t list; /**< Used to link jobs together in the scheduler queue */ + _mali_osk_list_t session_fb_lookup_list; /**< Used to link jobs together from the same frame builder in the session */ + u32 sub_jobs_started; /**< Total number of sub-jobs started (always started in ascending order) */ + + /* + * Set by executor/group on job completion, read by scheduler when + * returning job to user. Hold executor lock when setting, + * no lock needed when reading + */ + u32 perf_counter_value0[_MALI_PP_MAX_SUB_JOBS]; /**< Value of performance counter 0 (to be returned to user space), one for each sub job */ + u32 perf_counter_value1[_MALI_PP_MAX_SUB_JOBS]; /**< Value of performance counter 1 (to be returned to user space), one for each sub job */ +}; + +void mali_pp_job_initialize(void); +void mali_pp_job_terminate(void); + +struct mali_pp_job *mali_pp_job_create(struct mali_session_data *session, _mali_uk_pp_start_job_s *uargs, u32 id); +void mali_pp_job_delete(struct mali_pp_job *job); + +u32 mali_pp_job_get_perf_counter_src0(struct mali_pp_job *job, u32 sub_job); +u32 mali_pp_job_get_perf_counter_src1(struct mali_pp_job *job, u32 sub_job); + +void mali_pp_job_set_pp_counter_global_src0(u32 counter); +void mali_pp_job_set_pp_counter_global_src1(u32 counter); +void mali_pp_job_set_pp_counter_sub_job_src0(u32 sub_job, u32 counter); +void mali_pp_job_set_pp_counter_sub_job_src1(u32 sub_job, u32 counter); + +u32 mali_pp_job_get_pp_counter_global_src0(void); +u32 mali_pp_job_get_pp_counter_global_src1(void); +u32 mali_pp_job_get_pp_counter_sub_job_src0(u32 sub_job); +u32 mali_pp_job_get_pp_counter_sub_job_src1(u32 sub_job); + +MALI_STATIC_INLINE u32 mali_pp_job_get_id(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return (NULL == job) ? 0 : job->id; +} + +MALI_STATIC_INLINE void mali_pp_job_set_cache_order(struct mali_pp_job *job, + u32 cache_order) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + job->cache_order = cache_order; +} + +MALI_STATIC_INLINE u32 mali_pp_job_get_cache_order(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return (NULL == job) ? 0 : job->cache_order; +} + +MALI_STATIC_INLINE u64 mali_pp_job_get_user_id(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.user_job_ptr; +} + +MALI_STATIC_INLINE u32 mali_pp_job_get_frame_builder_id(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.frame_builder_id; +} + +MALI_STATIC_INLINE u32 mali_pp_job_get_flush_id(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.flush_id; +} + +MALI_STATIC_INLINE u32 mali_pp_job_get_pid(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->pid; +} + +MALI_STATIC_INLINE u32 mali_pp_job_get_tid(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->tid; +} + +MALI_STATIC_INLINE u32 *mali_pp_job_get_frame_registers(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.frame_registers; +} + +MALI_STATIC_INLINE u32 *mali_pp_job_get_dlbu_registers(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.dlbu_registers; +} + +MALI_STATIC_INLINE mali_bool mali_pp_job_is_virtual(struct mali_pp_job *job) +{ +#if (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) + MALI_DEBUG_ASSERT_POINTER(job); + return (0 == job->uargs.num_cores) ? MALI_TRUE : MALI_FALSE; +#else + return MALI_FALSE; +#endif +} + +MALI_STATIC_INLINE u32 mali_pp_job_get_addr_frame(struct mali_pp_job *job, u32 sub_job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + + if (mali_pp_job_is_virtual(job)) { + return MALI_DLBU_VIRT_ADDR; + } else if (0 == sub_job) { + return job->uargs.frame_registers[MALI200_REG_ADDR_FRAME / sizeof(u32)]; + } else if (sub_job < _MALI_PP_MAX_SUB_JOBS) { + return job->uargs.frame_registers_addr_frame[sub_job - 1]; + } + + return 0; +} + +MALI_STATIC_INLINE u32 mali_pp_job_get_addr_stack(struct mali_pp_job *job, u32 sub_job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + + if (0 == sub_job) { + return job->uargs.frame_registers[MALI200_REG_ADDR_STACK / sizeof(u32)]; + } else if (sub_job < _MALI_PP_MAX_SUB_JOBS) { + return job->uargs.frame_registers_addr_stack[sub_job - 1]; + } + + return 0; +} + +void mali_pp_job_list_add(struct mali_pp_job *job, _mali_osk_list_t *list); + +MALI_STATIC_INLINE void mali_pp_job_list_addtail(struct mali_pp_job *job, + _mali_osk_list_t *list) +{ + _mali_osk_list_addtail(&job->list, list); +} + +MALI_STATIC_INLINE void mali_pp_job_list_move(struct mali_pp_job *job, + _mali_osk_list_t *list) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + MALI_DEBUG_ASSERT(!_mali_osk_list_empty(&job->list)); + _mali_osk_list_move(&job->list, list); +} + +MALI_STATIC_INLINE void mali_pp_job_list_remove(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + _mali_osk_list_delinit(&job->list); +} + +MALI_STATIC_INLINE u32 *mali_pp_job_get_wb0_registers(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.wb0_registers; +} + +MALI_STATIC_INLINE u32 *mali_pp_job_get_wb1_registers(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.wb1_registers; +} + +MALI_STATIC_INLINE u32 *mali_pp_job_get_wb2_registers(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.wb2_registers; +} + +MALI_STATIC_INLINE u32 mali_pp_job_get_wb0_source_addr(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.wb0_registers[MALI200_REG_ADDR_WB_SOURCE_ADDR / sizeof(u32)]; +} + +MALI_STATIC_INLINE u32 mali_pp_job_get_wb1_source_addr(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.wb1_registers[MALI200_REG_ADDR_WB_SOURCE_ADDR / sizeof(u32)]; +} + +MALI_STATIC_INLINE u32 mali_pp_job_get_wb2_source_addr(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.wb2_registers[MALI200_REG_ADDR_WB_SOURCE_ADDR / sizeof(u32)]; +} + +MALI_STATIC_INLINE void mali_pp_job_disable_wb0(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + job->uargs.wb0_registers[MALI200_REG_ADDR_WB_SOURCE_SELECT] = 0; +} + +MALI_STATIC_INLINE void mali_pp_job_disable_wb1(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + job->uargs.wb1_registers[MALI200_REG_ADDR_WB_SOURCE_SELECT] = 0; +} + +MALI_STATIC_INLINE void mali_pp_job_disable_wb2(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + job->uargs.wb2_registers[MALI200_REG_ADDR_WB_SOURCE_SELECT] = 0; +} + +MALI_STATIC_INLINE mali_bool mali_pp_job_all_writeback_unit_disabled(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + + if (job->uargs.wb0_registers[MALI200_REG_ADDR_WB_SOURCE_SELECT] || + job->uargs.wb1_registers[MALI200_REG_ADDR_WB_SOURCE_SELECT] || + job->uargs.wb2_registers[MALI200_REG_ADDR_WB_SOURCE_SELECT] + ) { + /* At least one output unit active */ + return MALI_FALSE; + } + + /* All outputs are disabled - we can abort the job */ + return MALI_TRUE; +} + +MALI_STATIC_INLINE void mali_pp_job_fb_lookup_add(struct mali_pp_job *job) +{ + u32 fb_lookup_id; + + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + + fb_lookup_id = MALI_PP_JOB_FB_LOOKUP_LIST_MASK & job->uargs.frame_builder_id; + + MALI_DEBUG_ASSERT(MALI_PP_JOB_FB_LOOKUP_LIST_SIZE > fb_lookup_id); + + _mali_osk_list_addtail(&job->session_fb_lookup_list, + &job->session->pp_job_fb_lookup_list[fb_lookup_id]); +} + +MALI_STATIC_INLINE void mali_pp_job_fb_lookup_remove(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + _mali_osk_list_delinit(&job->session_fb_lookup_list); +} + +MALI_STATIC_INLINE struct mali_session_data *mali_pp_job_get_session(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->session; +} + +MALI_STATIC_INLINE mali_bool mali_pp_job_has_started_sub_jobs(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + return (0 < job->sub_jobs_started) ? MALI_TRUE : MALI_FALSE; +} + +MALI_STATIC_INLINE mali_bool mali_pp_job_has_unstarted_sub_jobs(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + return (job->sub_jobs_started < job->sub_jobs_num) ? MALI_TRUE : MALI_FALSE; +} + +/* Function used when we are terminating a session with jobs. Return TRUE if it has a rendering job. + Makes sure that no new subjobs are started. */ +MALI_STATIC_INLINE void mali_pp_job_mark_unstarted_failed(struct mali_pp_job *job) +{ + u32 jobs_remaining; + u32 i; + + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + + jobs_remaining = job->sub_jobs_num - job->sub_jobs_started; + job->sub_jobs_started += jobs_remaining; + + /* Not the most optimal way, but this is only used in error cases */ + for (i = 0; i < jobs_remaining; i++) { + _mali_osk_atomic_inc(&job->sub_jobs_completed); + _mali_osk_atomic_inc(&job->sub_job_errors); + } +} + +MALI_STATIC_INLINE mali_bool mali_pp_job_is_complete(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return (job->sub_jobs_num == + _mali_osk_atomic_read(&job->sub_jobs_completed)) ? + MALI_TRUE : MALI_FALSE; +} + +MALI_STATIC_INLINE u32 mali_pp_job_get_first_unstarted_sub_job(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + return job->sub_jobs_started; +} + +MALI_STATIC_INLINE u32 mali_pp_job_get_sub_job_count(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->sub_jobs_num; +} + +MALI_STATIC_INLINE u32 mali_pp_job_unstarted_sub_job_count(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + MALI_DEBUG_ASSERT(job->sub_jobs_num >= job->sub_jobs_started); + return (job->sub_jobs_num - job->sub_jobs_started); +} + +MALI_STATIC_INLINE u32 mali_pp_job_num_memory_cookies(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.num_memory_cookies; +} + +MALI_STATIC_INLINE u32 mali_pp_job_get_memory_cookie( + struct mali_pp_job *job, u32 index) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT(index < job->uargs.num_memory_cookies); + MALI_DEBUG_ASSERT_POINTER(job->memory_cookies); + return job->memory_cookies[index]; +} + +MALI_STATIC_INLINE mali_bool mali_pp_job_needs_dma_buf_mapping(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + + if (0 < job->uargs.num_memory_cookies) { + return MALI_TRUE; + } + + return MALI_FALSE; +} + +MALI_STATIC_INLINE void mali_pp_job_mark_sub_job_started(struct mali_pp_job *job, u32 sub_job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + + /* Assert that we are marking the "first unstarted sub job" as started */ + MALI_DEBUG_ASSERT(job->sub_jobs_started == sub_job); + + job->sub_jobs_started++; +} + +MALI_STATIC_INLINE void mali_pp_job_mark_sub_job_completed(struct mali_pp_job *job, mali_bool success) +{ + MALI_DEBUG_ASSERT_POINTER(job); + + _mali_osk_atomic_inc(&job->sub_jobs_completed); + if (MALI_FALSE == success) { + _mali_osk_atomic_inc(&job->sub_job_errors); + } +} + +MALI_STATIC_INLINE mali_bool mali_pp_job_was_success(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + if (0 == _mali_osk_atomic_read(&job->sub_job_errors)) { + return MALI_TRUE; + } + return MALI_FALSE; +} + +MALI_STATIC_INLINE mali_bool mali_pp_job_use_no_notification( + struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return (job->uargs.flags & _MALI_PP_JOB_FLAG_NO_NOTIFICATION) ? + MALI_TRUE : MALI_FALSE; +} + +MALI_STATIC_INLINE mali_bool mali_pp_job_is_pilot_job(struct mali_pp_job *job) +{ + /* + * A pilot job is currently identified as jobs which + * require no callback notification. + */ + return mali_pp_job_use_no_notification(job); +} + +MALI_STATIC_INLINE _mali_osk_notification_t * +mali_pp_job_get_finished_notification(struct mali_pp_job *job) +{ + _mali_osk_notification_t *notification; + + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_POINTER(job->finished_notification); + + notification = job->finished_notification; + job->finished_notification = NULL; + + return notification; +} + +MALI_STATIC_INLINE mali_bool mali_pp_job_is_window_surface( + struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return (job->uargs.flags & _MALI_PP_JOB_FLAG_IS_WINDOW_SURFACE) + ? MALI_TRUE : MALI_FALSE; +} + +MALI_STATIC_INLINE u32 mali_pp_job_get_perf_counter_flag(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.perf_counter_flag; +} + +MALI_STATIC_INLINE u32 mali_pp_job_get_perf_counter_value0(struct mali_pp_job *job, u32 sub_job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->perf_counter_value0[sub_job]; +} + +MALI_STATIC_INLINE u32 mali_pp_job_get_perf_counter_value1(struct mali_pp_job *job, u32 sub_job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->perf_counter_value1[sub_job]; +} + +MALI_STATIC_INLINE void mali_pp_job_set_perf_counter_value0(struct mali_pp_job *job, u32 sub_job, u32 value) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + job->perf_counter_value0[sub_job] = value; +} + +MALI_STATIC_INLINE void mali_pp_job_set_perf_counter_value1(struct mali_pp_job *job, u32 sub_job, u32 value) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + job->perf_counter_value1[sub_job] = value; +} + +MALI_STATIC_INLINE _mali_osk_errcode_t mali_pp_job_check(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + if (mali_pp_job_is_virtual(job) && job->sub_jobs_num != 1) { + return _MALI_OSK_ERR_FAULT; + } + return _MALI_OSK_ERR_OK; +} + +/** + * Returns MALI_TRUE if this job has more than two sub jobs and all sub jobs are unstarted. + * + * @param job Job to check. + * @return MALI_TRUE if job has more than two sub jobs and all sub jobs are unstarted, MALI_FALSE if not. + */ +MALI_STATIC_INLINE mali_bool mali_pp_job_is_large_and_unstarted(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + MALI_DEBUG_ASSERT(!mali_pp_job_is_virtual(job)); + + return (0 == job->sub_jobs_started && 2 < job->sub_jobs_num); +} + +/** + * Get PP job's Timeline tracker. + * + * @param job PP job. + * @return Pointer to Timeline tracker for the job. + */ +MALI_STATIC_INLINE struct mali_timeline_tracker *mali_pp_job_get_tracker(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return &(job->tracker); +} + +MALI_STATIC_INLINE u32 *mali_pp_job_get_timeline_point_ptr( + struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return (u32 __user *)(uintptr_t)job->uargs.timeline_point_ptr; +} + + +#endif /* __MALI_PP_JOB_H__ */ diff --git a/drivers/gpu/arm/utgard/common/mali_scheduler.c b/drivers/gpu/arm/utgard/common/mali_scheduler.c new file mode 100644 index 00000000000000..5547159db94cdd --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_scheduler.c @@ -0,0 +1,1354 @@ +/* + * Copyright (C) 2012-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_scheduler.h" +#include "mali_kernel_common.h" +#include "mali_osk.h" +#include "mali_osk_profiling.h" +#include "mali_kernel_utilization.h" +#include "mali_timeline.h" +#include "mali_gp_job.h" +#include "mali_pp_job.h" +#include "mali_executor.h" +#include "mali_group.h" +#include +#include + + +#if defined(CONFIG_DMA_SHARED_BUFFER) +#include "mali_memory_dma_buf.h" +#endif + +#if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS) +#include +#include +#endif +/* + * ---------- static defines/constants ---------- + */ + +/* + * If dma_buf with map on demand is used, we defer job queue + * if in atomic context, since both might sleep. + */ +#if defined(CONFIG_DMA_SHARED_BUFFER) +#if !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) +#define MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE 1 +#endif +#endif + + +/* + * ---------- global variables (exported due to inline functions) ---------- + */ + +/* Lock protecting this module */ +_mali_osk_spinlock_irq_t *mali_scheduler_lock_obj = NULL; + +/* Queue of jobs to be executed on the GP group */ +struct mali_scheduler_job_queue job_queue_gp; + +/* Queue of PP jobs */ +struct mali_scheduler_job_queue job_queue_pp; + +_mali_osk_atomic_t mali_job_id_autonumber; +_mali_osk_atomic_t mali_job_cache_order_autonumber; +/* + * ---------- static variables ---------- + */ + +_mali_osk_wq_work_t *scheduler_wq_pp_job_delete = NULL; +_mali_osk_spinlock_irq_t *scheduler_pp_job_delete_lock = NULL; +static _MALI_OSK_LIST_HEAD_STATIC_INIT(scheduler_pp_job_deletion_queue); + +#if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) +static _mali_osk_wq_work_t *scheduler_wq_pp_job_queue = NULL; +static _mali_osk_spinlock_irq_t *scheduler_pp_job_queue_lock = NULL; +static _MALI_OSK_LIST_HEAD_STATIC_INIT(scheduler_pp_job_queue_list); +#endif + +/* + * ---------- Forward declaration of static functions ---------- + */ + +static mali_timeline_point mali_scheduler_submit_gp_job( + struct mali_session_data *session, struct mali_gp_job *job); +static mali_timeline_point mali_scheduler_submit_pp_job( + struct mali_session_data *session, struct mali_pp_job *job); + +static mali_bool mali_scheduler_queue_gp_job(struct mali_gp_job *job); +static mali_bool mali_scheduler_queue_pp_job(struct mali_pp_job *job); + +static void mali_scheduler_return_gp_job_to_user(struct mali_gp_job *job, + mali_bool success); + +static void mali_scheduler_deferred_pp_job_delete(struct mali_pp_job *job); +void mali_scheduler_do_pp_job_delete(void *arg); + +#if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) +static void mali_scheduler_deferred_pp_job_queue(struct mali_pp_job *job); +static void mali_scheduler_do_pp_job_queue(void *arg); +#endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */ + +/* + * ---------- Actual implementation ---------- + */ + +_mali_osk_errcode_t mali_scheduler_initialize(void) +{ + _mali_osk_atomic_init(&mali_job_id_autonumber, 0); + _mali_osk_atomic_init(&mali_job_cache_order_autonumber, 0); + + _MALI_OSK_INIT_LIST_HEAD(&job_queue_gp.normal_pri); + _MALI_OSK_INIT_LIST_HEAD(&job_queue_gp.high_pri); + job_queue_gp.depth = 0; + job_queue_gp.big_job_num = 0; + + _MALI_OSK_INIT_LIST_HEAD(&job_queue_pp.normal_pri); + _MALI_OSK_INIT_LIST_HEAD(&job_queue_pp.high_pri); + job_queue_pp.depth = 0; + job_queue_pp.big_job_num = 0; + + mali_scheduler_lock_obj = _mali_osk_spinlock_irq_init( + _MALI_OSK_LOCKFLAG_ORDERED, + _MALI_OSK_LOCK_ORDER_SCHEDULER); + if (NULL == mali_scheduler_lock_obj) { + mali_scheduler_terminate(); + } + + scheduler_wq_pp_job_delete = _mali_osk_wq_create_work( + mali_scheduler_do_pp_job_delete, NULL); + if (NULL == scheduler_wq_pp_job_delete) { + mali_scheduler_terminate(); + return _MALI_OSK_ERR_FAULT; + } + + scheduler_pp_job_delete_lock = _mali_osk_spinlock_irq_init( + _MALI_OSK_LOCKFLAG_ORDERED, + _MALI_OSK_LOCK_ORDER_SCHEDULER_DEFERRED); + if (NULL == scheduler_pp_job_delete_lock) { + mali_scheduler_terminate(); + return _MALI_OSK_ERR_FAULT; + } + +#if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) + scheduler_wq_pp_job_queue = _mali_osk_wq_create_work( + mali_scheduler_do_pp_job_queue, NULL); + if (NULL == scheduler_wq_pp_job_queue) { + mali_scheduler_terminate(); + return _MALI_OSK_ERR_FAULT; + } + + scheduler_pp_job_queue_lock = _mali_osk_spinlock_irq_init( + _MALI_OSK_LOCKFLAG_ORDERED, + _MALI_OSK_LOCK_ORDER_SCHEDULER_DEFERRED); + if (NULL == scheduler_pp_job_queue_lock) { + mali_scheduler_terminate(); + return _MALI_OSK_ERR_FAULT; + } +#endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */ + + return _MALI_OSK_ERR_OK; +} + +void mali_scheduler_terminate(void) +{ +#if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) + if (NULL != scheduler_pp_job_queue_lock) { + _mali_osk_spinlock_irq_term(scheduler_pp_job_queue_lock); + scheduler_pp_job_queue_lock = NULL; + } + + if (NULL != scheduler_wq_pp_job_queue) { + _mali_osk_wq_delete_work(scheduler_wq_pp_job_queue); + scheduler_wq_pp_job_queue = NULL; + } +#endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */ + + if (NULL != scheduler_pp_job_delete_lock) { + _mali_osk_spinlock_irq_term(scheduler_pp_job_delete_lock); + scheduler_pp_job_delete_lock = NULL; + } + + if (NULL != scheduler_wq_pp_job_delete) { + _mali_osk_wq_delete_work(scheduler_wq_pp_job_delete); + scheduler_wq_pp_job_delete = NULL; + } + + if (NULL != mali_scheduler_lock_obj) { + _mali_osk_spinlock_irq_term(mali_scheduler_lock_obj); + mali_scheduler_lock_obj = NULL; + } + + _mali_osk_atomic_term(&mali_job_cache_order_autonumber); + _mali_osk_atomic_term(&mali_job_id_autonumber); +} + +u32 mali_scheduler_job_physical_head_count(void) +{ + /* + * Count how many physical sub jobs are present from the head of queue + * until the first virtual job is present. + * Early out when we have reached maximum number of PP cores (8) + */ + u32 count = 0; + struct mali_pp_job *job; + struct mali_pp_job *temp; + + /* Check for partially started normal pri jobs */ + if (!_mali_osk_list_empty(&job_queue_pp.normal_pri)) { + MALI_DEBUG_ASSERT(0 < job_queue_pp.depth); + + job = _MALI_OSK_LIST_ENTRY(job_queue_pp.normal_pri.next, + struct mali_pp_job, list); + + MALI_DEBUG_ASSERT_POINTER(job); + + if (MALI_TRUE == mali_pp_job_has_started_sub_jobs(job)) { + /* + * Remember; virtual jobs can't be queued and started + * at the same time, so this must be a physical job + */ + count += mali_pp_job_unstarted_sub_job_count(job); + if (MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS <= count) { + return MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS; + } + } + } + + _MALI_OSK_LIST_FOREACHENTRY(job, temp, &job_queue_pp.high_pri, + struct mali_pp_job, list) { + if (MALI_FALSE == mali_pp_job_is_virtual(job)) { + count += mali_pp_job_unstarted_sub_job_count(job); + if (MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS <= count) { + return MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS; + } + } else { + /* Came across a virtual job, so stop counting */ + return count; + } + } + + _MALI_OSK_LIST_FOREACHENTRY(job, temp, &job_queue_pp.normal_pri, + struct mali_pp_job, list) { + if (MALI_FALSE == mali_pp_job_is_virtual(job)) { + /* any partially started is already counted */ + if (MALI_FALSE == mali_pp_job_has_started_sub_jobs(job)) { + count += mali_pp_job_unstarted_sub_job_count(job); + if (MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS <= + count) { + return MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS; + } + } + } else { + /* Came across a virtual job, so stop counting */ + return count; + } + } + + return count; +} + +mali_bool mali_scheduler_job_next_is_virtual(void) +{ + struct mali_pp_job *job; + + job = mali_scheduler_job_pp_virtual_peek(); + if (NULL != job) { + MALI_DEBUG_ASSERT(mali_pp_job_is_virtual(job)); + + return MALI_TRUE; + } + + return MALI_FALSE; +} + +struct mali_gp_job *mali_scheduler_job_gp_get(void) +{ + _mali_osk_list_t *queue; + struct mali_gp_job *job = NULL; + + MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj); + MALI_DEBUG_ASSERT(0 < job_queue_gp.depth); + MALI_DEBUG_ASSERT(job_queue_gp.big_job_num <= job_queue_gp.depth); + + if (!_mali_osk_list_empty(&job_queue_gp.high_pri)) { + queue = &job_queue_gp.high_pri; + } else { + queue = &job_queue_gp.normal_pri; + MALI_DEBUG_ASSERT(!_mali_osk_list_empty(queue)); + } + + job = _MALI_OSK_LIST_ENTRY(queue->next, struct mali_gp_job, list); + + MALI_DEBUG_ASSERT_POINTER(job); + + mali_gp_job_list_remove(job); + job_queue_gp.depth--; + if (job->big_job) { + job_queue_gp.big_job_num --; + if (job_queue_gp.big_job_num < MALI_MAX_PENDING_BIG_JOB) { + /* wake up process */ + wait_queue_head_t *queue = mali_session_get_wait_queue(); + wake_up(queue); + } + } + return job; +} + +struct mali_pp_job *mali_scheduler_job_pp_physical_peek(void) +{ + struct mali_pp_job *job = NULL; + struct mali_pp_job *tmp_job = NULL; + + MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj); + + /* + * For PP jobs we favour partially started jobs in normal + * priority queue over unstarted jobs in high priority queue + */ + + if (!_mali_osk_list_empty(&job_queue_pp.normal_pri)) { + MALI_DEBUG_ASSERT(0 < job_queue_pp.depth); + + tmp_job = _MALI_OSK_LIST_ENTRY(job_queue_pp.normal_pri.next, + struct mali_pp_job, list); + MALI_DEBUG_ASSERT(NULL != tmp_job); + + if (MALI_FALSE == mali_pp_job_is_virtual(tmp_job)) { + job = tmp_job; + } + } + + if (NULL == job || + MALI_FALSE == mali_pp_job_has_started_sub_jobs(job)) { + /* + * There isn't a partially started job in normal queue, so + * look in high priority queue. + */ + if (!_mali_osk_list_empty(&job_queue_pp.high_pri)) { + MALI_DEBUG_ASSERT(0 < job_queue_pp.depth); + + tmp_job = _MALI_OSK_LIST_ENTRY(job_queue_pp.high_pri.next, + struct mali_pp_job, list); + MALI_DEBUG_ASSERT(NULL != tmp_job); + + if (MALI_FALSE == mali_pp_job_is_virtual(tmp_job)) { + job = tmp_job; + } + } + } + + return job; +} + +struct mali_pp_job *mali_scheduler_job_pp_virtual_peek(void) +{ + struct mali_pp_job *job = NULL; + struct mali_pp_job *tmp_job = NULL; + + MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj); + + if (!_mali_osk_list_empty(&job_queue_pp.high_pri)) { + MALI_DEBUG_ASSERT(0 < job_queue_pp.depth); + + tmp_job = _MALI_OSK_LIST_ENTRY(job_queue_pp.high_pri.next, + struct mali_pp_job, list); + + if (MALI_TRUE == mali_pp_job_is_virtual(tmp_job)) { + job = tmp_job; + } + } + + if (NULL == job) { + if (!_mali_osk_list_empty(&job_queue_pp.normal_pri)) { + MALI_DEBUG_ASSERT(0 < job_queue_pp.depth); + + tmp_job = _MALI_OSK_LIST_ENTRY(job_queue_pp.normal_pri.next, + struct mali_pp_job, list); + + if (MALI_TRUE == mali_pp_job_is_virtual(tmp_job)) { + job = tmp_job; + } + } + } + + return job; +} + +struct mali_pp_job *mali_scheduler_job_pp_physical_get(u32 *sub_job) +{ + struct mali_pp_job *job = mali_scheduler_job_pp_physical_peek(); + + MALI_DEBUG_ASSERT(MALI_FALSE == mali_pp_job_is_virtual(job)); + + if (NULL != job) { + *sub_job = mali_pp_job_get_first_unstarted_sub_job(job); + + mali_pp_job_mark_sub_job_started(job, *sub_job); + if (MALI_FALSE == mali_pp_job_has_unstarted_sub_jobs(job)) { + /* Remove from queue when last sub job has been retrieved */ + mali_pp_job_list_remove(job); + } + + job_queue_pp.depth--; + + /* + * Job about to start so it is no longer be + * possible to discard WB + */ + mali_pp_job_fb_lookup_remove(job); + } + + return job; +} + +struct mali_pp_job *mali_scheduler_job_pp_virtual_get(void) +{ + struct mali_pp_job *job = mali_scheduler_job_pp_virtual_peek(); + + MALI_DEBUG_ASSERT(MALI_TRUE == mali_pp_job_is_virtual(job)); + + if (NULL != job) { + MALI_DEBUG_ASSERT(0 == + mali_pp_job_get_first_unstarted_sub_job(job)); + MALI_DEBUG_ASSERT(1 == + mali_pp_job_get_sub_job_count(job)); + + mali_pp_job_mark_sub_job_started(job, 0); + + mali_pp_job_list_remove(job); + + job_queue_pp.depth--; + + /* + * Job about to start so it is no longer be + * possible to discard WB + */ + mali_pp_job_fb_lookup_remove(job); + } + + return job; +} + +mali_scheduler_mask mali_scheduler_activate_gp_job(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + + MALI_DEBUG_PRINT(4, ("Mali GP scheduler: Timeline activation for job %u (0x%08X).\n", + mali_gp_job_get_id(job), job)); + + mali_scheduler_lock(); + + if (!mali_scheduler_queue_gp_job(job)) { + /* Failed to enqueue job, release job (with error) */ + + mali_scheduler_unlock(); + + mali_timeline_tracker_release(mali_gp_job_get_tracker(job)); + mali_gp_job_signal_pp_tracker(job, MALI_FALSE); + + /* This will notify user space and close the job object */ + mali_scheduler_complete_gp_job(job, MALI_FALSE, + MALI_TRUE, MALI_FALSE); + + return MALI_SCHEDULER_MASK_EMPTY; + } + + mali_scheduler_unlock(); + + return MALI_SCHEDULER_MASK_GP; +} + +mali_scheduler_mask mali_scheduler_activate_pp_job(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + + MALI_DEBUG_PRINT(4, ("Mali PP scheduler: Timeline activation for job %u (0x%08X).\n", + mali_pp_job_get_id(job), job)); + + if (MALI_TRUE == mali_timeline_tracker_activation_error( + mali_pp_job_get_tracker(job))) { + MALI_DEBUG_PRINT(3, ("Mali PP scheduler: Job %u (0x%08X) activated with error, aborting.\n", + mali_pp_job_get_id(job), job)); + + mali_scheduler_lock(); + mali_pp_job_fb_lookup_remove(job); + mali_pp_job_mark_unstarted_failed(job); + mali_scheduler_unlock(); + + mali_timeline_tracker_release(mali_pp_job_get_tracker(job)); + + /* This will notify user space and close the job object */ + mali_scheduler_complete_pp_job(job, 0, MALI_TRUE, MALI_FALSE); + + return MALI_SCHEDULER_MASK_EMPTY; + } + +#if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) + if (mali_pp_job_needs_dma_buf_mapping(job)) { + mali_scheduler_deferred_pp_job_queue(job); + return MALI_SCHEDULER_MASK_EMPTY; + } +#endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */ + + mali_scheduler_lock(); + + if (!mali_scheduler_queue_pp_job(job)) { + /* Failed to enqueue job, release job (with error) */ + mali_pp_job_fb_lookup_remove(job); + mali_pp_job_mark_unstarted_failed(job); + mali_scheduler_unlock(); + + mali_timeline_tracker_release(mali_pp_job_get_tracker(job)); + + /* This will notify user space and close the job object */ + mali_scheduler_complete_pp_job(job, 0, MALI_TRUE, MALI_FALSE); + + return MALI_SCHEDULER_MASK_EMPTY; + } + + mali_scheduler_unlock(); + return MALI_SCHEDULER_MASK_PP; +} + +void mali_scheduler_complete_gp_job(struct mali_gp_job *job, + mali_bool success, + mali_bool user_notification, + mali_bool dequeued) +{ + if (user_notification) { + mali_scheduler_return_gp_job_to_user(job, success); + } + + if (dequeued) { + _mali_osk_pm_dev_ref_put(); + + if (mali_utilization_enabled()) { + mali_utilization_gp_end(); + } + } + + mali_gp_job_delete(job); +} + +void mali_scheduler_complete_pp_job(struct mali_pp_job *job, + u32 num_cores_in_virtual, + mali_bool user_notification, + mali_bool dequeued) +{ + job->user_notification = user_notification; + job->num_pp_cores_in_virtual = num_cores_in_virtual; + + if (dequeued) { +#if defined(CONFIG_MALI_DVFS) + if (mali_pp_job_is_window_surface(job)) { + struct mali_session_data *session; + session = mali_pp_job_get_session(job); + mali_session_inc_num_window_jobs(session); + } +#endif + + _mali_osk_pm_dev_ref_put(); + + if (mali_utilization_enabled()) { + mali_utilization_pp_end(); + } + } + + /* With ZRAM feature enabled, all pp jobs will be force to use deferred delete. */ + mali_scheduler_deferred_pp_job_delete(job); +} + +void mali_scheduler_abort_session(struct mali_session_data *session) +{ + struct mali_gp_job *gp_job; + struct mali_gp_job *gp_tmp; + struct mali_pp_job *pp_job; + struct mali_pp_job *pp_tmp; + _MALI_OSK_LIST_HEAD_STATIC_INIT(removed_jobs_gp); + _MALI_OSK_LIST_HEAD_STATIC_INIT(removed_jobs_pp); + + MALI_DEBUG_ASSERT_POINTER(session); + MALI_DEBUG_ASSERT(session->is_aborting); + + MALI_DEBUG_PRINT(3, ("Mali scheduler: Aborting all queued jobs from session 0x%08X.\n", + session)); + + mali_scheduler_lock(); + + /* Remove from GP normal priority queue */ + _MALI_OSK_LIST_FOREACHENTRY(gp_job, gp_tmp, &job_queue_gp.normal_pri, + struct mali_gp_job, list) { + if (mali_gp_job_get_session(gp_job) == session) { + mali_gp_job_list_move(gp_job, &removed_jobs_gp); + job_queue_gp.depth--; + job_queue_gp.big_job_num -= gp_job->big_job ? 1 : 0; + } + } + + /* Remove from GP high priority queue */ + _MALI_OSK_LIST_FOREACHENTRY(gp_job, gp_tmp, &job_queue_gp.high_pri, + struct mali_gp_job, list) { + if (mali_gp_job_get_session(gp_job) == session) { + mali_gp_job_list_move(gp_job, &removed_jobs_gp); + job_queue_gp.depth--; + job_queue_gp.big_job_num -= gp_job->big_job ? 1 : 0; + } + } + + /* Remove from PP normal priority queue */ + _MALI_OSK_LIST_FOREACHENTRY(pp_job, pp_tmp, + &job_queue_pp.normal_pri, + struct mali_pp_job, list) { + if (mali_pp_job_get_session(pp_job) == session) { + mali_pp_job_fb_lookup_remove(pp_job); + + job_queue_pp.depth -= + mali_pp_job_unstarted_sub_job_count( + pp_job); + mali_pp_job_mark_unstarted_failed(pp_job); + + if (MALI_FALSE == mali_pp_job_has_unstarted_sub_jobs(pp_job)) { + if (mali_pp_job_is_complete(pp_job)) { + mali_pp_job_list_move(pp_job, + &removed_jobs_pp); + } else { + mali_pp_job_list_remove(pp_job); + } + } + } + } + + /* Remove from PP high priority queue */ + _MALI_OSK_LIST_FOREACHENTRY(pp_job, pp_tmp, + &job_queue_pp.high_pri, + struct mali_pp_job, list) { + if (mali_pp_job_get_session(pp_job) == session) { + mali_pp_job_fb_lookup_remove(pp_job); + + job_queue_pp.depth -= + mali_pp_job_unstarted_sub_job_count( + pp_job); + mali_pp_job_mark_unstarted_failed(pp_job); + + if (MALI_FALSE == mali_pp_job_has_unstarted_sub_jobs(pp_job)) { + if (mali_pp_job_is_complete(pp_job)) { + mali_pp_job_list_move(pp_job, + &removed_jobs_pp); + } else { + mali_pp_job_list_remove(pp_job); + } + } + } + } + + /* + * Release scheduler lock so we can release trackers + * (which will potentially queue new jobs) + */ + mali_scheduler_unlock(); + + /* Release and complete all (non-running) found GP jobs */ + _MALI_OSK_LIST_FOREACHENTRY(gp_job, gp_tmp, &removed_jobs_gp, + struct mali_gp_job, list) { + mali_timeline_tracker_release(mali_gp_job_get_tracker(gp_job)); + mali_gp_job_signal_pp_tracker(gp_job, MALI_FALSE); + _mali_osk_list_delinit(&gp_job->list); + mali_scheduler_complete_gp_job(gp_job, + MALI_FALSE, MALI_FALSE, MALI_TRUE); + } + + /* Release and complete non-running PP jobs */ + _MALI_OSK_LIST_FOREACHENTRY(pp_job, pp_tmp, &removed_jobs_pp, + struct mali_pp_job, list) { + mali_timeline_tracker_release(mali_pp_job_get_tracker(pp_job)); + _mali_osk_list_delinit(&pp_job->list); + mali_scheduler_complete_pp_job(pp_job, 0, + MALI_FALSE, MALI_TRUE); + } +} + +_mali_osk_errcode_t _mali_ukk_gp_start_job(void *ctx, + _mali_uk_gp_start_job_s *uargs) +{ + struct mali_session_data *session; + struct mali_gp_job *job; + mali_timeline_point point; + u32 __user *point_ptr = NULL; + + MALI_DEBUG_ASSERT_POINTER(uargs); + MALI_DEBUG_ASSERT_POINTER(ctx); + + session = (struct mali_session_data *)(uintptr_t)ctx; + + job = mali_gp_job_create(session, uargs, mali_scheduler_get_new_id(), + NULL); + if (NULL == job) { + MALI_PRINT_ERROR(("Failed to create GP job.\n")); + return _MALI_OSK_ERR_NOMEM; + } + + point_ptr = (u32 __user *)(uintptr_t)mali_gp_job_get_timeline_point_ptr(job); + + point = mali_scheduler_submit_gp_job(session, job); + + if (0 != _mali_osk_put_user(((u32) point), point_ptr)) { + /* + * Let user space know that something failed + * after the job was started. + */ + return _MALI_OSK_ERR_ITEM_NOT_FOUND; + } + + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t _mali_ukk_pp_start_job(void *ctx, + _mali_uk_pp_start_job_s *uargs) +{ + struct mali_session_data *session; + struct mali_pp_job *job; + mali_timeline_point point; + u32 __user *point_ptr = NULL; + + MALI_DEBUG_ASSERT_POINTER(uargs); + MALI_DEBUG_ASSERT_POINTER(ctx); + + session = (struct mali_session_data *)(uintptr_t)ctx; + + job = mali_pp_job_create(session, uargs, mali_scheduler_get_new_id()); + if (NULL == job) { + MALI_PRINT_ERROR(("Failed to create PP job.\n")); + return _MALI_OSK_ERR_NOMEM; + } + + point_ptr = (u32 __user *)(uintptr_t)mali_pp_job_get_timeline_point_ptr(job); + + point = mali_scheduler_submit_pp_job(session, job); + job = NULL; + + if (0 != _mali_osk_put_user(((u32) point), point_ptr)) { + /* + * Let user space know that something failed + * after the job was started. + */ + return _MALI_OSK_ERR_ITEM_NOT_FOUND; + } + + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t _mali_ukk_pp_and_gp_start_job(void *ctx, + _mali_uk_pp_and_gp_start_job_s *uargs) +{ + struct mali_session_data *session; + _mali_uk_pp_and_gp_start_job_s kargs; + struct mali_pp_job *pp_job; + struct mali_gp_job *gp_job; + u32 __user *point_ptr = NULL; + mali_timeline_point point; + _mali_uk_pp_start_job_s __user *pp_args; + _mali_uk_gp_start_job_s __user *gp_args; + + MALI_DEBUG_ASSERT_POINTER(ctx); + MALI_DEBUG_ASSERT_POINTER(uargs); + + session = (struct mali_session_data *) ctx; + + if (0 != _mali_osk_copy_from_user(&kargs, uargs, + sizeof(_mali_uk_pp_and_gp_start_job_s))) { + return _MALI_OSK_ERR_NOMEM; + } + + pp_args = (_mali_uk_pp_start_job_s __user *)(uintptr_t)kargs.pp_args; + gp_args = (_mali_uk_gp_start_job_s __user *)(uintptr_t)kargs.gp_args; + + pp_job = mali_pp_job_create(session, pp_args, + mali_scheduler_get_new_id()); + if (NULL == pp_job) { + MALI_PRINT_ERROR(("Failed to create PP job.\n")); + return _MALI_OSK_ERR_NOMEM; + } + + gp_job = mali_gp_job_create(session, gp_args, + mali_scheduler_get_new_id(), + mali_pp_job_get_tracker(pp_job)); + if (NULL == gp_job) { + MALI_PRINT_ERROR(("Failed to create GP job.\n")); + mali_pp_job_delete(pp_job); + return _MALI_OSK_ERR_NOMEM; + } + + point_ptr = (u32 __user *)(uintptr_t)mali_pp_job_get_timeline_point_ptr(pp_job); + + /* Submit GP job. */ + mali_scheduler_submit_gp_job(session, gp_job); + gp_job = NULL; + + /* Submit PP job. */ + point = mali_scheduler_submit_pp_job(session, pp_job); + pp_job = NULL; + + if (0 != _mali_osk_put_user(((u32) point), point_ptr)) { + /* + * Let user space know that something failed + * after the jobs were started. + */ + return _MALI_OSK_ERR_ITEM_NOT_FOUND; + } + + return _MALI_OSK_ERR_OK; +} + +void _mali_ukk_pp_job_disable_wb(_mali_uk_pp_disable_wb_s *args) +{ + struct mali_session_data *session; + struct mali_pp_job *job; + struct mali_pp_job *tmp; + u32 fb_lookup_id; + + MALI_DEBUG_ASSERT_POINTER(args); + MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); + + session = (struct mali_session_data *)(uintptr_t)args->ctx; + + fb_lookup_id = args->fb_id & MALI_PP_JOB_FB_LOOKUP_LIST_MASK; + + mali_scheduler_lock(); + + /* Iterate over all jobs for given frame builder_id. */ + _MALI_OSK_LIST_FOREACHENTRY(job, tmp, + &session->pp_job_fb_lookup_list[fb_lookup_id], + struct mali_pp_job, session_fb_lookup_list) { + MALI_DEBUG_CODE(u32 disable_mask = 0); + + if (mali_pp_job_get_frame_builder_id(job) != + (u32) args->fb_id) { + MALI_DEBUG_PRINT(4, ("Mali PP scheduler: Disable WB mismatching FB.\n")); + continue; + } + + MALI_DEBUG_CODE(disable_mask |= 0xD << (4 * 3)); + + if (mali_pp_job_get_wb0_source_addr(job) == args->wb0_memory) { + MALI_DEBUG_CODE(disable_mask |= 0x1 << (4 * 1)); + mali_pp_job_disable_wb0(job); + } + + if (mali_pp_job_get_wb1_source_addr(job) == args->wb1_memory) { + MALI_DEBUG_CODE(disable_mask |= 0x2 << (4 * 2)); + mali_pp_job_disable_wb1(job); + } + + if (mali_pp_job_get_wb2_source_addr(job) == args->wb2_memory) { + MALI_DEBUG_CODE(disable_mask |= 0x3 << (4 * 3)); + mali_pp_job_disable_wb2(job); + } + MALI_DEBUG_PRINT(3, ("Mali PP scheduler: Disable WB: 0x%X.\n", + disable_mask)); + } + + mali_scheduler_unlock(); +} + +#if MALI_STATE_TRACKING +u32 mali_scheduler_dump_state(char *buf, u32 size) +{ + int n = 0; + + n += _mali_osk_snprintf(buf + n, size - n, "GP queues\n"); + n += _mali_osk_snprintf(buf + n, size - n, + "\tQueue depth: %u\n", job_queue_gp.depth); + n += _mali_osk_snprintf(buf + n, size - n, + "\tNormal priority queue is %s\n", + _mali_osk_list_empty(&job_queue_gp.normal_pri) ? + "empty" : "not empty"); + n += _mali_osk_snprintf(buf + n, size - n, + "\tHigh priority queue is %s\n", + _mali_osk_list_empty(&job_queue_gp.high_pri) ? + "empty" : "not empty"); + + n += _mali_osk_snprintf(buf + n, size - n, + "PP queues\n"); + n += _mali_osk_snprintf(buf + n, size - n, + "\tQueue depth: %u\n", job_queue_pp.depth); + n += _mali_osk_snprintf(buf + n, size - n, + "\tNormal priority queue is %s\n", + _mali_osk_list_empty(&job_queue_pp.normal_pri) + ? "empty" : "not empty"); + n += _mali_osk_snprintf(buf + n, size - n, + "\tHigh priority queue is %s\n", + _mali_osk_list_empty(&job_queue_pp.high_pri) + ? "empty" : "not empty"); + + n += _mali_osk_snprintf(buf + n, size - n, "\n"); + + return n; +} +#endif + +/* + * ---------- Implementation of static functions ---------- + */ + +static mali_timeline_point mali_scheduler_submit_gp_job( + struct mali_session_data *session, struct mali_gp_job *job) +{ + mali_timeline_point point; + + MALI_DEBUG_ASSERT_POINTER(session); + MALI_DEBUG_ASSERT_POINTER(job); + + /* Add job to Timeline system. */ + point = mali_timeline_system_add_tracker(session->timeline_system, + mali_gp_job_get_tracker(job), MALI_TIMELINE_GP); + + return point; +} + +static mali_timeline_point mali_scheduler_submit_pp_job( + struct mali_session_data *session, struct mali_pp_job *job) +{ + mali_timeline_point point; + + MALI_DEBUG_ASSERT_POINTER(session); + MALI_DEBUG_ASSERT_POINTER(job); + + mali_scheduler_lock(); + /* + * Adding job to the lookup list used to quickly discard + * writeback units of queued jobs. + */ + mali_pp_job_fb_lookup_add(job); + mali_scheduler_unlock(); + + /* Add job to Timeline system. */ + point = mali_timeline_system_add_tracker(session->timeline_system, + mali_pp_job_get_tracker(job), MALI_TIMELINE_PP); + + return point; +} + +static mali_bool mali_scheduler_queue_gp_job(struct mali_gp_job *job) +{ + struct mali_session_data *session; + _mali_osk_list_t *queue; + + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + MALI_DEBUG_ASSERT_POINTER(job); + + session = mali_gp_job_get_session(job); + MALI_DEBUG_ASSERT_POINTER(session); + + if (unlikely(session->is_aborting)) { + MALI_DEBUG_PRINT(4, ("Mali GP scheduler: Job %u (0x%08X) queued while session is aborting.\n", + mali_gp_job_get_id(job), job)); + return MALI_FALSE; /* job not queued */ + } + + mali_gp_job_set_cache_order(job, mali_scheduler_get_new_cache_order()); + + /* Determine which queue the job should be added to. */ + if (session->use_high_priority_job_queue) { + queue = &job_queue_gp.high_pri; + } else { + queue = &job_queue_gp.normal_pri; + } + + job_queue_gp.depth += 1; + job_queue_gp.big_job_num += (job->big_job) ? 1 : 0; + + /* Add job to queue (mali_gp_job_queue_add find correct place). */ + mali_gp_job_list_add(job, queue); + + /* + * We hold a PM reference for every job we hold queued (and running) + * It is important that we take this reference after job has been + * added the the queue so that any runtime resume could schedule this + * job right there and then. + */ + _mali_osk_pm_dev_ref_get_async(); + + if (mali_utilization_enabled()) { + /* + * We cheat a little bit by counting the GP as busy from the + * time a GP job is queued. This will be fine because we only + * loose the tiny idle gap between jobs, but we will instead + * get less utilization work to do (less locks taken) + */ + mali_utilization_gp_start(); + } + + /* Add profiling events for job enqueued */ + _mali_osk_profiling_add_event( + MALI_PROFILING_EVENT_TYPE_SINGLE | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_SINGLE_SW_GP_ENQUEUE, + mali_gp_job_get_pid(job), + mali_gp_job_get_tid(job), + mali_gp_job_get_frame_builder_id(job), + mali_gp_job_get_flush_id(job), + 0); + +#if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS) + trace_gpu_job_enqueue(mali_gp_job_get_tid(job), + mali_gp_job_get_id(job), "GP"); +#endif + + MALI_DEBUG_PRINT(3, ("Mali GP scheduler: Job %u (0x%08X) queued\n", + mali_gp_job_get_id(job), job)); + + return MALI_TRUE; /* job queued */ +} + +static mali_bool mali_scheduler_queue_pp_job(struct mali_pp_job *job) +{ + struct mali_session_data *session; + _mali_osk_list_t *queue = NULL; + + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + MALI_DEBUG_ASSERT_POINTER(job); + + session = mali_pp_job_get_session(job); + MALI_DEBUG_ASSERT_POINTER(session); + + if (unlikely(session->is_aborting)) { + MALI_DEBUG_PRINT(2, ("Mali PP scheduler: Job %u (0x%08X) queued while session is aborting.\n", + mali_pp_job_get_id(job), job)); + return MALI_FALSE; /* job not queued */ + } else if (unlikely(MALI_SWAP_IN_FAIL == job->swap_status)) { + MALI_DEBUG_PRINT(2, ("Mali PP scheduler: Job %u (0x%08X) queued while swap in failed.\n", + mali_pp_job_get_id(job), job)); + return MALI_FALSE; + } + + mali_pp_job_set_cache_order(job, mali_scheduler_get_new_cache_order()); + + if (session->use_high_priority_job_queue) { + queue = &job_queue_pp.high_pri; + } else { + queue = &job_queue_pp.normal_pri; + } + + job_queue_pp.depth += + mali_pp_job_get_sub_job_count(job); + + /* Add job to queue (mali_gp_job_queue_add find correct place). */ + mali_pp_job_list_add(job, queue); + + /* + * We hold a PM reference for every job we hold queued (and running) + * It is important that we take this reference after job has been + * added the the queue so that any runtime resume could schedule this + * job right there and then. + */ + _mali_osk_pm_dev_ref_get_async(); + + if (mali_utilization_enabled()) { + /* + * We cheat a little bit by counting the PP as busy from the + * time a PP job is queued. This will be fine because we only + * loose the tiny idle gap between jobs, but we will instead + * get less utilization work to do (less locks taken) + */ + mali_utilization_pp_start(); + } + + /* Add profiling events for job enqueued */ + + _mali_osk_profiling_add_event( + MALI_PROFILING_EVENT_TYPE_SINGLE | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_SINGLE_SW_PP_ENQUEUE, + mali_pp_job_get_pid(job), + mali_pp_job_get_tid(job), + mali_pp_job_get_frame_builder_id(job), + mali_pp_job_get_flush_id(job), + 0); + +#if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS) + trace_gpu_job_enqueue(mali_pp_job_get_tid(job), + mali_pp_job_get_id(job), "PP"); +#endif + + MALI_DEBUG_PRINT(3, ("Mali PP scheduler: %s job %u (0x%08X) with %u parts queued.\n", + mali_pp_job_is_virtual(job) + ? "Virtual" : "Physical", + mali_pp_job_get_id(job), job, + mali_pp_job_get_sub_job_count(job))); + + return MALI_TRUE; /* job queued */ +} + +static void mali_scheduler_return_gp_job_to_user(struct mali_gp_job *job, + mali_bool success) +{ + _mali_uk_gp_job_finished_s *jobres; + struct mali_session_data *session; + _mali_osk_notification_t *notification; + + MALI_DEBUG_ASSERT_POINTER(job); + + session = mali_gp_job_get_session(job); + MALI_DEBUG_ASSERT_POINTER(session); + + notification = mali_gp_job_get_finished_notification(job); + MALI_DEBUG_ASSERT_POINTER(notification); + + jobres = notification->result_buffer; + MALI_DEBUG_ASSERT_POINTER(jobres); + + jobres->pending_big_job_num = mali_scheduler_job_gp_big_job_count(); + + jobres->user_job_ptr = mali_gp_job_get_user_id(job); + if (MALI_TRUE == success) { + jobres->status = _MALI_UK_JOB_STATUS_END_SUCCESS; + } else { + jobres->status = _MALI_UK_JOB_STATUS_END_UNKNOWN_ERR; + } + jobres->heap_current_addr = mali_gp_job_get_current_heap_addr(job); + jobres->perf_counter0 = mali_gp_job_get_perf_counter_value0(job); + jobres->perf_counter1 = mali_gp_job_get_perf_counter_value1(job); + + mali_session_send_notification(session, notification); +} + +void mali_scheduler_return_pp_job_to_user(struct mali_pp_job *job, + u32 num_cores_in_virtual) +{ + u32 i; + u32 num_counters_to_copy; + _mali_uk_pp_job_finished_s *jobres; + struct mali_session_data *session; + _mali_osk_notification_t *notification; + + if (MALI_TRUE == mali_pp_job_use_no_notification(job)) { + return; + } + + MALI_DEBUG_ASSERT_POINTER(job); + + session = mali_pp_job_get_session(job); + MALI_DEBUG_ASSERT_POINTER(session); + + notification = mali_pp_job_get_finished_notification(job); + MALI_DEBUG_ASSERT_POINTER(notification); + + jobres = notification->result_buffer; + MALI_DEBUG_ASSERT_POINTER(jobres); + + jobres->user_job_ptr = mali_pp_job_get_user_id(job); + if (MALI_TRUE == mali_pp_job_was_success(job)) { + jobres->status = _MALI_UK_JOB_STATUS_END_SUCCESS; + } else { + jobres->status = _MALI_UK_JOB_STATUS_END_UNKNOWN_ERR; + } + + if (mali_pp_job_is_virtual(job)) { + num_counters_to_copy = num_cores_in_virtual; + } else { + num_counters_to_copy = mali_pp_job_get_sub_job_count(job); + } + + for (i = 0; i < num_counters_to_copy; i++) { + jobres->perf_counter0[i] = + mali_pp_job_get_perf_counter_value0(job, i); + jobres->perf_counter1[i] = + mali_pp_job_get_perf_counter_value1(job, i); + jobres->perf_counter_src0 = + mali_pp_job_get_pp_counter_global_src0(); + jobres->perf_counter_src1 = + mali_pp_job_get_pp_counter_global_src1(); + } + + mali_session_send_notification(session, notification); +} + +static void mali_scheduler_deferred_pp_job_delete(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + + _mali_osk_spinlock_irq_lock(scheduler_pp_job_delete_lock); + mali_pp_job_list_addtail(job, &scheduler_pp_job_deletion_queue); + _mali_osk_spinlock_irq_unlock(scheduler_pp_job_delete_lock); + + _mali_osk_wq_schedule_work(scheduler_wq_pp_job_delete); +} + +void mali_scheduler_do_pp_job_delete(void *arg) +{ + _MALI_OSK_LIST_HEAD_STATIC_INIT(list); + struct mali_pp_job *job; + struct mali_pp_job *tmp; + + MALI_IGNORE(arg); + + /* + * Quickly "unhook" the jobs pending to be deleted, so we can release + * the lock before we start deleting the job objects + * (without any locks held) + */ + _mali_osk_spinlock_irq_lock(scheduler_pp_job_delete_lock); + _mali_osk_list_move_list(&scheduler_pp_job_deletion_queue, &list); + _mali_osk_spinlock_irq_unlock(scheduler_pp_job_delete_lock); + + _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &list, + struct mali_pp_job, list) { + _mali_osk_list_delinit(&job->list); + + mali_pp_job_delete(job); /* delete the job object itself */ + } +} + +#if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) + +static void mali_scheduler_deferred_pp_job_queue(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + + _mali_osk_spinlock_irq_lock(scheduler_pp_job_queue_lock); + mali_pp_job_list_addtail(job, &scheduler_pp_job_queue_list); + _mali_osk_spinlock_irq_unlock(scheduler_pp_job_queue_lock); + + _mali_osk_wq_schedule_work(scheduler_wq_pp_job_queue); +} + +static void mali_scheduler_do_pp_job_queue(void *arg) +{ + _MALI_OSK_LIST_HEAD_STATIC_INIT(list); + struct mali_pp_job *job; + struct mali_pp_job *tmp; + mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; + + MALI_IGNORE(arg); + + /* + * Quickly "unhook" the jobs pending to be queued, so we can release + * the lock before we start queueing the job objects + * (without any locks held) + */ + _mali_osk_spinlock_irq_lock(scheduler_pp_job_queue_lock); + _mali_osk_list_move_list(&scheduler_pp_job_queue_list, &list); + _mali_osk_spinlock_irq_unlock(scheduler_pp_job_queue_lock); + + /* First loop through all jobs and do the pre-work (no locks needed) */ + _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &list, + struct mali_pp_job, list) { + if (mali_pp_job_needs_dma_buf_mapping(job)) { + /* + * This operation could fail, but we continue anyway, + * because the worst that could happen is that this + * job will fail due to a Mali page fault. + */ + mali_dma_buf_map_job(job); + } + } + + mali_scheduler_lock(); + + /* Then loop through all jobs again to queue them (lock needed) */ + _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &list, + struct mali_pp_job, list) { + + /* Remove from scheduler_pp_job_queue_list before queueing */ + mali_pp_job_list_remove(job); + + if (mali_scheduler_queue_pp_job(job)) { + /* Job queued successfully */ + schedule_mask |= MALI_SCHEDULER_MASK_PP; + } else { + /* Failed to enqueue job, release job (with error) */ + mali_pp_job_fb_lookup_remove(job); + mali_pp_job_mark_unstarted_failed(job); + + /* unlock scheduler in this uncommon case */ + mali_scheduler_unlock(); + + schedule_mask |= mali_timeline_tracker_release( + mali_pp_job_get_tracker(job)); + + /* Notify user space and close the job object */ + mali_scheduler_complete_pp_job(job, 0, MALI_TRUE, + MALI_FALSE); + + mali_scheduler_lock(); + } + } + + mali_scheduler_unlock(); + + /* Trigger scheduling of jobs */ + mali_executor_schedule_from_mask(schedule_mask, MALI_FALSE); +} + +#endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */ + +void mali_scheduler_gp_pp_job_queue_print(void) +{ + struct mali_gp_job *gp_job = NULL; + struct mali_gp_job *tmp_gp_job = NULL; + struct mali_pp_job *pp_job = NULL; + struct mali_pp_job *tmp_pp_job = NULL; + + MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj); + MALI_DEBUG_ASSERT_LOCK_HELD(mali_executor_lock_obj); + + /* dump job queup status */ + if ((0 == job_queue_gp.depth) && (0 == job_queue_pp.depth)) { + MALI_PRINT(("No GP&PP job in the job queue.\n")); + return; + } + + MALI_PRINT(("Total (%d) GP job in the job queue.\n", job_queue_gp.depth)); + if (job_queue_gp.depth > 0) { + if (!_mali_osk_list_empty(&job_queue_gp.high_pri)) { + _MALI_OSK_LIST_FOREACHENTRY(gp_job, tmp_gp_job, &job_queue_gp.high_pri, + struct mali_gp_job, list) { + MALI_PRINT(("GP job(%p) id = %d tid = %d pid = %d in the gp job high_pri queue\n", gp_job, gp_job->id, gp_job->tid, gp_job->pid)); + } + } + + if (!_mali_osk_list_empty(&job_queue_gp.normal_pri)) { + _MALI_OSK_LIST_FOREACHENTRY(gp_job, tmp_gp_job, &job_queue_gp.normal_pri, + struct mali_gp_job, list) { + MALI_PRINT(("GP job(%p) id = %d tid = %d pid = %d in the gp job normal_pri queue\n", gp_job, gp_job->id, gp_job->tid, gp_job->pid)); + } + } + } + + MALI_PRINT(("Total (%d) PP job in the job queue.\n", job_queue_pp.depth)); + if (job_queue_pp.depth > 0) { + if (!_mali_osk_list_empty(&job_queue_pp.high_pri)) { + _MALI_OSK_LIST_FOREACHENTRY(pp_job, tmp_pp_job, &job_queue_pp.high_pri, + struct mali_pp_job, list) { + if (mali_pp_job_is_virtual(pp_job)) { + MALI_PRINT(("PP Virtual job(%p) id = %d tid = %d pid = %d in the pp job high_pri queue\n", pp_job, pp_job->id, pp_job->tid, pp_job->pid)); + } else { + MALI_PRINT(("PP Physical job(%p) id = %d tid = %d pid = %d in the pp job high_pri queue\n", pp_job, pp_job->id, pp_job->tid, pp_job->pid)); + } + } + } + + if (!_mali_osk_list_empty(&job_queue_pp.normal_pri)) { + _MALI_OSK_LIST_FOREACHENTRY(pp_job, tmp_pp_job, &job_queue_pp.normal_pri, + struct mali_pp_job, list) { + if (mali_pp_job_is_virtual(pp_job)) { + MALI_PRINT(("PP Virtual job(%p) id = %d tid = %d pid = %d in the pp job normal_pri queue\n", pp_job, pp_job->id, pp_job->tid, pp_job->pid)); + } else { + MALI_PRINT(("PP Physical job(%p) id = %d tid = %d pid = %d in the pp job normal_pri queue\n", pp_job, pp_job->id, pp_job->tid, pp_job->pid)); + } + } + } + } + + /* dump group running job status */ + mali_executor_running_status_print(); +} diff --git a/drivers/gpu/arm/utgard/common/mali_scheduler.h b/drivers/gpu/arm/utgard/common/mali_scheduler.h new file mode 100644 index 00000000000000..f24cf42b8a794d --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_scheduler.h @@ -0,0 +1,130 @@ +/* + * Copyright (C) 2012-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_SCHEDULER_H__ +#define __MALI_SCHEDULER_H__ + +#include "mali_osk.h" +#include "mali_osk_list.h" +#include "mali_scheduler_types.h" +#include "mali_session.h" + +struct mali_scheduler_job_queue { + _MALI_OSK_LIST_HEAD(normal_pri); /* Queued jobs with normal priority */ + _MALI_OSK_LIST_HEAD(high_pri); /* Queued jobs with high priority */ + u32 depth; /* Depth of combined queues. */ + u32 big_job_num; +}; + +extern _mali_osk_spinlock_irq_t *mali_scheduler_lock_obj; + +/* Queue of jobs to be executed on the GP group */ +extern struct mali_scheduler_job_queue job_queue_gp; + +/* Queue of PP jobs */ +extern struct mali_scheduler_job_queue job_queue_pp; + +extern _mali_osk_atomic_t mali_job_id_autonumber; +extern _mali_osk_atomic_t mali_job_cache_order_autonumber; + +#define MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD() MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj); + +_mali_osk_errcode_t mali_scheduler_initialize(void); +void mali_scheduler_terminate(void); + +MALI_STATIC_INLINE void mali_scheduler_lock(void) +{ + _mali_osk_spinlock_irq_lock(mali_scheduler_lock_obj); + MALI_DEBUG_PRINT(5, ("Mali scheduler: scheduler lock taken.\n")); +} + +MALI_STATIC_INLINE void mali_scheduler_unlock(void) +{ + MALI_DEBUG_PRINT(5, ("Mali scheduler: Releasing scheduler lock.\n")); + _mali_osk_spinlock_irq_unlock(mali_scheduler_lock_obj); +} + +MALI_STATIC_INLINE u32 mali_scheduler_job_gp_count(void) +{ + return job_queue_gp.depth; +} +MALI_STATIC_INLINE u32 mali_scheduler_job_gp_big_job_count(void) +{ + return job_queue_gp.big_job_num; +} + +u32 mali_scheduler_job_physical_head_count(void); + +mali_bool mali_scheduler_job_next_is_virtual(void); + +struct mali_gp_job *mali_scheduler_job_gp_get(void); +struct mali_pp_job *mali_scheduler_job_pp_physical_peek(void); +struct mali_pp_job *mali_scheduler_job_pp_virtual_peek(void); +struct mali_pp_job *mali_scheduler_job_pp_physical_get(u32 *sub_job); +struct mali_pp_job *mali_scheduler_job_pp_virtual_get(void); + +MALI_STATIC_INLINE u32 mali_scheduler_get_new_id(void) +{ + return _mali_osk_atomic_inc_return(&mali_job_id_autonumber); +} + +MALI_STATIC_INLINE u32 mali_scheduler_get_new_cache_order(void) +{ + return _mali_osk_atomic_inc_return(&mali_job_cache_order_autonumber); +} + +/** + * @brief Used by the Timeline system to queue a GP job. + * + * @note @ref mali_executor_schedule_from_mask() should be called if this + * function returns non-zero. + * + * @param job The GP job that is being activated. + * + * @return A scheduling bitmask that can be used to decide if scheduling is + * necessary after this call. + */ +mali_scheduler_mask mali_scheduler_activate_gp_job(struct mali_gp_job *job); + +/** + * @brief Used by the Timeline system to queue a PP job. + * + * @note @ref mali_executor_schedule_from_mask() should be called if this + * function returns non-zero. + * + * @param job The PP job that is being activated. + * + * @return A scheduling bitmask that can be used to decide if scheduling is + * necessary after this call. + */ +mali_scheduler_mask mali_scheduler_activate_pp_job(struct mali_pp_job *job); + +void mali_scheduler_complete_gp_job(struct mali_gp_job *job, + mali_bool success, + mali_bool user_notification, + mali_bool dequeued); + +void mali_scheduler_complete_pp_job(struct mali_pp_job *job, + u32 num_cores_in_virtual, + mali_bool user_notification, + mali_bool dequeued); + +void mali_scheduler_abort_session(struct mali_session_data *session); + +void mali_scheduler_return_pp_job_to_user(struct mali_pp_job *job, + u32 num_cores_in_virtual); + +#if MALI_STATE_TRACKING +u32 mali_scheduler_dump_state(char *buf, u32 size); +#endif + +void mali_scheduler_gp_pp_job_queue_print(void); + +#endif /* __MALI_SCHEDULER_H__ */ diff --git a/drivers/gpu/arm/utgard/common/mali_scheduler_types.h b/drivers/gpu/arm/utgard/common/mali_scheduler_types.h new file mode 100644 index 00000000000000..f862961d146e7e --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_scheduler_types.h @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_SCHEDULER_TYPES_H__ +#define __MALI_SCHEDULER_TYPES_H__ + +#include "mali_osk.h" + +#define MALI_SCHEDULER_JOB_ID_SPAN 65535 + +/** + * Bitmask used for defered scheduling of subsystems. + */ +typedef u32 mali_scheduler_mask; + +#define MALI_SCHEDULER_MASK_GP (1<<0) +#define MALI_SCHEDULER_MASK_PP (1<<1) + +#define MALI_SCHEDULER_MASK_EMPTY 0 +#define MALI_SCHEDULER_MASK_ALL (MALI_SCHEDULER_MASK_GP | MALI_SCHEDULER_MASK_PP) + +#endif /* __MALI_SCHEDULER_TYPES_H__ */ diff --git a/drivers/gpu/arm/utgard/common/mali_session.c b/drivers/gpu/arm/utgard/common/mali_session.c new file mode 100644 index 00000000000000..e0a2805b13d3a2 --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_session.c @@ -0,0 +1,144 @@ +/* + * Copyright (C) 2012-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_osk.h" +#include "mali_osk_list.h" +#include "mali_session.h" +#include "mali_ukk.h" +#ifdef MALI_MEM_SWAP_TRACKING +#include "mali_memory_swap_alloc.h" +#endif + +_MALI_OSK_LIST_HEAD(mali_sessions); +static u32 mali_session_count = 0; + +_mali_osk_spinlock_irq_t *mali_sessions_lock = NULL; +wait_queue_head_t pending_queue; + +_mali_osk_errcode_t mali_session_initialize(void) +{ + _MALI_OSK_INIT_LIST_HEAD(&mali_sessions); + /* init wait queue for big varying job */ + init_waitqueue_head(&pending_queue); + + mali_sessions_lock = _mali_osk_spinlock_irq_init( + _MALI_OSK_LOCKFLAG_ORDERED, + _MALI_OSK_LOCK_ORDER_SESSIONS); + if (NULL == mali_sessions_lock) { + return _MALI_OSK_ERR_NOMEM; + } + + return _MALI_OSK_ERR_OK; +} + +void mali_session_terminate(void) +{ + if (NULL != mali_sessions_lock) { + _mali_osk_spinlock_irq_term(mali_sessions_lock); + mali_sessions_lock = NULL; + } +} + +void mali_session_add(struct mali_session_data *session) +{ + mali_session_lock(); + _mali_osk_list_add(&session->link, &mali_sessions); + mali_session_count++; + mali_session_unlock(); +} + +void mali_session_remove(struct mali_session_data *session) +{ + mali_session_lock(); + _mali_osk_list_delinit(&session->link); + mali_session_count--; + mali_session_unlock(); +} + +u32 mali_session_get_count(void) +{ + return mali_session_count; +} + +wait_queue_head_t *mali_session_get_wait_queue(void) +{ + return &pending_queue; +} + +/* + * Get the max completed window jobs from all active session, + * which will be used in window render frame per sec calculate + */ +#if defined(CONFIG_MALI_DVFS) +u32 mali_session_max_window_num(void) +{ + struct mali_session_data *session, *tmp; + u32 max_window_num = 0; + u32 tmp_number = 0; + + mali_session_lock(); + + MALI_SESSION_FOREACH(session, tmp, link) { + tmp_number = _mali_osk_atomic_xchg( + &session->number_of_window_jobs, 0); + if (max_window_num < tmp_number) { + max_window_num = tmp_number; + } + } + + mali_session_unlock(); + + return max_window_num; +} +#endif + +void mali_session_memory_tracking(_mali_osk_print_ctx *print_ctx) +{ + struct mali_session_data *session, *tmp; + u32 mali_mem_usage; + u32 total_mali_mem_size; +#ifdef MALI_MEM_SWAP_TRACKING + u32 swap_pool_size; + u32 swap_unlock_size; +#endif + + MALI_DEBUG_ASSERT_POINTER(print_ctx); + mali_session_lock(); + MALI_SESSION_FOREACH(session, tmp, link) { +#ifdef MALI_MEM_SWAP_TRACKING + _mali_osk_ctxprintf(print_ctx, " %-25s %-10u %-10u %-15u %-15u %-10u %-10u %-10u\n", + session->comm, session->pid, + (atomic_read(&session->mali_mem_allocated_pages)) * _MALI_OSK_MALI_PAGE_SIZE, + session->max_mali_mem_allocated_size, + (atomic_read(&session->mali_mem_array[MALI_MEM_EXTERNAL])) * _MALI_OSK_MALI_PAGE_SIZE, + (atomic_read(&session->mali_mem_array[MALI_MEM_UMP])) * _MALI_OSK_MALI_PAGE_SIZE, + (atomic_read(&session->mali_mem_array[MALI_MEM_DMA_BUF])) * _MALI_OSK_MALI_PAGE_SIZE, + (atomic_read(&session->mali_mem_array[MALI_MEM_SWAP])) * _MALI_OSK_MALI_PAGE_SIZE + ); +#else + _mali_osk_ctxprintf(print_ctx, " %-25s %-10u %-10u %-15u %-15u %-10u %-10u \n", + session->comm, session->pid, + (atomic_read(&session->mali_mem_allocated_pages)) * _MALI_OSK_MALI_PAGE_SIZE, + session->max_mali_mem_allocated_size, + (atomic_read(&session->mali_mem_array[MALI_MEM_EXTERNAL])) * _MALI_OSK_MALI_PAGE_SIZE, + (atomic_read(&session->mali_mem_array[MALI_MEM_UMP])) * _MALI_OSK_MALI_PAGE_SIZE, + (atomic_read(&session->mali_mem_array[MALI_MEM_DMA_BUF])) * _MALI_OSK_MALI_PAGE_SIZE + ); +#endif + } + mali_session_unlock(); + mali_mem_usage = _mali_ukk_report_memory_usage(); + total_mali_mem_size = _mali_ukk_report_total_memory_size(); + _mali_osk_ctxprintf(print_ctx, "Mali mem usage: %u\nMali mem limit: %u\n", mali_mem_usage, total_mali_mem_size); +#ifdef MALI_MEM_SWAP_TRACKING + mali_mem_swap_tracking(&swap_pool_size, &swap_unlock_size); + _mali_osk_ctxprintf(print_ctx, "Mali swap mem pool : %u\nMali swap mem unlock: %u\n", swap_pool_size, swap_unlock_size); +#endif +} diff --git a/drivers/gpu/arm/utgard/common/mali_session.h b/drivers/gpu/arm/utgard/common/mali_session.h new file mode 100644 index 00000000000000..6791b2b5f11072 --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_session.h @@ -0,0 +1,127 @@ +/* + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_SESSION_H__ +#define __MALI_SESSION_H__ + +#include "mali_mmu_page_directory.h" +#include "mali_osk.h" +#include "mali_osk_list.h" +#include "mali_memory_types.h" +#include "mali_memory_manager.h" + +struct mali_timeline_system; +struct mali_soft_system; + +/* Number of frame builder job lists per session. */ +#define MALI_PP_JOB_FB_LOOKUP_LIST_SIZE 16 +#define MALI_PP_JOB_FB_LOOKUP_LIST_MASK (MALI_PP_JOB_FB_LOOKUP_LIST_SIZE - 1) +/*Max pending big job allowed in kernel*/ +#define MALI_MAX_PENDING_BIG_JOB (2) + +struct mali_session_data { + _mali_osk_notification_queue_t *ioctl_queue; + + _mali_osk_mutex_t *memory_lock; /**< Lock protecting the vm manipulation */ +#if 0 + _mali_osk_list_t memory_head; /**< Track all the memory allocated in this session, for freeing on abnormal termination */ +#endif + struct mali_page_directory *page_directory; /**< MMU page directory for this session */ + + _MALI_OSK_LIST_HEAD(link); /**< Link for list of all sessions */ + _MALI_OSK_LIST_HEAD(pp_job_list); /**< List of all PP jobs on this session */ + +#if defined(CONFIG_MALI_DVFS) + _mali_osk_atomic_t number_of_window_jobs; /**< Record the window jobs completed on this session in a period */ +#endif + + _mali_osk_list_t pp_job_fb_lookup_list[MALI_PP_JOB_FB_LOOKUP_LIST_SIZE]; /**< List of PP job lists per frame builder id. Used to link jobs from same frame builder. */ + + struct mali_soft_job_system *soft_job_system; /**< Soft job system for this session. */ + struct mali_timeline_system *timeline_system; /**< Timeline system for this session. */ + + mali_bool is_aborting; /**< MALI_TRUE if the session is aborting, MALI_FALSE if not. */ + mali_bool use_high_priority_job_queue; /**< If MALI_TRUE, jobs added from this session will use the high priority job queues. */ + u32 pid; + char *comm; + atomic_t mali_mem_array[MALI_MEM_TYPE_MAX]; /**< The array to record mem types' usage for this session. */ + atomic_t mali_mem_allocated_pages; /** The current allocated mali memory pages, which include mali os memory and mali dedicated memory.*/ + size_t max_mali_mem_allocated_size; /**< The past max mali memory allocated size, which include mali os memory and mali dedicated memory. */ + /* Added for new memroy system */ + struct mali_allocation_manager allocation_mgr; +}; + +_mali_osk_errcode_t mali_session_initialize(void); +void mali_session_terminate(void); + +/* List of all sessions. Actual list head in mali_kernel_core.c */ +extern _mali_osk_list_t mali_sessions; +/* Lock to protect modification and access to the mali_sessions list */ +extern _mali_osk_spinlock_irq_t *mali_sessions_lock; + +MALI_STATIC_INLINE void mali_session_lock(void) +{ + _mali_osk_spinlock_irq_lock(mali_sessions_lock); +} + +MALI_STATIC_INLINE void mali_session_unlock(void) +{ + _mali_osk_spinlock_irq_unlock(mali_sessions_lock); +} + +void mali_session_add(struct mali_session_data *session); +void mali_session_remove(struct mali_session_data *session); +u32 mali_session_get_count(void); +wait_queue_head_t *mali_session_get_wait_queue(void); + +#define MALI_SESSION_FOREACH(session, tmp, link) \ + _MALI_OSK_LIST_FOREACHENTRY(session, tmp, &mali_sessions, struct mali_session_data, link) + +MALI_STATIC_INLINE struct mali_page_directory *mali_session_get_page_directory(struct mali_session_data *session) +{ + return session->page_directory; +} + +MALI_STATIC_INLINE void mali_session_memory_lock(struct mali_session_data *session) +{ + MALI_DEBUG_ASSERT_POINTER(session); + _mali_osk_mutex_wait(session->memory_lock); +} + +MALI_STATIC_INLINE void mali_session_memory_unlock(struct mali_session_data *session) +{ + MALI_DEBUG_ASSERT_POINTER(session); + _mali_osk_mutex_signal(session->memory_lock); +} + +MALI_STATIC_INLINE void mali_session_send_notification(struct mali_session_data *session, _mali_osk_notification_t *object) +{ + _mali_osk_notification_queue_send(session->ioctl_queue, object); +} + +#if defined(CONFIG_MALI_DVFS) + +MALI_STATIC_INLINE void mali_session_inc_num_window_jobs(struct mali_session_data *session) +{ + MALI_DEBUG_ASSERT_POINTER(session); + _mali_osk_atomic_inc(&session->number_of_window_jobs); +} + +/* + * Get the max completed window jobs from all active session, + * which will be used in window render frame per sec calculate + */ +u32 mali_session_max_window_num(void); + +#endif + +void mali_session_memory_tracking(_mali_osk_print_ctx *print_ctx); + +#endif /* __MALI_SESSION_H__ */ diff --git a/drivers/gpu/arm/utgard/common/mali_soft_job.c b/drivers/gpu/arm/utgard/common/mali_soft_job.c new file mode 100644 index 00000000000000..36ac982e1df08a --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_soft_job.c @@ -0,0 +1,438 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_soft_job.h" +#include "mali_osk.h" +#include "mali_timeline.h" +#include "mali_session.h" +#include "mali_kernel_common.h" +#include "mali_uk_types.h" +#include "mali_scheduler.h" +#include "mali_executor.h" + +MALI_STATIC_INLINE void mali_soft_job_system_lock(struct mali_soft_job_system *system) +{ + MALI_DEBUG_ASSERT_POINTER(system); + _mali_osk_spinlock_irq_lock(system->lock); + MALI_DEBUG_PRINT(5, ("Mali Soft Job: soft system %p lock taken\n", system)); + MALI_DEBUG_ASSERT(0 == system->lock_owner); + MALI_DEBUG_CODE(system->lock_owner = _mali_osk_get_tid()); +} + +MALI_STATIC_INLINE void mali_soft_job_system_unlock(struct mali_soft_job_system *system) +{ + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_PRINT(5, ("Mali Soft Job: releasing soft system %p lock\n", system)); + MALI_DEBUG_ASSERT(_mali_osk_get_tid() == system->lock_owner); + MALI_DEBUG_CODE(system->lock_owner = 0); + _mali_osk_spinlock_irq_unlock(system->lock); +} + +#if defined(DEBUG) +MALI_STATIC_INLINE void mali_soft_job_system_assert_locked(struct mali_soft_job_system *system) +{ + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT(_mali_osk_get_tid() == system->lock_owner); +} +#define MALI_ASSERT_SOFT_JOB_SYSTEM_LOCKED(system) mali_soft_job_system_assert_locked(system) +#else +#define MALI_ASSERT_SOFT_JOB_SYSTEM_LOCKED(system) +#endif /* defined(DEBUG) */ + +struct mali_soft_job_system *mali_soft_job_system_create(struct mali_session_data *session) +{ + struct mali_soft_job_system *system; + + MALI_DEBUG_ASSERT_POINTER(session); + + system = (struct mali_soft_job_system *) _mali_osk_calloc(1, sizeof(struct mali_soft_job_system)); + if (NULL == system) { + return NULL; + } + + system->session = session; + + system->lock = _mali_osk_spinlock_irq_init(_MALI_OSK_LOCKFLAG_ORDERED, _MALI_OSK_LOCK_ORDER_SCHEDULER); + if (NULL == system->lock) { + mali_soft_job_system_destroy(system); + return NULL; + } + system->lock_owner = 0; + system->last_job_id = 0; + + _MALI_OSK_INIT_LIST_HEAD(&(system->jobs_used)); + + return system; +} + +void mali_soft_job_system_destroy(struct mali_soft_job_system *system) +{ + MALI_DEBUG_ASSERT_POINTER(system); + + /* All jobs should be free at this point. */ + MALI_DEBUG_ASSERT(_mali_osk_list_empty(&(system->jobs_used))); + + if (NULL != system) { + if (NULL != system->lock) { + _mali_osk_spinlock_irq_term(system->lock); + } + _mali_osk_free(system); + } +} + +static void mali_soft_job_system_free_job(struct mali_soft_job_system *system, struct mali_soft_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_POINTER(system); + + mali_soft_job_system_lock(job->system); + + MALI_DEBUG_ASSERT(MALI_SOFT_JOB_INVALID_ID != job->id); + MALI_DEBUG_ASSERT(system == job->system); + + _mali_osk_list_del(&(job->system_list)); + + mali_soft_job_system_unlock(job->system); + + _mali_osk_free(job); +} + +MALI_STATIC_INLINE struct mali_soft_job *mali_soft_job_system_lookup_job(struct mali_soft_job_system *system, u32 job_id) +{ + struct mali_soft_job *job, *tmp; + + MALI_DEBUG_ASSERT_POINTER(system); + MALI_ASSERT_SOFT_JOB_SYSTEM_LOCKED(system); + + _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &system->jobs_used, struct mali_soft_job, system_list) { + if (job->id == job_id) + return job; + } + + return NULL; +} + +void mali_soft_job_destroy(struct mali_soft_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_POINTER(job->system); + + MALI_DEBUG_PRINT(4, ("Mali Soft Job: destroying soft job %u (0x%08X)\n", job->id, job)); + + if (NULL != job) { + if (0 < _mali_osk_atomic_dec_return(&job->refcount)) return; + + _mali_osk_atomic_term(&job->refcount); + + if (NULL != job->activated_notification) { + _mali_osk_notification_delete(job->activated_notification); + job->activated_notification = NULL; + } + + mali_soft_job_system_free_job(job->system, job); + } +} + +struct mali_soft_job *mali_soft_job_create(struct mali_soft_job_system *system, mali_soft_job_type type, u64 user_job) +{ + struct mali_soft_job *job; + _mali_osk_notification_t *notification = NULL; + + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT((MALI_SOFT_JOB_TYPE_USER_SIGNALED == type) || + (MALI_SOFT_JOB_TYPE_SELF_SIGNALED == type)); + + notification = _mali_osk_notification_create(_MALI_NOTIFICATION_SOFT_ACTIVATED, sizeof(_mali_uk_soft_job_activated_s)); + if (unlikely(NULL == notification)) { + MALI_PRINT_ERROR(("Mali Soft Job: failed to allocate notification")); + return NULL; + } + + job = _mali_osk_malloc(sizeof(struct mali_soft_job)); + if (unlikely(NULL == job)) { + MALI_DEBUG_PRINT(2, ("Mali Soft Job: system alloc job failed. \n")); + return NULL; + } + + mali_soft_job_system_lock(system); + + job->system = system; + job->id = system->last_job_id++; + job->state = MALI_SOFT_JOB_STATE_ALLOCATED; + + _mali_osk_list_add(&(job->system_list), &(system->jobs_used)); + + job->type = type; + job->user_job = user_job; + job->activated = MALI_FALSE; + + job->activated_notification = notification; + + _mali_osk_atomic_init(&job->refcount, 1); + + MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_ALLOCATED == job->state); + MALI_DEBUG_ASSERT(system == job->system); + MALI_DEBUG_ASSERT(MALI_SOFT_JOB_INVALID_ID != job->id); + + mali_soft_job_system_unlock(system); + + return job; +} + +mali_timeline_point mali_soft_job_start(struct mali_soft_job *job, struct mali_timeline_fence *fence) +{ + mali_timeline_point point; + struct mali_soft_job_system *system; + + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_POINTER(fence); + + MALI_DEBUG_ASSERT_POINTER(job->system); + system = job->system; + + MALI_DEBUG_ASSERT_POINTER(system->session); + MALI_DEBUG_ASSERT_POINTER(system->session->timeline_system); + + mali_soft_job_system_lock(system); + + MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_ALLOCATED == job->state); + job->state = MALI_SOFT_JOB_STATE_STARTED; + + mali_soft_job_system_unlock(system); + + MALI_DEBUG_PRINT(4, ("Mali Soft Job: starting soft job %u (0x%08X)\n", job->id, job)); + + mali_timeline_tracker_init(&job->tracker, MALI_TIMELINE_TRACKER_SOFT, fence, job); + point = mali_timeline_system_add_tracker(system->session->timeline_system, &job->tracker, MALI_TIMELINE_SOFT); + + return point; +} + +static mali_bool mali_soft_job_is_activated(void *data) +{ + struct mali_soft_job *job; + + job = (struct mali_soft_job *) data; + MALI_DEBUG_ASSERT_POINTER(job); + + return job->activated; +} + +_mali_osk_errcode_t mali_soft_job_system_signal_job(struct mali_soft_job_system *system, u32 job_id) +{ + struct mali_soft_job *job; + struct mali_timeline_system *timeline_system; + mali_scheduler_mask schedule_mask; + + MALI_DEBUG_ASSERT_POINTER(system); + + mali_soft_job_system_lock(system); + + job = mali_soft_job_system_lookup_job(system, job_id); + + if ((NULL == job) || (MALI_SOFT_JOB_TYPE_USER_SIGNALED != job->type) + || !(MALI_SOFT_JOB_STATE_STARTED == job->state || MALI_SOFT_JOB_STATE_TIMED_OUT == job->state)) { + mali_soft_job_system_unlock(system); + MALI_PRINT_ERROR(("Mali Soft Job: invalid soft job id %u", job_id)); + return _MALI_OSK_ERR_ITEM_NOT_FOUND; + } + + if (MALI_SOFT_JOB_STATE_TIMED_OUT == job->state) { + job->state = MALI_SOFT_JOB_STATE_SIGNALED; + mali_soft_job_system_unlock(system); + + MALI_DEBUG_ASSERT(MALI_TRUE == job->activated); + MALI_DEBUG_PRINT(4, ("Mali Soft Job: soft job %u (0x%08X) was timed out\n", job->id, job)); + mali_soft_job_destroy(job); + + return _MALI_OSK_ERR_TIMEOUT; + } + + MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_STARTED == job->state); + + job->state = MALI_SOFT_JOB_STATE_SIGNALED; + mali_soft_job_system_unlock(system); + + /* Since the job now is in signaled state, timeouts from the timeline system will be + * ignored, and it is not possible to signal this job again. */ + + timeline_system = system->session->timeline_system; + MALI_DEBUG_ASSERT_POINTER(timeline_system); + + /* Wait until activated. */ + _mali_osk_wait_queue_wait_event(timeline_system->wait_queue, mali_soft_job_is_activated, (void *) job); + + MALI_DEBUG_PRINT(4, ("Mali Soft Job: signaling soft job %u (0x%08X)\n", job->id, job)); + + schedule_mask = mali_timeline_tracker_release(&job->tracker); + mali_executor_schedule_from_mask(schedule_mask, MALI_FALSE); + + mali_soft_job_destroy(job); + + return _MALI_OSK_ERR_OK; +} + +static void mali_soft_job_send_activated_notification(struct mali_soft_job *job) +{ + if (NULL != job->activated_notification) { + _mali_uk_soft_job_activated_s *res = job->activated_notification->result_buffer; + res->user_job = job->user_job; + mali_session_send_notification(job->system->session, job->activated_notification); + } + job->activated_notification = NULL; +} + +mali_scheduler_mask mali_soft_job_system_activate_job(struct mali_soft_job *job) +{ + mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; + + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_POINTER(job->system); + MALI_DEBUG_ASSERT_POINTER(job->system->session); + + MALI_DEBUG_PRINT(4, ("Mali Soft Job: Timeline activation for soft job %u (0x%08X).\n", job->id, job)); + + mali_soft_job_system_lock(job->system); + + if (unlikely(job->system->session->is_aborting)) { + MALI_DEBUG_PRINT(3, ("Mali Soft Job: Soft job %u (0x%08X) activated while session is aborting.\n", job->id, job)); + + mali_soft_job_system_unlock(job->system); + + /* Since we are in shutdown, we can ignore the scheduling bitmask. */ + mali_timeline_tracker_release(&job->tracker); + mali_soft_job_destroy(job); + return schedule_mask; + } + + /* Send activated notification. */ + mali_soft_job_send_activated_notification(job); + + /* Wake up sleeping signaler. */ + job->activated = MALI_TRUE; + + /* If job type is self signaled, release tracker, move soft job to free list, and scheduler at once */ + if (MALI_SOFT_JOB_TYPE_SELF_SIGNALED == job->type) { + MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_STARTED == job->state); + + job->state = MALI_SOFT_JOB_STATE_SIGNALED; + mali_soft_job_system_unlock(job->system); + + schedule_mask |= mali_timeline_tracker_release(&job->tracker); + + mali_soft_job_destroy(job); + } else { + _mali_osk_wait_queue_wake_up(job->tracker.system->wait_queue); + + mali_soft_job_system_unlock(job->system); + } + + return schedule_mask; +} + +mali_scheduler_mask mali_soft_job_system_timeout_job(struct mali_soft_job *job) +{ + mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; + + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_POINTER(job->system); + MALI_DEBUG_ASSERT_POINTER(job->system->session); + MALI_DEBUG_ASSERT(MALI_TRUE == job->activated); + + MALI_DEBUG_PRINT(4, ("Mali Soft Job: Timeline timeout for soft job %u (0x%08X).\n", job->id, job)); + + mali_soft_job_system_lock(job->system); + + MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_STARTED == job->state || + MALI_SOFT_JOB_STATE_SIGNALED == job->state); + + if (unlikely(job->system->session->is_aborting)) { + /* The session is aborting. This job will be released and destroyed by @ref + * mali_soft_job_system_abort(). */ + mali_soft_job_system_unlock(job->system); + + return MALI_SCHEDULER_MASK_EMPTY; + } + + if (MALI_SOFT_JOB_STATE_STARTED != job->state) { + MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_SIGNALED == job->state); + + /* The job is about to be signaled, ignore timeout. */ + MALI_DEBUG_PRINT(4, ("Mali Soft Job: Timeout on soft job %u (0x%08X) in signaled state.\n", job->id, job)); + mali_soft_job_system_unlock(job->system); + return schedule_mask; + } + + MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_STARTED == job->state); + + job->state = MALI_SOFT_JOB_STATE_TIMED_OUT; + _mali_osk_atomic_inc(&job->refcount); + + mali_soft_job_system_unlock(job->system); + + schedule_mask = mali_timeline_tracker_release(&job->tracker); + + mali_soft_job_destroy(job); + + return schedule_mask; +} + +void mali_soft_job_system_abort(struct mali_soft_job_system *system) +{ + struct mali_soft_job *job, *tmp; + _MALI_OSK_LIST_HEAD_STATIC_INIT(jobs); + + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT_POINTER(system->session); + MALI_DEBUG_ASSERT(system->session->is_aborting); + + MALI_DEBUG_PRINT(3, ("Mali Soft Job: Aborting soft job system for session 0x%08X.\n", system->session)); + + mali_soft_job_system_lock(system); + + _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &system->jobs_used, struct mali_soft_job, system_list) { + MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_STARTED == job->state || + MALI_SOFT_JOB_STATE_TIMED_OUT == job->state); + + if (MALI_SOFT_JOB_STATE_STARTED == job->state) { + /* If the job has been activated, we have to release the tracker and destroy + * the job. If not, the tracker will be released and the job destroyed when + * it is activated. */ + if (MALI_TRUE == job->activated) { + MALI_DEBUG_PRINT(3, ("Mali Soft Job: Aborting unsignaled soft job %u (0x%08X).\n", job->id, job)); + + job->state = MALI_SOFT_JOB_STATE_SIGNALED; + _mali_osk_list_move(&job->system_list, &jobs); + } + } else if (MALI_SOFT_JOB_STATE_TIMED_OUT == job->state) { + MALI_DEBUG_PRINT(3, ("Mali Soft Job: Aborting timed out soft job %u (0x%08X).\n", job->id, job)); + + /* We need to destroy this soft job. */ + _mali_osk_list_move(&job->system_list, &jobs); + } + } + + mali_soft_job_system_unlock(system); + + /* Release and destroy jobs. */ + _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &jobs, struct mali_soft_job, system_list) { + MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_SIGNALED == job->state || + MALI_SOFT_JOB_STATE_TIMED_OUT == job->state); + + if (MALI_SOFT_JOB_STATE_SIGNALED == job->state) { + mali_timeline_tracker_release(&job->tracker); + } + + /* Move job back to used list before destroying. */ + _mali_osk_list_move(&job->system_list, &system->jobs_used); + + mali_soft_job_destroy(job); + } +} diff --git a/drivers/gpu/arm/utgard/common/mali_soft_job.h b/drivers/gpu/arm/utgard/common/mali_soft_job.h new file mode 100644 index 00000000000000..f35394e6038466 --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_soft_job.h @@ -0,0 +1,190 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_SOFT_JOB_H__ +#define __MALI_SOFT_JOB_H__ + +#include "mali_osk.h" + +#include "mali_timeline.h" + +struct mali_timeline_fence; +struct mali_session_data; +struct mali_soft_job; +struct mali_soft_job_system; + +/** + * Soft job types. + * + * Soft jobs of type MALI_SOFT_JOB_TYPE_USER_SIGNALED will only complete after activation if either + * they are signaled by user-space (@ref mali_soft_job_system_signaled_job) or if they are timed out + * by the Timeline system. + * Soft jobs of type MALI_SOFT_JOB_TYPE_SELF_SIGNALED will release job resource automatically + * in kernel when the job is activated. + */ +typedef enum mali_soft_job_type { + MALI_SOFT_JOB_TYPE_SELF_SIGNALED, + MALI_SOFT_JOB_TYPE_USER_SIGNALED, +} mali_soft_job_type; + +/** + * Soft job state. + * + * mali_soft_job_system_start_job a job will first be allocated.The job's state set to MALI_SOFT_JOB_STATE_ALLOCATED. + * Once the job is added to the timeline system, the state changes to MALI_SOFT_JOB_STATE_STARTED. + * + * For soft jobs of type MALI_SOFT_JOB_TYPE_USER_SIGNALED the state is changed to + * MALI_SOFT_JOB_STATE_SIGNALED when @ref mali_soft_job_system_signal_job is called and the soft + * job's state is MALI_SOFT_JOB_STATE_STARTED or MALI_SOFT_JOB_STATE_TIMED_OUT. + * + * If a soft job of type MALI_SOFT_JOB_TYPE_USER_SIGNALED is timed out before being signaled, the + * state is changed to MALI_SOFT_JOB_STATE_TIMED_OUT. This can only happen to soft jobs in state + * MALI_SOFT_JOB_STATE_STARTED. + * + */ +typedef enum mali_soft_job_state { + MALI_SOFT_JOB_STATE_ALLOCATED, + MALI_SOFT_JOB_STATE_STARTED, + MALI_SOFT_JOB_STATE_SIGNALED, + MALI_SOFT_JOB_STATE_TIMED_OUT, +} mali_soft_job_state; + +#define MALI_SOFT_JOB_INVALID_ID ((u32) -1) + +/** + * Soft job struct. + * + * Soft job can be used to represent any kind of CPU work done in kernel-space. + */ +typedef struct mali_soft_job { + mali_soft_job_type type; /**< Soft job type. Must be one of MALI_SOFT_JOB_TYPE_*. */ + u64 user_job; /**< Identifier for soft job in user space. */ + _mali_osk_atomic_t refcount; /**< Soft jobs are reference counted to prevent premature deletion. */ + struct mali_timeline_tracker tracker; /**< Timeline tracker for soft job. */ + mali_bool activated; /**< MALI_TRUE if the job has been activated, MALI_FALSE if not. */ + _mali_osk_notification_t *activated_notification; /**< Pre-allocated notification object for ACTIVATED_NOTIFICATION. */ + + /* Protected by soft job system lock. */ + u32 id; /**< Used by user-space to find corresponding soft job in kernel-space. */ + mali_soft_job_state state; /**< State of soft job, must be one of MALI_SOFT_JOB_STATE_*. */ + struct mali_soft_job_system *system; /**< The soft job system this job is in. */ + _mali_osk_list_t system_list; /**< List element used by soft job system. */ +} mali_soft_job; + +/** + * Per-session soft job system. + * + * The soft job system is used to manage all soft jobs that belongs to a session. + */ +typedef struct mali_soft_job_system { + struct mali_session_data *session; /**< The session this soft job system belongs to. */ + _MALI_OSK_LIST_HEAD(jobs_used); /**< List of all allocated soft jobs. */ + + _mali_osk_spinlock_irq_t *lock; /**< Lock used to protect soft job system and its soft jobs. */ + u32 lock_owner; /**< Contains tid of thread that locked the system or 0, if not locked. */ + u32 last_job_id; /**< Recored the last job id protected by lock. */ +} mali_soft_job_system; + +/** + * Create a soft job system. + * + * @param session The session this soft job system will belong to. + * @return The new soft job system, or NULL if unsuccessful. + */ +struct mali_soft_job_system *mali_soft_job_system_create(struct mali_session_data *session); + +/** + * Destroy a soft job system. + * + * @note The soft job must not have any started or activated jobs. Call @ref + * mali_soft_job_system_abort first. + * + * @param system The soft job system we are destroying. + */ +void mali_soft_job_system_destroy(struct mali_soft_job_system *system); + +/** + * Create a soft job. + * + * @param system Soft job system to create soft job from. + * @param type Type of the soft job. + * @param user_job Identifier for soft job in user space. + * @return New soft job if successful, NULL if not. + */ +struct mali_soft_job *mali_soft_job_create(struct mali_soft_job_system *system, mali_soft_job_type type, u64 user_job); + +/** + * Destroy soft job. + * + * @param job Soft job to destroy. + */ +void mali_soft_job_destroy(struct mali_soft_job *job); + +/** + * Start a soft job. + * + * The soft job will be added to the Timeline system which will then activate it after all + * dependencies have been resolved. + * + * Create soft jobs with @ref mali_soft_job_create before starting them. + * + * @param job Soft job to start. + * @param fence Fence representing dependencies for this soft job. + * @return Point on soft job timeline. + */ +mali_timeline_point mali_soft_job_start(struct mali_soft_job *job, struct mali_timeline_fence *fence); + +/** + * Use by user-space to signal that a soft job has completed. + * + * @note Only valid for soft jobs with type MALI_SOFT_JOB_TYPE_USER_SIGNALED. + * + * @note The soft job must be in state MALI_SOFT_JOB_STATE_STARTED for the signal to be successful. + * + * @note If the soft job was signaled successfully, or it received a time out, the soft job will be + * destroyed after this call and should no longer be used. + * + * @note This function will block until the soft job has been activated. + * + * @param system The soft job system the job was started in. + * @param job_id ID of soft job we are signaling. + * + * @return _MALI_OSK_ERR_ITEM_NOT_FOUND if the soft job ID was invalid, _MALI_OSK_ERR_TIMEOUT if the + * soft job was timed out or _MALI_OSK_ERR_OK if we successfully signaled the soft job. + */ +_mali_osk_errcode_t mali_soft_job_system_signal_job(struct mali_soft_job_system *system, u32 job_id); + +/** + * Used by the Timeline system to activate a soft job. + * + * @param job The soft job that is being activated. + * @return A scheduling bitmask. + */ +mali_scheduler_mask mali_soft_job_system_activate_job(struct mali_soft_job *job); + +/** + * Used by the Timeline system to timeout a soft job. + * + * A soft job is timed out if it completes or is signaled later than MALI_TIMELINE_TIMEOUT_HZ after + * activation. + * + * @param job The soft job that is being timed out. + * @return A scheduling bitmask. + */ +mali_scheduler_mask mali_soft_job_system_timeout_job(struct mali_soft_job *job); + +/** + * Used to cleanup activated soft jobs in the soft job system on session abort. + * + * @param system The soft job system that is being aborted. + */ +void mali_soft_job_system_abort(struct mali_soft_job_system *system); + +#endif /* __MALI_SOFT_JOB_H__ */ diff --git a/drivers/gpu/arm/utgard/common/mali_spinlock_reentrant.c b/drivers/gpu/arm/utgard/common/mali_spinlock_reentrant.c new file mode 100644 index 00000000000000..178abaf43ba1ea --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_spinlock_reentrant.c @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2013, 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_spinlock_reentrant.h" + +#include "mali_osk.h" +#include "mali_kernel_common.h" + +struct mali_spinlock_reentrant *mali_spinlock_reentrant_init(_mali_osk_lock_order_t lock_order) +{ + struct mali_spinlock_reentrant *spinlock; + + spinlock = _mali_osk_calloc(1, sizeof(struct mali_spinlock_reentrant)); + if (NULL == spinlock) { + return NULL; + } + + spinlock->lock = _mali_osk_spinlock_irq_init(_MALI_OSK_LOCKFLAG_ORDERED, lock_order); + if (NULL == spinlock->lock) { + mali_spinlock_reentrant_term(spinlock); + return NULL; + } + + return spinlock; +} + +void mali_spinlock_reentrant_term(struct mali_spinlock_reentrant *spinlock) +{ + MALI_DEBUG_ASSERT_POINTER(spinlock); + MALI_DEBUG_ASSERT(0 == spinlock->counter && 0 == spinlock->owner); + + if (NULL != spinlock->lock) { + _mali_osk_spinlock_irq_term(spinlock->lock); + } + + _mali_osk_free(spinlock); +} + +void mali_spinlock_reentrant_wait(struct mali_spinlock_reentrant *spinlock, u32 tid) +{ + MALI_DEBUG_ASSERT_POINTER(spinlock); + MALI_DEBUG_ASSERT_POINTER(spinlock->lock); + MALI_DEBUG_ASSERT(0 != tid); + + MALI_DEBUG_PRINT(5, ("%s ^\n", __FUNCTION__)); + + if (tid != spinlock->owner) { + _mali_osk_spinlock_irq_lock(spinlock->lock); + MALI_DEBUG_ASSERT(0 == spinlock->owner && 0 == spinlock->counter); + spinlock->owner = tid; + } + + MALI_DEBUG_PRINT(5, ("%s v\n", __FUNCTION__)); + + ++spinlock->counter; +} + +void mali_spinlock_reentrant_signal(struct mali_spinlock_reentrant *spinlock, u32 tid) +{ + MALI_DEBUG_ASSERT_POINTER(spinlock); + MALI_DEBUG_ASSERT_POINTER(spinlock->lock); + MALI_DEBUG_ASSERT(0 != tid && tid == spinlock->owner); + + --spinlock->counter; + if (0 == spinlock->counter) { + spinlock->owner = 0; + MALI_DEBUG_PRINT(5, ("%s release last\n", __FUNCTION__)); + _mali_osk_spinlock_irq_unlock(spinlock->lock); + } +} diff --git a/drivers/gpu/arm/utgard/common/mali_spinlock_reentrant.h b/drivers/gpu/arm/utgard/common/mali_spinlock_reentrant.h new file mode 100644 index 00000000000000..6a62df850b2f56 --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_spinlock_reentrant.h @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2013, 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_SPINLOCK_REENTRANT_H__ +#define __MALI_SPINLOCK_REENTRANT_H__ + +#include "mali_osk.h" +#include "mali_kernel_common.h" + +/** + * Reentrant spinlock. + */ +struct mali_spinlock_reentrant { + _mali_osk_spinlock_irq_t *lock; + u32 owner; + u32 counter; +}; + +/** + * Create a new reentrant spinlock. + * + * @param lock_order Lock order. + * @return New reentrant spinlock. + */ +struct mali_spinlock_reentrant *mali_spinlock_reentrant_init(_mali_osk_lock_order_t lock_order); + +/** + * Terminate reentrant spinlock and free any associated resources. + * + * @param spinlock Reentrant spinlock to terminate. + */ +void mali_spinlock_reentrant_term(struct mali_spinlock_reentrant *spinlock); + +/** + * Wait for reentrant spinlock to be signaled. + * + * @param spinlock Reentrant spinlock. + * @param tid Thread ID. + */ +void mali_spinlock_reentrant_wait(struct mali_spinlock_reentrant *spinlock, u32 tid); + +/** + * Signal reentrant spinlock. + * + * @param spinlock Reentrant spinlock. + * @param tid Thread ID. + */ +void mali_spinlock_reentrant_signal(struct mali_spinlock_reentrant *spinlock, u32 tid); + +/** + * Check if thread is holding reentrant spinlock. + * + * @param spinlock Reentrant spinlock. + * @param tid Thread ID. + * @return MALI_TRUE if thread is holding spinlock, MALI_FALSE if not. + */ +MALI_STATIC_INLINE mali_bool mali_spinlock_reentrant_is_held(struct mali_spinlock_reentrant *spinlock, u32 tid) +{ + MALI_DEBUG_ASSERT_POINTER(spinlock->lock); + return (tid == spinlock->owner && 0 < spinlock->counter); +} + +#endif /* __MALI_SPINLOCK_REENTRANT_H__ */ diff --git a/drivers/gpu/arm/utgard/common/mali_timeline.c b/drivers/gpu/arm/utgard/common/mali_timeline.c new file mode 100644 index 00000000000000..5a767b39d56e02 --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_timeline.c @@ -0,0 +1,1586 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_timeline.h" +#include "mali_kernel_common.h" +#include "mali_scheduler.h" +#include "mali_soft_job.h" +#include "mali_timeline_fence_wait.h" +#include "mali_timeline_sync_fence.h" +#include "mali_executor.h" +#include "mali_pp_job.h" + +#define MALI_TIMELINE_SYSTEM_LOCKED(system) (mali_spinlock_reentrant_is_held((system)->spinlock, _mali_osk_get_tid())) + +/* + * Following three elements are used to record how many + * gp, physical pp or virtual pp jobs are delayed in the whole + * timeline system, we can use these three value to decide + * if need to deactivate idle group. + */ +_mali_osk_atomic_t gp_tracker_count; +_mali_osk_atomic_t phy_pp_tracker_count; +_mali_osk_atomic_t virt_pp_tracker_count; + +static mali_scheduler_mask mali_timeline_system_release_waiter(struct mali_timeline_system *system, + struct mali_timeline_waiter *waiter); + +#if defined(CONFIG_SYNC) +#include +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0) +#include +#include +#include + +struct mali_deferred_fence_put_entry { + struct hlist_node list; + struct sync_fence *fence; +}; + +static HLIST_HEAD(mali_timeline_sync_fence_to_free_list); +static DEFINE_SPINLOCK(mali_timeline_sync_fence_to_free_lock); + +static void put_sync_fences(struct work_struct *ignore) +{ + struct hlist_head list; + struct hlist_node *tmp, *pos; + unsigned long flags; + struct mali_deferred_fence_put_entry *o; + + spin_lock_irqsave(&mali_timeline_sync_fence_to_free_lock, flags); + hlist_move_list(&mali_timeline_sync_fence_to_free_list, &list); + spin_unlock_irqrestore(&mali_timeline_sync_fence_to_free_lock, flags); + + hlist_for_each_entry_safe(o, pos, tmp, &list, list) { + sync_fence_put(o->fence); + kfree(o); + } +} + +static DECLARE_DELAYED_WORK(delayed_sync_fence_put, put_sync_fences); +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0) */ + +/* Callback that is called when a sync fence a tracker is waiting on is signaled. */ +static void mali_timeline_sync_fence_callback(struct sync_fence *sync_fence, struct sync_fence_waiter *sync_fence_waiter) +{ + struct mali_timeline_system *system; + struct mali_timeline_waiter *waiter; + struct mali_timeline_tracker *tracker; + mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; + u32 tid = _mali_osk_get_tid(); + mali_bool is_aborting = MALI_FALSE; +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) + int fence_status = sync_fence->status; +#else + int fence_status = atomic_read(&sync_fence->status); +#endif + + MALI_DEBUG_ASSERT_POINTER(sync_fence); + MALI_DEBUG_ASSERT_POINTER(sync_fence_waiter); + + tracker = _MALI_OSK_CONTAINER_OF(sync_fence_waiter, struct mali_timeline_tracker, sync_fence_waiter); + MALI_DEBUG_ASSERT_POINTER(tracker); + + system = tracker->system; + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT_POINTER(system->session); + + mali_spinlock_reentrant_wait(system->spinlock, tid); + + is_aborting = system->session->is_aborting; + if (!is_aborting && (0 > fence_status)) { + MALI_PRINT_ERROR(("Mali Timeline: sync fence fd %d signaled with error %d\n", tracker->fence.sync_fd, fence_status)); + tracker->activation_error |= MALI_TIMELINE_ACTIVATION_ERROR_SYNC_BIT; + } + + waiter = tracker->waiter_sync; + MALI_DEBUG_ASSERT_POINTER(waiter); + + tracker->sync_fence = NULL; + tracker->fence.sync_fd = -1; + + schedule_mask |= mali_timeline_system_release_waiter(system, waiter); + + /* If aborting, wake up sleepers that are waiting for sync fence callbacks to complete. */ + if (is_aborting) { + _mali_osk_wait_queue_wake_up(system->wait_queue); + } + + mali_spinlock_reentrant_signal(system->spinlock, tid); + + /* + * Older versions of Linux, before 3.5, doesn't support fput() in interrupt + * context. For those older kernels, allocate a list object and put the + * fence object on that and defer the call to sync_fence_put() to a workqueue. + */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0) + { + struct mali_deferred_fence_put_entry *obj; + + obj = kzalloc(sizeof(struct mali_deferred_fence_put_entry), GFP_ATOMIC); + if (obj) { + unsigned long flags; + mali_bool schedule = MALI_FALSE; + + obj->fence = sync_fence; + + spin_lock_irqsave(&mali_timeline_sync_fence_to_free_lock, flags); + if (hlist_empty(&mali_timeline_sync_fence_to_free_list)) + schedule = MALI_TRUE; + hlist_add_head(&obj->list, &mali_timeline_sync_fence_to_free_list); + spin_unlock_irqrestore(&mali_timeline_sync_fence_to_free_lock, flags); + + if (schedule) + schedule_delayed_work(&delayed_sync_fence_put, 0); + } + } +#else + sync_fence_put(sync_fence); +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0) */ + + if (!is_aborting) { + mali_executor_schedule_from_mask(schedule_mask, MALI_TRUE); + } +} +#endif /* defined(CONFIG_SYNC) */ + +static mali_scheduler_mask mali_timeline_tracker_time_out(struct mali_timeline_tracker *tracker) +{ + MALI_DEBUG_ASSERT_POINTER(tracker); + MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_SOFT == tracker->type); + + return mali_soft_job_system_timeout_job((struct mali_soft_job *) tracker->job); +} + +static void mali_timeline_timer_callback(void *data) +{ + struct mali_timeline_system *system; + struct mali_timeline_tracker *tracker; + struct mali_timeline *timeline; + mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; + u32 tid = _mali_osk_get_tid(); + + timeline = (struct mali_timeline *) data; + MALI_DEBUG_ASSERT_POINTER(timeline); + + system = timeline->system; + MALI_DEBUG_ASSERT_POINTER(system); + + mali_spinlock_reentrant_wait(system->spinlock, tid); + + if (!system->timer_enabled) { + mali_spinlock_reentrant_signal(system->spinlock, tid); + return; + } + + tracker = timeline->tracker_tail; + timeline->timer_active = MALI_FALSE; + + if (NULL != tracker && MALI_TRUE == tracker->timer_active) { + /* This is likely the delayed work that has been schedule out before cancelled. */ + if (MALI_TIMELINE_TIMEOUT_HZ > (_mali_osk_time_tickcount() - tracker->os_tick_activate)) { + mali_spinlock_reentrant_signal(system->spinlock, tid); + return; + } + + schedule_mask = mali_timeline_tracker_time_out(tracker); + tracker->timer_active = MALI_FALSE; + } else { + MALI_PRINT_ERROR(("Mali Timeline: Soft job timer callback without a waiting tracker.\n")); + } + + mali_spinlock_reentrant_signal(system->spinlock, tid); + + mali_executor_schedule_from_mask(schedule_mask, MALI_FALSE); +} + +void mali_timeline_system_stop_timer(struct mali_timeline_system *system) +{ + u32 i; + u32 tid = _mali_osk_get_tid(); + + MALI_DEBUG_ASSERT_POINTER(system); + + mali_spinlock_reentrant_wait(system->spinlock, tid); + system->timer_enabled = MALI_FALSE; + mali_spinlock_reentrant_signal(system->spinlock, tid); + + for (i = 0; i < MALI_TIMELINE_MAX; ++i) { + struct mali_timeline *timeline = system->timelines[i]; + + MALI_DEBUG_ASSERT_POINTER(timeline); + + if (NULL != timeline->delayed_work) { + _mali_osk_wq_delayed_cancel_work_sync(timeline->delayed_work); + timeline->timer_active = MALI_FALSE; + } + } +} + +static void mali_timeline_destroy(struct mali_timeline *timeline) +{ + MALI_DEBUG_ASSERT_POINTER(timeline); + if (NULL != timeline) { + /* Assert that the timeline object has been properly cleaned up before destroying it. */ + MALI_DEBUG_ASSERT(timeline->point_oldest == timeline->point_next); + MALI_DEBUG_ASSERT(NULL == timeline->tracker_head); + MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail); + MALI_DEBUG_ASSERT(NULL == timeline->waiter_head); + MALI_DEBUG_ASSERT(NULL == timeline->waiter_tail); + MALI_DEBUG_ASSERT(NULL != timeline->system); + MALI_DEBUG_ASSERT(MALI_TIMELINE_MAX > timeline->id); + +#if defined(CONFIG_SYNC) + if (NULL != timeline->sync_tl) { + sync_timeline_destroy(timeline->sync_tl); + } +#endif /* defined(CONFIG_SYNC) */ + + if (NULL != timeline->delayed_work) { + _mali_osk_wq_delayed_cancel_work_sync(timeline->delayed_work); + _mali_osk_wq_delayed_delete_work_nonflush(timeline->delayed_work); + } + +#ifndef CONFIG_SYNC + _mali_osk_free(timeline); +#endif + } +} + +static struct mali_timeline *mali_timeline_create(struct mali_timeline_system *system, enum mali_timeline_id id) +{ + struct mali_timeline *timeline; + + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT(id < MALI_TIMELINE_MAX); + + timeline = (struct mali_timeline *) _mali_osk_calloc(1, sizeof(struct mali_timeline)); + if (NULL == timeline) { + return NULL; + } + + /* Initially the timeline is empty. */ +#if defined(MALI_TIMELINE_DEBUG_START_POINT) + /* Start the timeline a bit before wrapping when debugging. */ + timeline->point_next = UINT_MAX - MALI_TIMELINE_MAX_POINT_SPAN - 128; +#else + timeline->point_next = 1; +#endif + timeline->point_oldest = timeline->point_next; + + /* The tracker and waiter lists will initially be empty. */ + + timeline->system = system; + timeline->id = id; + + timeline->delayed_work = _mali_osk_wq_delayed_create_work(mali_timeline_timer_callback, timeline); + if (NULL == timeline->delayed_work) { + mali_timeline_destroy(timeline); + return NULL; + } + + timeline->timer_active = MALI_FALSE; + +#if defined(CONFIG_SYNC) + { + char timeline_name[32]; + + switch (id) { + case MALI_TIMELINE_GP: + _mali_osk_snprintf(timeline_name, 32, "mali-%u-gp", _mali_osk_get_pid()); + break; + case MALI_TIMELINE_PP: + _mali_osk_snprintf(timeline_name, 32, "mali-%u-pp", _mali_osk_get_pid()); + break; + case MALI_TIMELINE_SOFT: + _mali_osk_snprintf(timeline_name, 32, "mali-%u-soft", _mali_osk_get_pid()); + break; + default: + MALI_PRINT_ERROR(("Mali Timeline: Invalid timeline id %d\n", id)); + mali_timeline_destroy(timeline); + return NULL; + } + + timeline->destroyed = MALI_FALSE; + + timeline->sync_tl = mali_sync_timeline_create(timeline, timeline_name); + if (NULL == timeline->sync_tl) { + mali_timeline_destroy(timeline); + return NULL; + } + + timeline->spinlock = mali_spinlock_reentrant_init(_MALI_OSK_LOCK_ORDER_TIMELINE_SYSTEM); + if (NULL == timeline->spinlock) { + mali_timeline_destroy(timeline); + return NULL; + } + } +#endif /* defined(CONFIG_SYNC) */ + + return timeline; +} + +static void mali_timeline_insert_tracker(struct mali_timeline *timeline, struct mali_timeline_tracker *tracker) +{ + MALI_DEBUG_ASSERT_POINTER(timeline); + MALI_DEBUG_ASSERT_POINTER(tracker); + + if (mali_timeline_is_full(timeline)) { + /* Don't add tracker if timeline is full. */ + tracker->point = MALI_TIMELINE_NO_POINT; + return; + } + + tracker->timeline = timeline; + tracker->point = timeline->point_next; + + /* Find next available point. */ + timeline->point_next++; + if (MALI_TIMELINE_NO_POINT == timeline->point_next) { + timeline->point_next++; + } + + MALI_DEBUG_ASSERT(!mali_timeline_is_empty(timeline)); + + if (MALI_TIMELINE_TRACKER_GP == tracker->type) { + _mali_osk_atomic_inc(&gp_tracker_count); + } else if (MALI_TIMELINE_TRACKER_PP == tracker->type) { + if (mali_pp_job_is_virtual((struct mali_pp_job *)tracker->job)) { + _mali_osk_atomic_inc(&virt_pp_tracker_count); + } else { + _mali_osk_atomic_inc(&phy_pp_tracker_count); + } + } + + /* Add tracker as new head on timeline's tracker list. */ + if (NULL == timeline->tracker_head) { + /* Tracker list is empty. */ + MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail); + + timeline->tracker_tail = tracker; + + MALI_DEBUG_ASSERT(NULL == tracker->timeline_next); + MALI_DEBUG_ASSERT(NULL == tracker->timeline_prev); + } else { + MALI_DEBUG_ASSERT(NULL == timeline->tracker_head->timeline_next); + + tracker->timeline_prev = timeline->tracker_head; + timeline->tracker_head->timeline_next = tracker; + + MALI_DEBUG_ASSERT(NULL == tracker->timeline_next); + } + timeline->tracker_head = tracker; + + MALI_DEBUG_ASSERT(NULL == timeline->tracker_head->timeline_next); + MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail->timeline_prev); +} + +/* Inserting the waiter object into the given timeline */ +static void mali_timeline_insert_waiter(struct mali_timeline *timeline, struct mali_timeline_waiter *waiter_new) +{ + struct mali_timeline_waiter *waiter_prev; + struct mali_timeline_waiter *waiter_next; + + /* Waiter time must be between timeline head and tail, and there must + * be less than MALI_TIMELINE_MAX_POINT_SPAN elements between */ + MALI_DEBUG_ASSERT((waiter_new->point - timeline->point_oldest) < MALI_TIMELINE_MAX_POINT_SPAN); + MALI_DEBUG_ASSERT((-waiter_new->point + timeline->point_next) < MALI_TIMELINE_MAX_POINT_SPAN); + + /* Finding out where to put this waiter, in the linked waiter list of the given timeline **/ + waiter_prev = timeline->waiter_head; /* Insert new after waiter_prev */ + waiter_next = NULL; /* Insert new before waiter_next */ + + /* Iterating backwards from head (newest) to tail (oldest) until we + * find the correct spot to insert the new waiter */ + while (waiter_prev && mali_timeline_point_after(waiter_prev->point, waiter_new->point)) { + waiter_next = waiter_prev; + waiter_prev = waiter_prev->timeline_prev; + } + + if (NULL == waiter_prev && NULL == waiter_next) { + /* list is empty */ + timeline->waiter_head = waiter_new; + timeline->waiter_tail = waiter_new; + } else if (NULL == waiter_next) { + /* insert at head */ + waiter_new->timeline_prev = timeline->waiter_head; + timeline->waiter_head->timeline_next = waiter_new; + timeline->waiter_head = waiter_new; + } else if (NULL == waiter_prev) { + /* insert at tail */ + waiter_new->timeline_next = timeline->waiter_tail; + timeline->waiter_tail->timeline_prev = waiter_new; + timeline->waiter_tail = waiter_new; + } else { + /* insert between */ + waiter_new->timeline_next = waiter_next; + waiter_new->timeline_prev = waiter_prev; + waiter_next->timeline_prev = waiter_new; + waiter_prev->timeline_next = waiter_new; + } +} + +static void mali_timeline_update_delayed_work(struct mali_timeline *timeline) +{ + struct mali_timeline_system *system; + struct mali_timeline_tracker *oldest_tracker; + + MALI_DEBUG_ASSERT_POINTER(timeline); + MALI_DEBUG_ASSERT(MALI_TIMELINE_SOFT == timeline->id); + + system = timeline->system; + MALI_DEBUG_ASSERT_POINTER(system); + + MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); + + /* Timer is disabled, early out. */ + if (!system->timer_enabled) return; + + oldest_tracker = timeline->tracker_tail; + if (NULL != oldest_tracker && 0 == oldest_tracker->trigger_ref_count) { + if (MALI_FALSE == oldest_tracker->timer_active) { + if (MALI_TRUE == timeline->timer_active) { + _mali_osk_wq_delayed_cancel_work_async(timeline->delayed_work); + } + _mali_osk_wq_delayed_schedule_work(timeline->delayed_work, MALI_TIMELINE_TIMEOUT_HZ); + oldest_tracker->timer_active = MALI_TRUE; + timeline->timer_active = MALI_TRUE; + } + } else if (MALI_TRUE == timeline->timer_active) { + _mali_osk_wq_delayed_cancel_work_async(timeline->delayed_work); + timeline->timer_active = MALI_FALSE; + } +} + +static mali_scheduler_mask mali_timeline_update_oldest_point(struct mali_timeline *timeline) +{ + mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; + + MALI_DEBUG_ASSERT_POINTER(timeline); + + MALI_DEBUG_CODE({ + struct mali_timeline_system *system = timeline->system; + MALI_DEBUG_ASSERT_POINTER(system); + + MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); + }); + + if (NULL != timeline->tracker_tail) { + /* Set oldest point to oldest tracker's point */ + timeline->point_oldest = timeline->tracker_tail->point; + } else { + /* No trackers, mark point list as empty */ + timeline->point_oldest = timeline->point_next; + } + + /* Release all waiters no longer on the timeline's point list. + * Releasing a waiter can trigger this function to be called again, so + * we do not store any pointers on stack. */ + while (NULL != timeline->waiter_tail) { + u32 waiter_time_relative; + u32 time_head_relative; + struct mali_timeline_waiter *waiter = timeline->waiter_tail; + + time_head_relative = timeline->point_next - timeline->point_oldest; + waiter_time_relative = waiter->point - timeline->point_oldest; + + if (waiter_time_relative < time_head_relative) { + /* This and all following waiters are on the point list, so we are done. */ + break; + } + + /* Remove waiter from timeline's waiter list. */ + if (NULL != waiter->timeline_next) { + waiter->timeline_next->timeline_prev = NULL; + } else { + /* This was the last waiter */ + timeline->waiter_head = NULL; + } + timeline->waiter_tail = waiter->timeline_next; + + /* Release waiter. This could activate a tracker, if this was + * the last waiter for the tracker. */ + schedule_mask |= mali_timeline_system_release_waiter(timeline->system, waiter); + } + + return schedule_mask; +} + +void mali_timeline_tracker_init(struct mali_timeline_tracker *tracker, + mali_timeline_tracker_type type, + struct mali_timeline_fence *fence, + void *job) +{ + MALI_DEBUG_ASSERT_POINTER(tracker); + MALI_DEBUG_ASSERT_POINTER(job); + + MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAX > type); + + /* Zero out all tracker members. */ + _mali_osk_memset(tracker, 0, sizeof(*tracker)); + + tracker->type = type; + tracker->job = job; + tracker->trigger_ref_count = 1; /* Prevents any callback from trigging while adding it */ + tracker->os_tick_create = _mali_osk_time_tickcount(); + MALI_DEBUG_CODE(tracker->magic = MALI_TIMELINE_TRACKER_MAGIC); + + tracker->activation_error = MALI_TIMELINE_ACTIVATION_ERROR_NONE; + + /* Copy fence. */ + if (NULL != fence) { + _mali_osk_memcpy(&tracker->fence, fence, sizeof(struct mali_timeline_fence)); + } +} + +mali_scheduler_mask mali_timeline_tracker_release(struct mali_timeline_tracker *tracker) +{ + struct mali_timeline *timeline; + struct mali_timeline_system *system; + struct mali_timeline_tracker *tracker_next, *tracker_prev; + mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; + u32 tid = _mali_osk_get_tid(); + + /* Upon entry a group lock will be held, but not a scheduler lock. */ + MALI_DEBUG_ASSERT_POINTER(tracker); + MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAGIC == tracker->magic); + + /* Tracker should have been triggered */ + MALI_DEBUG_ASSERT(0 == tracker->trigger_ref_count); + + /* All waiters should have been released at this point */ + MALI_DEBUG_ASSERT(NULL == tracker->waiter_head); + MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail); + + MALI_DEBUG_PRINT(3, ("Mali Timeline: releasing tracker for job 0x%08X\n", tracker->job)); + + timeline = tracker->timeline; + if (NULL == timeline) { + /* Tracker was not on a timeline, there is nothing to release. */ + return MALI_SCHEDULER_MASK_EMPTY; + } + + system = timeline->system; + MALI_DEBUG_ASSERT_POINTER(system); + + mali_spinlock_reentrant_wait(system->spinlock, tid); + + /* Tracker should still be on timeline */ + MALI_DEBUG_ASSERT(!mali_timeline_is_empty(timeline)); + MALI_DEBUG_ASSERT(mali_timeline_is_point_on(timeline, tracker->point)); + + /* Tracker is no longer valid. */ + MALI_DEBUG_CODE(tracker->magic = 0); + + tracker_next = tracker->timeline_next; + tracker_prev = tracker->timeline_prev; + tracker->timeline_next = NULL; + tracker->timeline_prev = NULL; + + /* Removing tracker from timeline's tracker list */ + if (NULL == tracker_next) { + /* This tracker was the head */ + timeline->tracker_head = tracker_prev; + } else { + tracker_next->timeline_prev = tracker_prev; + } + + if (NULL == tracker_prev) { + /* This tracker was the tail */ + timeline->tracker_tail = tracker_next; + MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); + /* Update the timeline's oldest time and release any waiters */ + schedule_mask |= mali_timeline_update_oldest_point(timeline); + MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); + } else { + tracker_prev->timeline_next = tracker_next; + } + + MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); + + /* Update delayed work only when it is the soft job timeline */ + if (MALI_TIMELINE_SOFT == tracker->timeline->id) { + mali_timeline_update_delayed_work(tracker->timeline); + } + + mali_spinlock_reentrant_signal(system->spinlock, tid); + + return schedule_mask; +} + +void mali_timeline_system_release_waiter_list(struct mali_timeline_system *system, + struct mali_timeline_waiter *tail, + struct mali_timeline_waiter *head) +{ + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT_POINTER(head); + MALI_DEBUG_ASSERT_POINTER(tail); + MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); + + head->tracker_next = system->waiter_empty_list; + system->waiter_empty_list = tail; +} + +static mali_scheduler_mask mali_timeline_tracker_activate(struct mali_timeline_tracker *tracker) +{ + mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; + struct mali_timeline_system *system; + struct mali_timeline *timeline; + u32 tid = _mali_osk_get_tid(); + + MALI_DEBUG_ASSERT_POINTER(tracker); + MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAGIC == tracker->magic); + + system = tracker->system; + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); + + tracker->os_tick_activate = _mali_osk_time_tickcount(); + + if (NULL != tracker->waiter_head) { + mali_timeline_system_release_waiter_list(system, tracker->waiter_tail, tracker->waiter_head); + tracker->waiter_head = NULL; + tracker->waiter_tail = NULL; + } + + switch (tracker->type) { + case MALI_TIMELINE_TRACKER_GP: + schedule_mask = mali_scheduler_activate_gp_job((struct mali_gp_job *) tracker->job); + + _mali_osk_atomic_dec(&gp_tracker_count); + break; + case MALI_TIMELINE_TRACKER_PP: + if (mali_pp_job_is_virtual((struct mali_pp_job *)tracker->job)) { + _mali_osk_atomic_dec(&virt_pp_tracker_count); + } else { + _mali_osk_atomic_dec(&phy_pp_tracker_count); + } + schedule_mask = mali_scheduler_activate_pp_job((struct mali_pp_job *) tracker->job); + break; + case MALI_TIMELINE_TRACKER_SOFT: + timeline = tracker->timeline; + MALI_DEBUG_ASSERT_POINTER(timeline); + + schedule_mask |= mali_soft_job_system_activate_job((struct mali_soft_job *) tracker->job); + + /* Start a soft timer to make sure the soft job be released in a limited time */ + mali_spinlock_reentrant_wait(system->spinlock, tid); + mali_timeline_update_delayed_work(timeline); + mali_spinlock_reentrant_signal(system->spinlock, tid); + break; + case MALI_TIMELINE_TRACKER_WAIT: + mali_timeline_fence_wait_activate((struct mali_timeline_fence_wait_tracker *) tracker->job); + break; + case MALI_TIMELINE_TRACKER_SYNC: +#if defined(CONFIG_SYNC) + mali_timeline_sync_fence_activate((struct mali_timeline_sync_fence_tracker *) tracker->job); +#else + MALI_PRINT_ERROR(("Mali Timeline: sync tracker not supported\n", tracker->type)); +#endif /* defined(CONFIG_SYNC) */ + break; + default: + MALI_PRINT_ERROR(("Mali Timeline - Illegal tracker type: %d\n", tracker->type)); + break; + } + + return schedule_mask; +} + +void mali_timeline_system_tracker_get(struct mali_timeline_system *system, struct mali_timeline_tracker *tracker) +{ + u32 tid = _mali_osk_get_tid(); + + MALI_DEBUG_ASSERT_POINTER(tracker); + MALI_DEBUG_ASSERT_POINTER(system); + + mali_spinlock_reentrant_wait(system->spinlock, tid); + + MALI_DEBUG_ASSERT(0 < tracker->trigger_ref_count); + tracker->trigger_ref_count++; + + mali_spinlock_reentrant_signal(system->spinlock, tid); +} + +mali_scheduler_mask mali_timeline_system_tracker_put(struct mali_timeline_system *system, struct mali_timeline_tracker *tracker, mali_timeline_activation_error activation_error) +{ + u32 tid = _mali_osk_get_tid(); + mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; + + MALI_DEBUG_ASSERT_POINTER(tracker); + MALI_DEBUG_ASSERT_POINTER(system); + + mali_spinlock_reentrant_wait(system->spinlock, tid); + + MALI_DEBUG_ASSERT(0 < tracker->trigger_ref_count); + tracker->trigger_ref_count--; + + tracker->activation_error |= activation_error; + + if (0 == tracker->trigger_ref_count) { + schedule_mask |= mali_timeline_tracker_activate(tracker); + tracker = NULL; + } + + mali_spinlock_reentrant_signal(system->spinlock, tid); + + return schedule_mask; +} + +void mali_timeline_fence_copy_uk_fence(struct mali_timeline_fence *fence, _mali_uk_fence_t *uk_fence) +{ + u32 i; + + MALI_DEBUG_ASSERT_POINTER(fence); + MALI_DEBUG_ASSERT_POINTER(uk_fence); + + for (i = 0; i < MALI_TIMELINE_MAX; ++i) { + fence->points[i] = uk_fence->points[i]; + } + + fence->sync_fd = uk_fence->sync_fd; +} + +struct mali_timeline_system *mali_timeline_system_create(struct mali_session_data *session) +{ + u32 i; + struct mali_timeline_system *system; + + MALI_DEBUG_ASSERT_POINTER(session); + MALI_DEBUG_PRINT(4, ("Mali Timeline: creating timeline system\n")); + + system = (struct mali_timeline_system *) _mali_osk_calloc(1, sizeof(struct mali_timeline_system)); + if (NULL == system) { + return NULL; + } + + system->spinlock = mali_spinlock_reentrant_init(_MALI_OSK_LOCK_ORDER_TIMELINE_SYSTEM); + if (NULL == system->spinlock) { + mali_timeline_system_destroy(system); + return NULL; + } + + for (i = 0; i < MALI_TIMELINE_MAX; ++i) { + system->timelines[i] = mali_timeline_create(system, (enum mali_timeline_id)i); + if (NULL == system->timelines[i]) { + mali_timeline_system_destroy(system); + return NULL; + } + } + +#if defined(CONFIG_SYNC) + system->signaled_sync_tl = mali_sync_timeline_create(NULL, "mali-always-signaled"); + if (NULL == system->signaled_sync_tl) { + mali_timeline_system_destroy(system); + return NULL; + } +#endif /* defined(CONFIG_SYNC) */ + + system->waiter_empty_list = NULL; + system->session = session; + system->timer_enabled = MALI_TRUE; + + system->wait_queue = _mali_osk_wait_queue_init(); + if (NULL == system->wait_queue) { + mali_timeline_system_destroy(system); + return NULL; + } + + return system; +} + +#if defined(CONFIG_SYNC) + +/** + * Check if there are any trackers left on timeline. + * + * Used as a wait queue conditional. + * + * @param data Timeline. + * @return MALI_TRUE if there are no trackers on timeline, MALI_FALSE if not. + */ +static mali_bool mali_timeline_has_no_trackers(void *data) +{ + struct mali_timeline *timeline = (struct mali_timeline *) data; + + MALI_DEBUG_ASSERT_POINTER(timeline); + + return mali_timeline_is_empty(timeline); +} + +/** + * Cancel sync fence waiters waited upon by trackers on all timelines. + * + * Will return after all timelines have no trackers left. + * + * @param system Timeline system. + */ +static void mali_timeline_cancel_sync_fence_waiters(struct mali_timeline_system *system) +{ + u32 i; + u32 tid = _mali_osk_get_tid(); + struct mali_timeline_tracker *tracker, *tracker_next; + _MALI_OSK_LIST_HEAD_STATIC_INIT(tracker_list); + + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT_POINTER(system->session); + MALI_DEBUG_ASSERT(system->session->is_aborting); + + mali_spinlock_reentrant_wait(system->spinlock, tid); + + /* Cancel sync fence waiters. */ + for (i = 0; i < MALI_TIMELINE_MAX; ++i) { + struct mali_timeline *timeline = system->timelines[i]; + + MALI_DEBUG_ASSERT_POINTER(timeline); + + tracker_next = timeline->tracker_tail; + while (NULL != tracker_next) { + tracker = tracker_next; + tracker_next = tracker->timeline_next; + + if (NULL == tracker->sync_fence) continue; + + MALI_DEBUG_PRINT(3, ("Mali Timeline: Cancelling sync fence wait for tracker 0x%08X.\n", tracker)); + + /* Cancel sync fence waiter. */ + if (0 == sync_fence_cancel_async(tracker->sync_fence, &tracker->sync_fence_waiter)) { + /* Callback was not called, move tracker to local list. */ + _mali_osk_list_add(&tracker->sync_fence_cancel_list, &tracker_list); + } + } + } + + mali_spinlock_reentrant_signal(system->spinlock, tid); + + /* Manually call sync fence callback in order to release waiter and trigger activation of tracker. */ + _MALI_OSK_LIST_FOREACHENTRY(tracker, tracker_next, &tracker_list, struct mali_timeline_tracker, sync_fence_cancel_list) { + mali_timeline_sync_fence_callback(tracker->sync_fence, &tracker->sync_fence_waiter); + } + + /* Sleep until all sync fence callbacks are done and all timelines are empty. */ + for (i = 0; i < MALI_TIMELINE_MAX; ++i) { + struct mali_timeline *timeline = system->timelines[i]; + + MALI_DEBUG_ASSERT_POINTER(timeline); + + _mali_osk_wait_queue_wait_event(system->wait_queue, mali_timeline_has_no_trackers, (void *) timeline); + } +} + +#endif /* defined(CONFIG_SYNC) */ + +void mali_timeline_system_abort(struct mali_timeline_system *system) +{ + MALI_DEBUG_CODE(u32 tid = _mali_osk_get_tid();); + + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT_POINTER(system->session); + MALI_DEBUG_ASSERT(system->session->is_aborting); + + MALI_DEBUG_PRINT(3, ("Mali Timeline: Aborting timeline system for session 0x%08X.\n", system->session)); + +#if defined(CONFIG_SYNC) + mali_timeline_cancel_sync_fence_waiters(system); +#endif /* defined(CONFIG_SYNC) */ + + /* Should not be any waiters or trackers left at this point. */ + MALI_DEBUG_CODE({ + u32 i; + mali_spinlock_reentrant_wait(system->spinlock, tid); + for (i = 0; i < MALI_TIMELINE_MAX; ++i) + { + struct mali_timeline *timeline = system->timelines[i]; + MALI_DEBUG_ASSERT_POINTER(timeline); + MALI_DEBUG_ASSERT(timeline->point_oldest == timeline->point_next); + MALI_DEBUG_ASSERT(NULL == timeline->tracker_head); + MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail); + MALI_DEBUG_ASSERT(NULL == timeline->waiter_head); + MALI_DEBUG_ASSERT(NULL == timeline->waiter_tail); + } + mali_spinlock_reentrant_signal(system->spinlock, tid); + }); +} + +void mali_timeline_system_destroy(struct mali_timeline_system *system) +{ + u32 i; + struct mali_timeline_waiter *waiter, *next; +#if defined(CONFIG_SYNC) + u32 tid = _mali_osk_get_tid(); +#endif + + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT_POINTER(system->session); + + MALI_DEBUG_PRINT(4, ("Mali Timeline: destroying timeline system\n")); + + if (NULL != system) { + + /* There should be no waiters left on this queue. */ + if (NULL != system->wait_queue) { + _mali_osk_wait_queue_term(system->wait_queue); + system->wait_queue = NULL; + } + + /* Free all waiters in empty list */ + waiter = system->waiter_empty_list; + while (NULL != waiter) { + next = waiter->tracker_next; + _mali_osk_free(waiter); + waiter = next; + } + +#if defined(CONFIG_SYNC) + if (NULL != system->signaled_sync_tl) { + sync_timeline_destroy(system->signaled_sync_tl); + } + + for (i = 0; i < MALI_TIMELINE_MAX; ++i) { + if ((NULL != system->timelines[i]) && (NULL != system->timelines[i]->spinlock)) { + mali_spinlock_reentrant_wait(system->timelines[i]->spinlock, tid); + system->timelines[i]->destroyed = MALI_TRUE; + mali_spinlock_reentrant_signal(system->timelines[i]->spinlock, tid); + } + } +#endif /* defined(CONFIG_SYNC) */ + + for (i = 0; i < MALI_TIMELINE_MAX; ++i) { + if (NULL != system->timelines[i]) { + mali_timeline_destroy(system->timelines[i]); + } + } + + if (NULL != system->spinlock) { + mali_spinlock_reentrant_term(system->spinlock); + } + + _mali_osk_free(system); + } +} + +/** + * Find how many waiters are needed for a given fence. + * + * @param fence The fence to check. + * @return Number of waiters needed for fence. + */ +static u32 mali_timeline_fence_num_waiters(struct mali_timeline_fence *fence) +{ + u32 i, num_waiters = 0; + + MALI_DEBUG_ASSERT_POINTER(fence); + + for (i = 0; i < MALI_TIMELINE_MAX; ++i) { + if (MALI_TIMELINE_NO_POINT != fence->points[i]) { + ++num_waiters; + } + } + +#if defined(CONFIG_SYNC) + if (-1 != fence->sync_fd) ++num_waiters; +#endif /* defined(CONFIG_SYNC) */ + + return num_waiters; +} + +static struct mali_timeline_waiter *mali_timeline_system_get_zeroed_waiter(struct mali_timeline_system *system) +{ + struct mali_timeline_waiter *waiter; + + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); + + waiter = system->waiter_empty_list; + if (NULL != waiter) { + /* Remove waiter from empty list and zero it */ + system->waiter_empty_list = waiter->tracker_next; + _mali_osk_memset(waiter, 0, sizeof(*waiter)); + } + + /* Return NULL if list was empty. */ + return waiter; +} + +static void mali_timeline_system_allocate_waiters(struct mali_timeline_system *system, + struct mali_timeline_waiter **tail, + struct mali_timeline_waiter **head, + int max_num_waiters) +{ + u32 i, tid = _mali_osk_get_tid(); + mali_bool do_alloc; + struct mali_timeline_waiter *waiter; + + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT_POINTER(tail); + MALI_DEBUG_ASSERT_POINTER(head); + + MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); + + *head = *tail = NULL; + do_alloc = MALI_FALSE; + i = 0; + while (i < max_num_waiters) { + if (MALI_FALSE == do_alloc) { + waiter = mali_timeline_system_get_zeroed_waiter(system); + if (NULL == waiter) { + do_alloc = MALI_TRUE; + mali_spinlock_reentrant_signal(system->spinlock, tid); + continue; + } + } else { + waiter = _mali_osk_calloc(1, sizeof(struct mali_timeline_waiter)); + if (NULL == waiter) break; + } + ++i; + if (NULL == *tail) { + *tail = waiter; + *head = waiter; + } else { + (*head)->tracker_next = waiter; + *head = waiter; + } + } + if (MALI_TRUE == do_alloc) { + mali_spinlock_reentrant_wait(system->spinlock, tid); + } +} + +/** + * Create waiters for the given tracker. The tracker is activated when all waiters are release. + * + * @note Tracker can potentially be activated before this function returns. + * + * @param system Timeline system. + * @param tracker Tracker we will create waiters for. + * @param waiter_tail List of pre-allocated waiters. + * @param waiter_head List of pre-allocated waiters. + */ +static void mali_timeline_system_create_waiters_and_unlock(struct mali_timeline_system *system, + struct mali_timeline_tracker *tracker, + struct mali_timeline_waiter *waiter_tail, + struct mali_timeline_waiter *waiter_head) +{ + int i; + u32 tid = _mali_osk_get_tid(); + mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; +#if defined(CONFIG_SYNC) + struct sync_fence *sync_fence = NULL; +#endif /* defined(CONFIG_SYNC) */ + + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT_POINTER(tracker); + + MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); + + MALI_DEBUG_ASSERT(NULL == tracker->waiter_head); + MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail); + MALI_DEBUG_ASSERT(NULL != tracker->job); + + /* Creating waiter object for all the timelines the fence is put on. Inserting this waiter + * into the timelines sorted list of waiters */ + for (i = 0; i < MALI_TIMELINE_MAX; ++i) { + mali_timeline_point point; + struct mali_timeline *timeline; + struct mali_timeline_waiter *waiter; + + /* Get point on current timeline from tracker's fence. */ + point = tracker->fence.points[i]; + + if (likely(MALI_TIMELINE_NO_POINT == point)) { + /* Fence contains no point on this timeline so we don't need a waiter. */ + continue; + } + + timeline = system->timelines[i]; + MALI_DEBUG_ASSERT_POINTER(timeline); + + if (unlikely(!mali_timeline_is_point_valid(timeline, point))) { + MALI_PRINT_ERROR(("Mali Timeline: point %d is not valid (oldest=%d, next=%d)\n", + point, timeline->point_oldest, timeline->point_next)); + continue; + } + + if (likely(mali_timeline_is_point_released(timeline, point))) { + /* Tracker representing the point has been released so we don't need a + * waiter. */ + continue; + } + + /* The point is on timeline. */ + MALI_DEBUG_ASSERT(mali_timeline_is_point_on(timeline, point)); + + /* Get a new zeroed waiter object. */ + if (likely(NULL != waiter_tail)) { + waiter = waiter_tail; + waiter_tail = waiter_tail->tracker_next; + } else { + MALI_PRINT_ERROR(("Mali Timeline: failed to allocate memory for waiter\n")); + continue; + } + + /* Yanking the trigger ref count of the tracker. */ + tracker->trigger_ref_count++; + + waiter->point = point; + waiter->tracker = tracker; + + /* Insert waiter on tracker's singly-linked waiter list. */ + if (NULL == tracker->waiter_head) { + /* list is empty */ + MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail); + tracker->waiter_tail = waiter; + } else { + tracker->waiter_head->tracker_next = waiter; + } + tracker->waiter_head = waiter; + + /* Add waiter to timeline. */ + mali_timeline_insert_waiter(timeline, waiter); + } +#if defined(CONFIG_SYNC) + if (-1 != tracker->fence.sync_fd) { + int ret; + struct mali_timeline_waiter *waiter; + + sync_fence = sync_fence_fdget(tracker->fence.sync_fd); + if (unlikely(NULL == sync_fence)) { + MALI_PRINT_ERROR(("Mali Timeline: failed to get sync fence from fd %d\n", tracker->fence.sync_fd)); + goto exit; + } + + /* Check if we have a zeroed waiter object available. */ + if (unlikely(NULL == waiter_tail)) { + MALI_PRINT_ERROR(("Mali Timeline: failed to allocate memory for waiter\n")); + goto exit; + } + + /* Start asynchronous wait that will release waiter when the fence is signaled. */ + sync_fence_waiter_init(&tracker->sync_fence_waiter, mali_timeline_sync_fence_callback); + ret = sync_fence_wait_async(sync_fence, &tracker->sync_fence_waiter); + if (1 == ret) { + /* Fence already signaled, no waiter needed. */ + tracker->fence.sync_fd = -1; + goto exit; + } else if (0 != ret) { + MALI_PRINT_ERROR(("Mali Timeline: sync fence fd %d signaled with error %d\n", tracker->fence.sync_fd, ret)); + tracker->activation_error |= MALI_TIMELINE_ACTIVATION_ERROR_SYNC_BIT; + goto exit; + } + + /* Grab new zeroed waiter object. */ + waiter = waiter_tail; + waiter_tail = waiter_tail->tracker_next; + + /* Increase the trigger ref count of the tracker. */ + tracker->trigger_ref_count++; + + waiter->point = MALI_TIMELINE_NO_POINT; + waiter->tracker = tracker; + + /* Insert waiter on tracker's singly-linked waiter list. */ + if (NULL == tracker->waiter_head) { + /* list is empty */ + MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail); + tracker->waiter_tail = waiter; + } else { + tracker->waiter_head->tracker_next = waiter; + } + tracker->waiter_head = waiter; + + /* Also store waiter in separate field for easy access by sync callback. */ + tracker->waiter_sync = waiter; + + /* Store the sync fence in tracker so we can retrieve in abort session, if needed. */ + tracker->sync_fence = sync_fence; + + sync_fence = NULL; + } +exit: +#endif /* defined(CONFIG_SYNC) */ + + if (NULL != waiter_tail) { + mali_timeline_system_release_waiter_list(system, waiter_tail, waiter_head); + } + + /* Release the initial trigger ref count. */ + tracker->trigger_ref_count--; + + /* If there were no waiters added to this tracker we activate immediately. */ + if (0 == tracker->trigger_ref_count) { + schedule_mask |= mali_timeline_tracker_activate(tracker); + } + + mali_spinlock_reentrant_signal(system->spinlock, tid); + +#if defined(CONFIG_SYNC) + if (NULL != sync_fence) { + sync_fence_put(sync_fence); + } +#endif /* defined(CONFIG_SYNC) */ + + mali_executor_schedule_from_mask(schedule_mask, MALI_FALSE); +} + +mali_timeline_point mali_timeline_system_add_tracker(struct mali_timeline_system *system, + struct mali_timeline_tracker *tracker, + enum mali_timeline_id timeline_id) +{ + int num_waiters = 0; + struct mali_timeline_waiter *waiter_tail, *waiter_head; + u32 tid = _mali_osk_get_tid(); + mali_timeline_point point = MALI_TIMELINE_NO_POINT; + + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT_POINTER(system->session); + MALI_DEBUG_ASSERT_POINTER(tracker); + + MALI_DEBUG_ASSERT(MALI_FALSE == system->session->is_aborting); + MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAX > tracker->type); + MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAGIC == tracker->magic); + + MALI_DEBUG_PRINT(4, ("Mali Timeline: adding tracker for job %p, timeline: %d\n", tracker->job, timeline_id)); + + MALI_DEBUG_ASSERT(0 < tracker->trigger_ref_count); + tracker->system = system; + + mali_spinlock_reentrant_wait(system->spinlock, tid); + + num_waiters = mali_timeline_fence_num_waiters(&tracker->fence); + + /* Allocate waiters. */ + mali_timeline_system_allocate_waiters(system, &waiter_tail, &waiter_head, num_waiters); + MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); + + /* Add tracker to timeline. This will allocate a point for the tracker on the timeline. If + * timeline ID is MALI_TIMELINE_NONE the tracker will NOT be added to a timeline and the + * point will be MALI_TIMELINE_NO_POINT. + * + * NOTE: the tracker can fail to be added if the timeline is full. If this happens, the + * point will be MALI_TIMELINE_NO_POINT. */ + MALI_DEBUG_ASSERT(timeline_id < MALI_TIMELINE_MAX || timeline_id == MALI_TIMELINE_NONE); + if (likely(timeline_id < MALI_TIMELINE_MAX)) { + struct mali_timeline *timeline = system->timelines[timeline_id]; + mali_timeline_insert_tracker(timeline, tracker); + MALI_DEBUG_ASSERT(!mali_timeline_is_empty(timeline)); + } + + point = tracker->point; + + /* Create waiters for tracker based on supplied fence. Each waiter will increase the + * trigger ref count. */ + mali_timeline_system_create_waiters_and_unlock(system, tracker, waiter_tail, waiter_head); + tracker = NULL; + + /* At this point the tracker object might have been freed so we should no longer + * access it. */ + + + /* The tracker will always be activated after calling add_tracker, even if NO_POINT is + * returned. */ + return point; +} + +static mali_scheduler_mask mali_timeline_system_release_waiter(struct mali_timeline_system *system, + struct mali_timeline_waiter *waiter) +{ + struct mali_timeline_tracker *tracker; + mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; + + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT_POINTER(waiter); + + MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); + + tracker = waiter->tracker; + MALI_DEBUG_ASSERT_POINTER(tracker); + + /* At this point the waiter has been removed from the timeline's waiter list, but it is + * still on the tracker's waiter list. All of the tracker's waiters will be released when + * the tracker is activated. */ + + waiter->point = MALI_TIMELINE_NO_POINT; + waiter->tracker = NULL; + + tracker->trigger_ref_count--; + if (0 == tracker->trigger_ref_count) { + /* This was the last waiter; activate tracker */ + schedule_mask |= mali_timeline_tracker_activate(tracker); + tracker = NULL; + } + + return schedule_mask; +} + +mali_timeline_point mali_timeline_system_get_latest_point(struct mali_timeline_system *system, + enum mali_timeline_id timeline_id) +{ + mali_timeline_point point; + struct mali_timeline *timeline; + u32 tid = _mali_osk_get_tid(); + + MALI_DEBUG_ASSERT_POINTER(system); + + if (MALI_TIMELINE_MAX <= timeline_id) { + return MALI_TIMELINE_NO_POINT; + } + + mali_spinlock_reentrant_wait(system->spinlock, tid); + + timeline = system->timelines[timeline_id]; + MALI_DEBUG_ASSERT_POINTER(timeline); + + point = MALI_TIMELINE_NO_POINT; + if (timeline->point_oldest != timeline->point_next) { + point = timeline->point_next - 1; + if (MALI_TIMELINE_NO_POINT == point) point--; + } + + mali_spinlock_reentrant_signal(system->spinlock, tid); + + return point; +} + +void mali_timeline_initialize(void) +{ + _mali_osk_atomic_init(&gp_tracker_count, 0); + _mali_osk_atomic_init(&phy_pp_tracker_count, 0); + _mali_osk_atomic_init(&virt_pp_tracker_count, 0); +} + +void mali_timeline_terminate(void) +{ + _mali_osk_atomic_term(&gp_tracker_count); + _mali_osk_atomic_term(&phy_pp_tracker_count); + _mali_osk_atomic_term(&virt_pp_tracker_count); +} + +#if defined(MALI_TIMELINE_DEBUG_FUNCTIONS) + +static mali_bool is_waiting_on_timeline(struct mali_timeline_tracker *tracker, enum mali_timeline_id id) +{ + struct mali_timeline *timeline; + struct mali_timeline_system *system; + + MALI_DEBUG_ASSERT_POINTER(tracker); + + MALI_DEBUG_ASSERT_POINTER(tracker->timeline); + timeline = tracker->timeline; + + MALI_DEBUG_ASSERT_POINTER(timeline->system); + system = timeline->system; + + if (MALI_TIMELINE_MAX > id) { + if (MALI_TIMELINE_NO_POINT != tracker->fence.points[id]) { + return mali_timeline_is_point_on(system->timelines[id], tracker->fence.points[id]); + } else { + return MALI_FALSE; + } + } else { + MALI_DEBUG_ASSERT(MALI_TIMELINE_NONE == id); + return MALI_FALSE; + } +} + +static const char *timeline_id_to_string(enum mali_timeline_id id) +{ + switch (id) { + case MALI_TIMELINE_GP: + return "GP"; + case MALI_TIMELINE_PP: + return "PP"; + case MALI_TIMELINE_SOFT: + return "SOFT"; + default: + return "NONE"; + } +} + +static const char *timeline_tracker_type_to_string(enum mali_timeline_tracker_type type) +{ + switch (type) { + case MALI_TIMELINE_TRACKER_GP: + return "GP"; + case MALI_TIMELINE_TRACKER_PP: + return "PP"; + case MALI_TIMELINE_TRACKER_SOFT: + return "SOFT"; + case MALI_TIMELINE_TRACKER_WAIT: + return "WAIT"; + case MALI_TIMELINE_TRACKER_SYNC: + return "SYNC"; + default: + return "INVALID"; + } +} + +mali_timeline_tracker_state mali_timeline_debug_get_tracker_state(struct mali_timeline_tracker *tracker) +{ + struct mali_timeline *timeline = NULL; + + MALI_DEBUG_ASSERT_POINTER(tracker); + timeline = tracker->timeline; + + if (0 != tracker->trigger_ref_count) { + return MALI_TIMELINE_TS_WAITING; + } + + if (timeline && (timeline->tracker_tail == tracker || NULL != tracker->timeline_prev)) { + return MALI_TIMELINE_TS_ACTIVE; + } + + if (timeline && (MALI_TIMELINE_NO_POINT == tracker->point)) { + return MALI_TIMELINE_TS_INIT; + } + + return MALI_TIMELINE_TS_FINISH; +} + +void mali_timeline_debug_print_tracker(struct mali_timeline_tracker *tracker, _mali_osk_print_ctx *print_ctx) +{ + const char *tracker_state = "IWAF"; + char state_char = 'I'; + char tracker_type[32] = {0}; + + MALI_DEBUG_ASSERT_POINTER(tracker); + + state_char = *(tracker_state + mali_timeline_debug_get_tracker_state(tracker)); + _mali_osk_snprintf(tracker_type, sizeof(tracker_type), "%s", timeline_tracker_type_to_string(tracker->type)); + +#if defined(CONFIG_SYNC) + if (0 != tracker->trigger_ref_count) { + _mali_osk_ctxprintf(print_ctx, "TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u), fd:%d, fence:(0x%08X)] job:(0x%08X)\n", + tracker_type, tracker->point, state_char, tracker->trigger_ref_count, + is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0], + is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1], + is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2], + tracker->fence.sync_fd, tracker->sync_fence, tracker->job); + } else { + _mali_osk_ctxprintf(print_ctx, "TL: %s %u %c fd:%d fence:(0x%08X) job:(0x%08X)\n", + tracker_type, tracker->point, state_char, + tracker->fence.sync_fd, tracker->sync_fence, tracker->job); + } +#else + if (0 != tracker->trigger_ref_count) { + _mali_osk_ctxprintf(print_ctx, "TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u)] job:(0x%08X)\n", + tracker_type, tracker->point, state_char, tracker->trigger_ref_count, + is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0], + is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1], + is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2], + tracker->job); + } else { + _mali_osk_ctxprintf(print_ctx, "TL: %s %u %c job:(0x%08X)\n", + tracker_type, tracker->point, state_char, + tracker->job); + } +#endif +} + +void mali_timeline_debug_print_timeline(struct mali_timeline *timeline, _mali_osk_print_ctx *print_ctx) +{ + struct mali_timeline_tracker *tracker = NULL; + + MALI_DEBUG_ASSERT_POINTER(timeline); + + tracker = timeline->tracker_tail; + while (NULL != tracker) { + mali_timeline_debug_print_tracker(tracker, print_ctx); + tracker = tracker->timeline_next; + } +} + +#if !(LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)) +void mali_timeline_debug_direct_print_tracker(struct mali_timeline_tracker *tracker) +{ + const char *tracker_state = "IWAF"; + char state_char = 'I'; + char tracker_type[32] = {0}; + + MALI_DEBUG_ASSERT_POINTER(tracker); + + state_char = *(tracker_state + mali_timeline_debug_get_tracker_state(tracker)); + _mali_osk_snprintf(tracker_type, sizeof(tracker_type), "%s", timeline_tracker_type_to_string(tracker->type)); + +#if defined(CONFIG_SYNC) + if (0 != tracker->trigger_ref_count) { + MALI_PRINT(("TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u), fd:%d, fence:(0x%08X)] job:(0x%08X)\n", + tracker_type, tracker->point, state_char, tracker->trigger_ref_count, + is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0], + is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1], + is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2], + tracker->fence.sync_fd, tracker->sync_fence, tracker->job)); + } else { + MALI_PRINT(("TL: %s %u %c fd:%d fence:(0x%08X) job:(0x%08X)\n", + tracker_type, tracker->point, state_char, + tracker->fence.sync_fd, tracker->sync_fence, tracker->job)); + } +#else + if (0 != tracker->trigger_ref_count) { + MALI_PRINT(("TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u)] job:(0x%08X)\n", + tracker_type, tracker->point, state_char, tracker->trigger_ref_count, + is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0], + is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1], + is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2], + tracker->job)); + } else { + MALI_PRINT(("TL: %s %u %c job:(0x%08X)\n", + tracker_type, tracker->point, state_char, + tracker->job)); + } +#endif +} + +void mali_timeline_debug_direct_print_timeline(struct mali_timeline *timeline) +{ + struct mali_timeline_tracker *tracker = NULL; + + MALI_DEBUG_ASSERT_POINTER(timeline); + + tracker = timeline->tracker_tail; + while (NULL != tracker) { + mali_timeline_debug_direct_print_tracker(tracker); + tracker = tracker->timeline_next; + } +} + +#endif + +void mali_timeline_debug_print_system(struct mali_timeline_system *system, _mali_osk_print_ctx *print_ctx) +{ + int i; + int num_printed = 0; + u32 tid = _mali_osk_get_tid(); + + MALI_DEBUG_ASSERT_POINTER(system); + + mali_spinlock_reentrant_wait(system->spinlock, tid); + + /* Print all timelines */ + for (i = 0; i < MALI_TIMELINE_MAX; ++i) { + struct mali_timeline *timeline = system->timelines[i]; + + MALI_DEBUG_ASSERT_POINTER(timeline); + + if (NULL == timeline->tracker_head) continue; + + _mali_osk_ctxprintf(print_ctx, "TL: Timeline %s:\n", + timeline_id_to_string((enum mali_timeline_id)i)); + + mali_timeline_debug_print_timeline(timeline, print_ctx); + num_printed++; + } + + if (0 == num_printed) { + _mali_osk_ctxprintf(print_ctx, "TL: All timelines empty\n"); + } + + mali_spinlock_reentrant_signal(system->spinlock, tid); +} + +#endif /* defined(MALI_TIMELINE_DEBUG_FUNCTIONS) */ diff --git a/drivers/gpu/arm/utgard/common/mali_timeline.h b/drivers/gpu/arm/utgard/common/mali_timeline.h new file mode 100644 index 00000000000000..58d83839f4fec3 --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_timeline.h @@ -0,0 +1,535 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_TIMELINE_H__ +#define __MALI_TIMELINE_H__ + +#include "mali_osk.h" +#include "mali_ukk.h" +#include "mali_session.h" +#include "mali_kernel_common.h" +#include "mali_spinlock_reentrant.h" +#include "mali_sync.h" +#include "mali_scheduler_types.h" +#include + +/** + * Soft job timeout. + * + * Soft jobs have to be signaled as complete after activation. Normally this is done by user space, + * but in order to guarantee that every soft job is completed, we also have a timer. + */ +#define MALI_TIMELINE_TIMEOUT_HZ ((unsigned long) (HZ * 3 / 2)) /* 1500 ms. */ + +/** + * Timeline type. + */ +typedef enum mali_timeline_id { + MALI_TIMELINE_GP = MALI_UK_TIMELINE_GP, /**< GP job timeline. */ + MALI_TIMELINE_PP = MALI_UK_TIMELINE_PP, /**< PP job timeline. */ + MALI_TIMELINE_SOFT = MALI_UK_TIMELINE_SOFT, /**< Soft job timeline. */ + MALI_TIMELINE_MAX = MALI_UK_TIMELINE_MAX +} mali_timeline_id; + +/** + * Used by trackers that should not be added to a timeline (@ref mali_timeline_system_add_tracker). + */ +#define MALI_TIMELINE_NONE MALI_TIMELINE_MAX + +/** + * Tracker type. + */ +typedef enum mali_timeline_tracker_type { + MALI_TIMELINE_TRACKER_GP = 0, /**< Tracker used by GP jobs. */ + MALI_TIMELINE_TRACKER_PP = 1, /**< Tracker used by PP jobs. */ + MALI_TIMELINE_TRACKER_SOFT = 2, /**< Tracker used by soft jobs. */ + MALI_TIMELINE_TRACKER_WAIT = 3, /**< Tracker used for fence wait. */ + MALI_TIMELINE_TRACKER_SYNC = 4, /**< Tracker used for sync fence. */ + MALI_TIMELINE_TRACKER_MAX = 5, +} mali_timeline_tracker_type; + +/** + * Tracker activation error. + */ +typedef u32 mali_timeline_activation_error; +#define MALI_TIMELINE_ACTIVATION_ERROR_NONE 0 +#define MALI_TIMELINE_ACTIVATION_ERROR_SYNC_BIT (1<<1) +#define MALI_TIMELINE_ACTIVATION_ERROR_FATAL_BIT (1<<0) + +/** + * Type used to represent a point on a timeline. + */ +typedef u32 mali_timeline_point; + +/** + * Used to represent that no point on a timeline. + */ +#define MALI_TIMELINE_NO_POINT ((mali_timeline_point) 0) + +/** + * The maximum span of points on a timeline. A timeline will be considered full if the difference + * between the oldest and newest points is equal or larger to this value. + */ +#define MALI_TIMELINE_MAX_POINT_SPAN 65536 + +/** + * Magic value used to assert on validity of trackers. + */ +#define MALI_TIMELINE_TRACKER_MAGIC 0xabcdabcd + +struct mali_timeline; +struct mali_timeline_waiter; +struct mali_timeline_tracker; + +/** + * Timeline fence. + */ +struct mali_timeline_fence { + mali_timeline_point points[MALI_TIMELINE_MAX]; /**< For each timeline, a point or MALI_TIMELINE_NO_POINT. */ + s32 sync_fd; /**< A file descriptor representing a sync fence, or -1. */ +}; + +/** + * Timeline system. + * + * The Timeline system has a set of timelines associated with a session. + */ +struct mali_timeline_system { + struct mali_spinlock_reentrant *spinlock; /**< Spin lock protecting the timeline system */ + struct mali_timeline *timelines[MALI_TIMELINE_MAX]; /**< The timelines in this system */ + + /* Single-linked list of unused waiter objects. Uses the tracker_next field in tracker. */ + struct mali_timeline_waiter *waiter_empty_list; + + struct mali_session_data *session; /**< Session that owns this system. */ + + mali_bool timer_enabled; /**< Set to MALI_TRUE if soft job timer should be enabled, MALI_FALSE if not. */ + + _mali_osk_wait_queue_t *wait_queue; /**< Wait queue. */ + +#if defined(CONFIG_SYNC) + struct sync_timeline *signaled_sync_tl; /**< Special sync timeline used to create pre-signaled sync fences */ +#endif /* defined(CONFIG_SYNC) */ +}; + +/** + * Timeline. Each Timeline system will have MALI_TIMELINE_MAX timelines. + */ +struct mali_timeline { + mali_timeline_point point_next; /**< The next available point. */ + mali_timeline_point point_oldest; /**< The oldest point not released. */ + + /* Double-linked list of trackers. Sorted in ascending order by tracker->time_number with + * tail pointing to the tracker with the oldest time. */ + struct mali_timeline_tracker *tracker_head; + struct mali_timeline_tracker *tracker_tail; + + /* Double-linked list of waiters. Sorted in ascending order by waiter->time_number_wait + * with tail pointing to the waiter with oldest wait time. */ + struct mali_timeline_waiter *waiter_head; + struct mali_timeline_waiter *waiter_tail; + + struct mali_timeline_system *system; /**< Timeline system this timeline belongs to. */ + enum mali_timeline_id id; /**< Timeline type. */ + +#if defined(CONFIG_SYNC) + struct sync_timeline *sync_tl; /**< Sync timeline that corresponds to this timeline. */ + mali_bool destroyed; + struct mali_spinlock_reentrant *spinlock; /**< Spin lock protecting the timeline system */ +#endif /* defined(CONFIG_SYNC) */ + + /* The following fields are used to time out soft job trackers. */ + _mali_osk_wq_delayed_work_t *delayed_work; + mali_bool timer_active; +}; + +/** + * Timeline waiter. + */ +struct mali_timeline_waiter { + mali_timeline_point point; /**< Point on timeline we are waiting for to be released. */ + struct mali_timeline_tracker *tracker; /**< Tracker that is waiting. */ + + struct mali_timeline_waiter *timeline_next; /**< Next waiter on timeline's waiter list. */ + struct mali_timeline_waiter *timeline_prev; /**< Previous waiter on timeline's waiter list. */ + + struct mali_timeline_waiter *tracker_next; /**< Next waiter on tracker's waiter list. */ +}; + +/** + * Timeline tracker. + */ +struct mali_timeline_tracker { + MALI_DEBUG_CODE(u32 magic); /**< Should always be MALI_TIMELINE_TRACKER_MAGIC for a valid tracker. */ + + mali_timeline_point point; /**< Point on timeline for this tracker */ + + struct mali_timeline_tracker *timeline_next; /**< Next tracker on timeline's tracker list */ + struct mali_timeline_tracker *timeline_prev; /**< Previous tracker on timeline's tracker list */ + + u32 trigger_ref_count; /**< When zero tracker will be activated */ + mali_timeline_activation_error activation_error; /**< Activation error. */ + struct mali_timeline_fence fence; /**< Fence used to create this tracker */ + + /* Single-linked list of waiters. Sorted in order of insertions with + * tail pointing to first waiter. */ + struct mali_timeline_waiter *waiter_head; + struct mali_timeline_waiter *waiter_tail; + +#if defined(CONFIG_SYNC) + /* These are only used if the tracker is waiting on a sync fence. */ + struct mali_timeline_waiter *waiter_sync; /**< A direct pointer to timeline waiter representing sync fence. */ + struct sync_fence_waiter sync_fence_waiter; /**< Used to connect sync fence and tracker in sync fence wait callback. */ + struct sync_fence *sync_fence; /**< The sync fence this tracker is waiting on. */ + _mali_osk_list_t sync_fence_cancel_list; /**< List node used to cancel sync fence waiters. */ +#endif /* defined(CONFIG_SYNC) */ + + struct mali_timeline_system *system; /**< Timeline system. */ + struct mali_timeline *timeline; /**< Timeline, or NULL if not on a timeline. */ + enum mali_timeline_tracker_type type; /**< Type of tracker. */ + void *job; /**< Owner of tracker. */ + + /* The following fields are used to time out soft job trackers. */ + unsigned long os_tick_create; + unsigned long os_tick_activate; + mali_bool timer_active; +}; + +extern _mali_osk_atomic_t gp_tracker_count; +extern _mali_osk_atomic_t phy_pp_tracker_count; +extern _mali_osk_atomic_t virt_pp_tracker_count; + +/** + * What follows is a set of functions to check the state of a timeline and to determine where on a + * timeline a given point is. Most of these checks will translate the timeline so the oldest point + * on the timeline is aligned with zero. Remember that all of these calculation are done on + * unsigned integers. + * + * The following example illustrates the three different states a point can be in. The timeline has + * been translated to put the oldest point at zero: + * + * + * + * [ point is in forbidden zone ] + * 64k wide + * MALI_TIMELINE_MAX_POINT_SPAN + * + * [ point is on timeline ) ( point is released ] + * + * 0--------------------------##############################--------------------2^32 - 1 + * ^ ^ + * \ | + * oldest point on timeline | + * \ + * next point on timeline + */ + +/** + * Compare two timeline points + * + * Returns true if a is after b, false if a is before or equal to b. + * + * This funcion ignores MALI_TIMELINE_MAX_POINT_SPAN. Wrapping is supported and + * the result will be correct if the points is less then UINT_MAX/2 apart. + * + * @param a Point on timeline + * @param b Point on timeline + * @return MALI_TRUE if a is after b + */ +MALI_STATIC_INLINE mali_bool mali_timeline_point_after(mali_timeline_point a, mali_timeline_point b) +{ + return 0 > ((s32)b) - ((s32)a); +} + +/** + * Check if a point is on timeline. A point is on a timeline if it is greater than, or equal to, + * the oldest point, and less than the next point. + * + * @param timeline Timeline. + * @param point Point on timeline. + * @return MALI_TRUE if point is on timeline, MALI_FALSE if not. + */ +MALI_STATIC_INLINE mali_bool mali_timeline_is_point_on(struct mali_timeline *timeline, mali_timeline_point point) +{ + MALI_DEBUG_ASSERT_POINTER(timeline); + MALI_DEBUG_ASSERT(MALI_TIMELINE_NO_POINT != point); + + return (point - timeline->point_oldest) < (timeline->point_next - timeline->point_oldest); +} + +/** + * Check if a point has been released. A point is released if it is older than the oldest point on + * the timeline, newer than the next point, and also not in the forbidden zone. + * + * @param timeline Timeline. + * @param point Point on timeline. + * @return MALI_TRUE if point has been release, MALI_FALSE if not. + */ +MALI_STATIC_INLINE mali_bool mali_timeline_is_point_released(struct mali_timeline *timeline, mali_timeline_point point) +{ + mali_timeline_point point_normalized; + mali_timeline_point next_normalized; + + MALI_DEBUG_ASSERT_POINTER(timeline); + MALI_DEBUG_ASSERT(MALI_TIMELINE_NO_POINT != point); + + point_normalized = point - timeline->point_oldest; + next_normalized = timeline->point_next - timeline->point_oldest; + + return point_normalized > (next_normalized + MALI_TIMELINE_MAX_POINT_SPAN); +} + +/** + * Check if a point is valid. A point is valid if is on the timeline or has been released. + * + * @param timeline Timeline. + * @param point Point on timeline. + * @return MALI_TRUE if point is valid, MALI_FALSE if not. + */ +MALI_STATIC_INLINE mali_bool mali_timeline_is_point_valid(struct mali_timeline *timeline, mali_timeline_point point) +{ + MALI_DEBUG_ASSERT_POINTER(timeline); + return mali_timeline_is_point_on(timeline, point) || mali_timeline_is_point_released(timeline, point); +} + +/** + * Check if timeline is empty (has no points on it). A timeline is empty if next == oldest. + * + * @param timeline Timeline. + * @return MALI_TRUE if timeline is empty, MALI_FALSE if not. + */ +MALI_STATIC_INLINE mali_bool mali_timeline_is_empty(struct mali_timeline *timeline) +{ + MALI_DEBUG_ASSERT_POINTER(timeline); + return timeline->point_next == timeline->point_oldest; +} + +/** + * Check if timeline is full. A valid timeline cannot span more than 64k points (@ref + * MALI_TIMELINE_MAX_POINT_SPAN). + * + * @param timeline Timeline. + * @return MALI_TRUE if timeline is full, MALI_FALSE if not. + */ +MALI_STATIC_INLINE mali_bool mali_timeline_is_full(struct mali_timeline *timeline) +{ + MALI_DEBUG_ASSERT_POINTER(timeline); + return MALI_TIMELINE_MAX_POINT_SPAN <= (timeline->point_next - timeline->point_oldest); +} + +/** + * Create a new timeline system. + * + * @param session The session this timeline system will belong to. + * @return New timeline system. + */ +struct mali_timeline_system *mali_timeline_system_create(struct mali_session_data *session); + +/** + * Abort timeline system. + * + * This will release all pending waiters in the timeline system causing all trackers to be + * activated. + * + * @param system Timeline system to abort all jobs from. + */ +void mali_timeline_system_abort(struct mali_timeline_system *system); + +/** + * Destroy an empty timeline system. + * + * @note @ref mali_timeline_system_abort() should be called prior to this function. + * + * @param system Timeline system to destroy. + */ +void mali_timeline_system_destroy(struct mali_timeline_system *system); + +/** + * Stop the soft job timer. + * + * @param system Timeline system + */ +void mali_timeline_system_stop_timer(struct mali_timeline_system *system); + +/** + * Add a tracker to a timeline system and optionally also on a timeline. + * + * Once added to the timeline system, the tracker is guaranteed to be activated. The tracker can be + * activated before this function returns. Thus, it is also possible that the tracker is released + * before this function returns, depending on the tracker type. + * + * @note Tracker must be initialized (@ref mali_timeline_tracker_init) before being added to the + * timeline system. + * + * @param system Timeline system the tracker will be added to. + * @param tracker The tracker to be added. + * @param timeline_id Id of the timeline the tracker will be added to, or + * MALI_TIMELINE_NONE if it should not be added on a timeline. + * @return Point on timeline identifying this tracker, or MALI_TIMELINE_NO_POINT if not on timeline. + */ +mali_timeline_point mali_timeline_system_add_tracker(struct mali_timeline_system *system, + struct mali_timeline_tracker *tracker, + enum mali_timeline_id timeline_id); + +/** + * Get latest point on timeline. + * + * @param system Timeline system. + * @param timeline_id Id of timeline to get latest point from. + * @return Latest point on timeline, or MALI_TIMELINE_NO_POINT if the timeline is empty. + */ +mali_timeline_point mali_timeline_system_get_latest_point(struct mali_timeline_system *system, + enum mali_timeline_id timeline_id); + +/** + * Initialize tracker. + * + * Must be called before tracker is added to timeline system (@ref mali_timeline_system_add_tracker). + * + * @param tracker Tracker to initialize. + * @param type Type of tracker. + * @param fence Fence used to set up dependencies for tracker. + * @param job Pointer to job struct this tracker is associated with. + */ +void mali_timeline_tracker_init(struct mali_timeline_tracker *tracker, + mali_timeline_tracker_type type, + struct mali_timeline_fence *fence, + void *job); + +/** + * Grab trigger ref count on tracker. + * + * This will prevent tracker from being activated until the trigger ref count reaches zero. + * + * @note Tracker must have been initialized (@ref mali_timeline_tracker_init). + * + * @param system Timeline system. + * @param tracker Tracker. + */ +void mali_timeline_system_tracker_get(struct mali_timeline_system *system, struct mali_timeline_tracker *tracker); + +/** + * Release trigger ref count on tracker. + * + * If the trigger ref count reaches zero, the tracker will be activated. + * + * @param system Timeline system. + * @param tracker Tracker. + * @param activation_error Error bitmask if activated with error, or MALI_TIMELINE_ACTIVATION_ERROR_NONE if no error. + * @return Scheduling bitmask. + */ +mali_scheduler_mask mali_timeline_system_tracker_put(struct mali_timeline_system *system, struct mali_timeline_tracker *tracker, mali_timeline_activation_error activation_error); + +/** + * Release a tracker from the timeline system. + * + * This is used to signal that the job being tracker is finished, either due to normal circumstances + * (job complete/abort) or due to a timeout. + * + * We may need to schedule some subsystems after a tracker has been released and the returned + * bitmask will tell us if it is necessary. If the return value is non-zero, this value needs to be + * sent as an input parameter to @ref mali_scheduler_schedule_from_mask() to do the scheduling. + * + * @note Tracker must have been activated before being released. + * @warning Not calling @ref mali_scheduler_schedule_from_mask() after releasing a tracker can lead + * to a deadlock. + * + * @param tracker Tracker being released. + * @return Scheduling bitmask. + */ +mali_scheduler_mask mali_timeline_tracker_release(struct mali_timeline_tracker *tracker); + +MALI_STATIC_INLINE mali_bool mali_timeline_tracker_activation_error( + struct mali_timeline_tracker *tracker) +{ + MALI_DEBUG_ASSERT_POINTER(tracker); + return (MALI_TIMELINE_ACTIVATION_ERROR_FATAL_BIT & + tracker->activation_error) ? MALI_TRUE : MALI_FALSE; +} + +/** + * Copy data from a UK fence to a Timeline fence. + * + * @param fence Timeline fence. + * @param uk_fence UK fence. + */ +void mali_timeline_fence_copy_uk_fence(struct mali_timeline_fence *fence, _mali_uk_fence_t *uk_fence); + +void mali_timeline_initialize(void); + +void mali_timeline_terminate(void); + +MALI_STATIC_INLINE mali_bool mali_timeline_has_gp_job(void) +{ + return 0 < _mali_osk_atomic_read(&gp_tracker_count); +} + +MALI_STATIC_INLINE mali_bool mali_timeline_has_physical_pp_job(void) +{ + return 0 < _mali_osk_atomic_read(&phy_pp_tracker_count); +} + +MALI_STATIC_INLINE mali_bool mali_timeline_has_virtual_pp_job(void) +{ + return 0 < _mali_osk_atomic_read(&virt_pp_tracker_count); +} + +#if defined(DEBUG) +#define MALI_TIMELINE_DEBUG_FUNCTIONS +#endif /* DEBUG */ +#if defined(MALI_TIMELINE_DEBUG_FUNCTIONS) + +/** + * Tracker state. Used for debug printing. + */ +typedef enum mali_timeline_tracker_state { + MALI_TIMELINE_TS_INIT = 0, + MALI_TIMELINE_TS_WAITING = 1, + MALI_TIMELINE_TS_ACTIVE = 2, + MALI_TIMELINE_TS_FINISH = 3, +} mali_timeline_tracker_state; + +/** + * Get tracker state. + * + * @param tracker Tracker to check. + * @return State of tracker. + */ +mali_timeline_tracker_state mali_timeline_debug_get_tracker_state(struct mali_timeline_tracker *tracker); + +/** + * Print debug information about tracker. + * + * @param tracker Tracker to print. + */ +void mali_timeline_debug_print_tracker(struct mali_timeline_tracker *tracker, _mali_osk_print_ctx *print_ctx); + +/** + * Print debug information about timeline. + * + * @param timeline Timeline to print. + */ +void mali_timeline_debug_print_timeline(struct mali_timeline *timeline, _mali_osk_print_ctx *print_ctx); + +#if !(LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)) +void mali_timeline_debug_direct_print_tracker(struct mali_timeline_tracker *tracker); +void mali_timeline_debug_direct_print_timeline(struct mali_timeline *timeline); +#endif + +/** + * Print debug information about timeline system. + * + * @param system Timeline system to print. + */ +void mali_timeline_debug_print_system(struct mali_timeline_system *system, _mali_osk_print_ctx *print_ctx); + +#endif /* defined(MALI_TIMELINE_DEBUG_FUNCTIONS) */ + +#endif /* __MALI_TIMELINE_H__ */ diff --git a/drivers/gpu/arm/utgard/common/mali_timeline_fence_wait.c b/drivers/gpu/arm/utgard/common/mali_timeline_fence_wait.c new file mode 100644 index 00000000000000..3c58928dd3a2d4 --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_timeline_fence_wait.c @@ -0,0 +1,202 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_timeline_fence_wait.h" + +#include "mali_osk.h" +#include "mali_kernel_common.h" +#include "mali_spinlock_reentrant.h" + +/** + * Allocate a fence waiter tracker. + * + * @return New fence waiter if successful, NULL if not. + */ +static struct mali_timeline_fence_wait_tracker *mali_timeline_fence_wait_tracker_alloc(void) +{ + return (struct mali_timeline_fence_wait_tracker *) _mali_osk_calloc(1, sizeof(struct mali_timeline_fence_wait_tracker)); +} + +/** + * Free fence waiter tracker. + * + * @param wait Fence wait tracker to free. + */ +static void mali_timeline_fence_wait_tracker_free(struct mali_timeline_fence_wait_tracker *wait) +{ + MALI_DEBUG_ASSERT_POINTER(wait); + _mali_osk_atomic_term(&wait->refcount); + _mali_osk_free(wait); +} + +/** + * Check if fence wait tracker has been activated. Used as a wait queue condition. + * + * @param data Fence waiter. + * @return MALI_TRUE if tracker has been activated, MALI_FALSE if not. + */ +static mali_bool mali_timeline_fence_wait_tracker_is_activated(void *data) +{ + struct mali_timeline_fence_wait_tracker *wait; + + wait = (struct mali_timeline_fence_wait_tracker *) data; + MALI_DEBUG_ASSERT_POINTER(wait); + + return wait->activated; +} + +/** + * Check if fence has been signaled. + * + * @param system Timeline system. + * @param fence Timeline fence. + * @return MALI_TRUE if fence is signaled, MALI_FALSE if not. + */ +static mali_bool mali_timeline_fence_wait_check_status(struct mali_timeline_system *system, struct mali_timeline_fence *fence) +{ + int i; + u32 tid = _mali_osk_get_tid(); + mali_bool ret = MALI_TRUE; +#if defined(CONFIG_SYNC) + struct sync_fence *sync_fence = NULL; +#endif + + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT_POINTER(fence); + + mali_spinlock_reentrant_wait(system->spinlock, tid); + + for (i = 0; i < MALI_TIMELINE_MAX; ++i) { + struct mali_timeline *timeline; + mali_timeline_point point; + + point = fence->points[i]; + + if (likely(MALI_TIMELINE_NO_POINT == point)) { + /* Fence contains no point on this timeline. */ + continue; + } + + timeline = system->timelines[i]; + MALI_DEBUG_ASSERT_POINTER(timeline); + + if (unlikely(!mali_timeline_is_point_valid(timeline, point))) { + MALI_PRINT_ERROR(("Mali Timeline: point %d is not valid (oldest=%d, next=%d)\n", point, timeline->point_oldest, timeline->point_next)); + } + + if (!mali_timeline_is_point_released(timeline, point)) { + ret = MALI_FALSE; + goto exit; + } + } + +#if defined(CONFIG_SYNC) + if (-1 != fence->sync_fd) { + sync_fence = sync_fence_fdget(fence->sync_fd); + if (likely(NULL != sync_fence)) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) + if (0 == sync_fence->status) { +#else + if (0 == atomic_read(&sync_fence->status)) { +#endif + ret = MALI_FALSE; + } + } else { + MALI_PRINT_ERROR(("Mali Timeline: failed to get sync fence from fd %d\n", fence->sync_fd)); + } + } +#endif /* defined(CONFIG_SYNC) */ + +exit: + mali_spinlock_reentrant_signal(system->spinlock, tid); + +#if defined(CONFIG_SYNC) + if (NULL != sync_fence) { + sync_fence_put(sync_fence); + } +#endif /* defined(CONFIG_SYNC) */ + + return ret; +} + +mali_bool mali_timeline_fence_wait(struct mali_timeline_system *system, struct mali_timeline_fence *fence, u32 timeout) +{ + struct mali_timeline_fence_wait_tracker *wait; + mali_timeline_point point; + mali_bool ret; + + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT_POINTER(fence); + + MALI_DEBUG_PRINT(4, ("Mali Timeline: wait on fence\n")); + + if (MALI_TIMELINE_FENCE_WAIT_TIMEOUT_IMMEDIATELY == timeout) { + return mali_timeline_fence_wait_check_status(system, fence); + } + + wait = mali_timeline_fence_wait_tracker_alloc(); + if (unlikely(NULL == wait)) { + MALI_PRINT_ERROR(("Mali Timeline: failed to allocate data for fence wait\n")); + return MALI_FALSE; + } + + wait->activated = MALI_FALSE; + wait->system = system; + + /* Initialize refcount to two references. The reference first will be released by this + * function after the wait is over. The second reference will be released when the tracker + * is activated. */ + _mali_osk_atomic_init(&wait->refcount, 2); + + /* Add tracker to timeline system, but not to a timeline. */ + mali_timeline_tracker_init(&wait->tracker, MALI_TIMELINE_TRACKER_WAIT, fence, wait); + point = mali_timeline_system_add_tracker(system, &wait->tracker, MALI_TIMELINE_NONE); + MALI_DEBUG_ASSERT(MALI_TIMELINE_NO_POINT == point); + MALI_IGNORE(point); + + /* Wait for the tracker to be activated or time out. */ + if (MALI_TIMELINE_FENCE_WAIT_TIMEOUT_NEVER == timeout) { + _mali_osk_wait_queue_wait_event(system->wait_queue, mali_timeline_fence_wait_tracker_is_activated, (void *) wait); + } else { + _mali_osk_wait_queue_wait_event_timeout(system->wait_queue, mali_timeline_fence_wait_tracker_is_activated, (void *) wait, timeout); + } + + ret = wait->activated; + + if (0 == _mali_osk_atomic_dec_return(&wait->refcount)) { + mali_timeline_fence_wait_tracker_free(wait); + } + + return ret; +} + +void mali_timeline_fence_wait_activate(struct mali_timeline_fence_wait_tracker *wait) +{ + mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; + + MALI_DEBUG_ASSERT_POINTER(wait); + MALI_DEBUG_ASSERT_POINTER(wait->system); + + MALI_DEBUG_PRINT(4, ("Mali Timeline: activation for fence wait tracker\n")); + + MALI_DEBUG_ASSERT(MALI_FALSE == wait->activated); + wait->activated = MALI_TRUE; + + _mali_osk_wait_queue_wake_up(wait->system->wait_queue); + + /* Nothing can wait on this tracker, so nothing to schedule after release. */ + schedule_mask = mali_timeline_tracker_release(&wait->tracker); + MALI_DEBUG_ASSERT(MALI_SCHEDULER_MASK_EMPTY == schedule_mask); + MALI_IGNORE(schedule_mask); + + if (0 == _mali_osk_atomic_dec_return(&wait->refcount)) { + mali_timeline_fence_wait_tracker_free(wait); + } +} diff --git a/drivers/gpu/arm/utgard/common/mali_timeline_fence_wait.h b/drivers/gpu/arm/utgard/common/mali_timeline_fence_wait.h new file mode 100644 index 00000000000000..f5440ab6fc6d8c --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_timeline_fence_wait.h @@ -0,0 +1,67 @@ +/* + * Copyright (C) 2013, 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_timeline_fence_wait.h + * + * This file contains functions used to wait until a Timeline fence is signaled. + */ + +#ifndef __MALI_TIMELINE_FENCE_WAIT_H__ +#define __MALI_TIMELINE_FENCE_WAIT_H__ + +#include "mali_osk.h" +#include "mali_timeline.h" + +/** + * If used as the timeout argument in @ref mali_timeline_fence_wait, a timer is not used and the + * function only returns when the fence is signaled. + */ +#define MALI_TIMELINE_FENCE_WAIT_TIMEOUT_NEVER ((u32) -1) + +/** + * If used as the timeout argument in @ref mali_timeline_fence_wait, the function will return + * immediately with the current state of the fence. + */ +#define MALI_TIMELINE_FENCE_WAIT_TIMEOUT_IMMEDIATELY 0 + +/** + * Fence wait tracker. + * + * The fence wait tracker is added to the Timeline system with the fence we are waiting on as a + * dependency. We will then perform a blocking wait, possibly with a timeout, until the tracker is + * activated, which happens when the fence is signaled. + */ +struct mali_timeline_fence_wait_tracker { + mali_bool activated; /**< MALI_TRUE if the tracker has been activated, MALI_FALSE if not. */ + _mali_osk_atomic_t refcount; /**< Reference count. */ + struct mali_timeline_system *system; /**< Timeline system. */ + struct mali_timeline_tracker tracker; /**< Timeline tracker. */ +}; + +/** + * Wait for a fence to be signaled, or timeout is reached. + * + * @param system Timeline system. + * @param fence Fence to wait on. + * @param timeout Timeout in ms, or MALI_TIMELINE_FENCE_WAIT_TIMEOUT_NEVER or + * MALI_TIMELINE_FENCE_WAIT_TIMEOUT_IMMEDIATELY. + * @return MALI_TRUE if signaled, MALI_FALSE if timed out. + */ +mali_bool mali_timeline_fence_wait(struct mali_timeline_system *system, struct mali_timeline_fence *fence, u32 timeout); + +/** + * Used by the Timeline system to activate a fence wait tracker. + * + * @param fence_wait_tracker Fence waiter tracker. + */ +void mali_timeline_fence_wait_activate(struct mali_timeline_fence_wait_tracker *fence_wait_tracker); + +#endif /* __MALI_TIMELINE_FENCE_WAIT_H__ */ diff --git a/drivers/gpu/arm/utgard/common/mali_timeline_sync_fence.c b/drivers/gpu/arm/utgard/common/mali_timeline_sync_fence.c new file mode 100644 index 00000000000000..73843f07c9905b --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_timeline_sync_fence.c @@ -0,0 +1,158 @@ +/* + * Copyright (C) 2013, 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_timeline_sync_fence.h" + +#include "mali_osk.h" +#include "mali_kernel_common.h" +#include "mali_sync.h" + +#if defined(CONFIG_SYNC) + +/** + * Creates a sync fence tracker and a sync fence. Adds sync fence tracker to Timeline system and + * returns sync fence. The sync fence will be signaled when the sync fence tracker is activated. + * + * @param timeline Timeline. + * @param point Point on timeline. + * @return Sync fence that will be signaled when tracker is activated. + */ +static struct sync_fence *mali_timeline_sync_fence_create_and_add_tracker(struct mali_timeline *timeline, mali_timeline_point point) +{ + struct mali_timeline_sync_fence_tracker *sync_fence_tracker; + struct sync_fence *sync_fence; + struct mali_timeline_fence fence; + + MALI_DEBUG_ASSERT_POINTER(timeline); + MALI_DEBUG_ASSERT(MALI_TIMELINE_NO_POINT != point); + + /* Allocate sync fence tracker. */ + sync_fence_tracker = _mali_osk_calloc(1, sizeof(struct mali_timeline_sync_fence_tracker)); + if (NULL == sync_fence_tracker) { + MALI_PRINT_ERROR(("Mali Timeline: sync_fence_tracker allocation failed\n")); + return NULL; + } + + /* Create sync flag. */ + MALI_DEBUG_ASSERT_POINTER(timeline->sync_tl); + sync_fence_tracker->flag = mali_sync_flag_create(timeline->sync_tl, point); + if (NULL == sync_fence_tracker->flag) { + MALI_PRINT_ERROR(("Mali Timeline: sync_flag creation failed\n")); + _mali_osk_free(sync_fence_tracker); + return NULL; + } + + /* Create sync fence from sync flag. */ + sync_fence = mali_sync_flag_create_fence(sync_fence_tracker->flag); + if (NULL == sync_fence) { + MALI_PRINT_ERROR(("Mali Timeline: sync_fence creation failed\n")); + mali_sync_flag_put(sync_fence_tracker->flag); + _mali_osk_free(sync_fence_tracker); + return NULL; + } + + /* Setup fence for tracker. */ + _mali_osk_memset(&fence, 0, sizeof(struct mali_timeline_fence)); + fence.sync_fd = -1; + fence.points[timeline->id] = point; + + /* Finally, add the tracker to Timeline system. */ + mali_timeline_tracker_init(&sync_fence_tracker->tracker, MALI_TIMELINE_TRACKER_SYNC, &fence, sync_fence_tracker); + point = mali_timeline_system_add_tracker(timeline->system, &sync_fence_tracker->tracker, MALI_TIMELINE_NONE); + MALI_DEBUG_ASSERT(MALI_TIMELINE_NO_POINT == point); + + return sync_fence; +} + +s32 mali_timeline_sync_fence_create(struct mali_timeline_system *system, struct mali_timeline_fence *fence) +{ + u32 i; + struct sync_fence *sync_fence_acc = NULL; + + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT_POINTER(fence); + + for (i = 0; i < MALI_TIMELINE_MAX; ++i) { + struct mali_timeline *timeline; + struct sync_fence *sync_fence; + + if (MALI_TIMELINE_NO_POINT == fence->points[i]) continue; + + timeline = system->timelines[i]; + MALI_DEBUG_ASSERT_POINTER(timeline); + + sync_fence = mali_timeline_sync_fence_create_and_add_tracker(timeline, fence->points[i]); + if (NULL == sync_fence) goto error; + + if (NULL != sync_fence_acc) { + /* Merge sync fences. */ + sync_fence_acc = mali_sync_fence_merge(sync_fence_acc, sync_fence); + if (NULL == sync_fence_acc) goto error; + } else { + /* This was the first sync fence created. */ + sync_fence_acc = sync_fence; + } + } + + if (-1 != fence->sync_fd) { + struct sync_fence *sync_fence; + + sync_fence = sync_fence_fdget(fence->sync_fd); + if (NULL == sync_fence) goto error; + + if (NULL != sync_fence_acc) { + sync_fence_acc = mali_sync_fence_merge(sync_fence_acc, sync_fence); + if (NULL == sync_fence_acc) goto error; + } else { + sync_fence_acc = sync_fence; + } + } + + if (NULL == sync_fence_acc) { + MALI_DEBUG_ASSERT_POINTER(system->signaled_sync_tl); + + /* There was nothing to wait on, so return an already signaled fence. */ + + sync_fence_acc = mali_sync_timeline_create_signaled_fence(system->signaled_sync_tl); + if (NULL == sync_fence_acc) goto error; + } + + /* Return file descriptor for the accumulated sync fence. */ + return mali_sync_fence_fd_alloc(sync_fence_acc); + +error: + if (NULL != sync_fence_acc) { + sync_fence_put(sync_fence_acc); + } + + return -1; +} + +void mali_timeline_sync_fence_activate(struct mali_timeline_sync_fence_tracker *sync_fence_tracker) +{ + mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; + + MALI_DEBUG_ASSERT_POINTER(sync_fence_tracker); + MALI_DEBUG_ASSERT_POINTER(sync_fence_tracker->flag); + + MALI_DEBUG_PRINT(4, ("Mali Timeline: activation for sync fence tracker\n")); + + /* Signal flag and release reference. */ + mali_sync_flag_signal(sync_fence_tracker->flag, 0); + mali_sync_flag_put(sync_fence_tracker->flag); + + /* Nothing can wait on this tracker, so nothing to schedule after release. */ + schedule_mask = mali_timeline_tracker_release(&sync_fence_tracker->tracker); + MALI_DEBUG_ASSERT(MALI_SCHEDULER_MASK_EMPTY == schedule_mask); + + _mali_osk_free(sync_fence_tracker); +} + +#endif /* defined(CONFIG_SYNC) */ diff --git a/drivers/gpu/arm/utgard/common/mali_timeline_sync_fence.h b/drivers/gpu/arm/utgard/common/mali_timeline_sync_fence.h new file mode 100644 index 00000000000000..29a3822457e9b7 --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_timeline_sync_fence.h @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2013, 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_timeline_sync_fence.h + * + * This file contains code related to creating sync fences from timeline fences. + */ + +#ifndef __MALI_TIMELINE_SYNC_FENCE_H__ +#define __MALI_TIMELINE_SYNC_FENCE_H__ + +#include "mali_timeline.h" + +#if defined(CONFIG_SYNC) + +/** + * Sync fence tracker. + */ +struct mali_timeline_sync_fence_tracker { + struct mali_sync_flag *flag; /**< Sync flag used to connect tracker and sync fence. */ + struct mali_timeline_tracker tracker; /**< Timeline tracker. */ +}; + +/** + * Create a sync fence that will be signaled when @ref fence is signaled. + * + * @param system Timeline system. + * @param fence Fence to create sync fence from. + * @return File descriptor for new sync fence, or -1 on error. + */ +s32 mali_timeline_sync_fence_create(struct mali_timeline_system *system, struct mali_timeline_fence *fence); + +/** + * Used by the Timeline system to activate a sync fence tracker. + * + * @param sync_fence_tracker Sync fence tracker. + * + */ +void mali_timeline_sync_fence_activate(struct mali_timeline_sync_fence_tracker *sync_fence_tracker); + +#endif /* defined(CONFIG_SYNC) */ + +#endif /* __MALI_TIMELINE_SYNC_FENCE_H__ */ diff --git a/drivers/gpu/arm/utgard/common/mali_ukk.h b/drivers/gpu/arm/utgard/common/mali_ukk.h new file mode 100644 index 00000000000000..597685a53f3b60 --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_ukk.h @@ -0,0 +1,551 @@ +/* + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_ukk.h + * Defines the kernel-side interface of the user-kernel interface + */ + +#ifndef __MALI_UKK_H__ +#define __MALI_UKK_H__ + +#include "mali_osk.h" +#include "mali_uk_types.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @addtogroup uddapi Unified Device Driver (UDD) APIs + * + * @{ + */ + +/** + * @addtogroup u_k_api UDD User/Kernel Interface (U/K) APIs + * + * - The _mali_uk functions are an abstraction of the interface to the device + * driver. On certain OSs, this would be implemented via the IOCTL interface. + * On other OSs, it could be via extension of some Device Driver Class, or + * direct function call for Bare metal/RTOSs. + * - It is important to note that: + * - The Device Driver has implemented the _mali_ukk set of functions + * - The Base Driver calls the corresponding set of _mali_uku functions. + * - What requires porting is solely the calling mechanism from User-side to + * Kernel-side, and propagating back the results. + * - Each U/K function is associated with a (group, number) pair from + * \ref _mali_uk_functions to make it possible for a common function in the + * Base Driver and Device Driver to route User/Kernel calls from/to the + * correct _mali_uk function. For example, in an IOCTL system, the IOCTL number + * would be formed based on the group and number assigned to the _mali_uk + * function, as listed in \ref _mali_uk_functions. On the user-side, each + * _mali_uku function would just make an IOCTL with the IOCTL-code being an + * encoded form of the (group, number) pair. On the kernel-side, the Device + * Driver's IOCTL handler decodes the IOCTL-code back into a (group, number) + * pair, and uses this to determine which corresponding _mali_ukk should be + * called. + * - Refer to \ref _mali_uk_functions for more information about this + * (group, number) pairing. + * - In a system where there is no distinction between user and kernel-side, + * the U/K interface may be implemented as:@code + * MALI_STATIC_INLINE _mali_osk_errcode_t _mali_uku_examplefunction( _mali_uk_examplefunction_s *args ) + * { + * return mali_ukk_examplefunction( args ); + * } + * @endcode + * - Therefore, all U/K calls behave \em as \em though they were direct + * function calls (but the \b implementation \em need \em not be a direct + * function calls) + * + * @note Naming the _mali_uk functions the same on both User and Kernel sides + * on non-RTOS systems causes debugging issues when setting breakpoints. In + * this case, it is not clear which function the breakpoint is put on. + * Therefore the _mali_uk functions in user space are prefixed with \c _mali_uku + * and in kernel space with \c _mali_ukk. The naming for the argument + * structures is unaffected. + * + * - The _mali_uk functions are synchronous. + * - Arguments to the _mali_uk functions are passed in a structure. The only + * parameter passed to the _mali_uk functions is a pointer to this structure. + * This first member of this structure, ctx, is a pointer to a context returned + * by _mali_uku_open(). For example:@code + * typedef struct + * { + * void *ctx; + * u32 number_of_cores; + * } _mali_uk_get_gp_number_of_cores_s; + * @endcode + * + * - Each _mali_uk function has its own argument structure named after the + * function. The argument is distinguished by the _s suffix. + * - The argument types are defined by the base driver and user-kernel + * interface. + * - All _mali_uk functions return a standard \ref _mali_osk_errcode_t. + * - Only arguments of type input or input/output need be initialized before + * calling a _mali_uk function. + * - Arguments of type output and input/output are only valid when the + * _mali_uk function returns \ref _MALI_OSK_ERR_OK. + * - The \c ctx member is always invalid after it has been used by a + * _mali_uk function, except for the context management functions + * + * + * \b Interface \b restrictions + * + * The requirements of the interface mean that an implementation of the + * User-kernel interface may do no 'real' work. For example, the following are + * illegal in the User-kernel implementation: + * - Calling functions necessary for operation on all systems, which would + * not otherwise get called on RTOS systems. + * - For example, a U/K interface that calls multiple _mali_ukk functions + * during one particular U/K call. This could not be achieved by the same code + * which uses direct function calls for the U/K interface. + * - Writing in values to the args members, when otherwise these members would + * not hold a useful value for a direct function call U/K interface. + * - For example, U/K interface implementation that take NULL members in + * their arguments structure from the user side, but those members are + * replaced with non-NULL values in the kernel-side of the U/K interface + * implementation. A scratch area for writing data is one such example. In this + * case, a direct function call U/K interface would segfault, because no code + * would be present to replace the NULL pointer with a meaningful pointer. + * - Note that we discourage the case where the U/K implementation changes + * a NULL argument member to non-NULL, and then the Device Driver code (outside + * of the U/K layer) re-checks this member for NULL, and corrects it when + * necessary. Whilst such code works even on direct function call U/K + * intefaces, it reduces the testing coverage of the Device Driver code. This + * is because we have no way of testing the NULL == value path on an OS + * implementation. + * + * A number of allowable examples exist where U/K interfaces do 'real' work: + * - The 'pointer switching' technique for \ref _mali_ukk_get_system_info + * - In this case, without the pointer switching on direct function call + * U/K interface, the Device Driver code still sees the same thing: a pointer + * to which it can write memory. This is because such a system has no + * distinction between a user and kernel pointer. + * - Writing an OS-specific value into the ukk_private member for + * _mali_ukk_mem_mmap(). + * - In this case, this value is passed around by Device Driver code, but + * its actual value is never checked. Device Driver code simply passes it from + * the U/K layer to the OSK layer, where it can be acted upon. In this case, + * \em some OS implementations of the U/K (_mali_ukk_mem_mmap()) and OSK + * (_mali_osk_mem_mapregion_init()) functions will collaborate on the + * meaning of ukk_private member. On other OSs, it may be unused by both + * U/K and OSK layers + * - Therefore, on error inside the U/K interface implementation itself, + * it will be as though the _mali_ukk function itself had failed, and cleaned + * up after itself. + * - Compare this to a direct function call U/K implementation, where all + * error cleanup is handled by the _mali_ukk function itself. The direct + * function call U/K interface implementation is automatically atomic. + * + * The last example highlights a consequence of all U/K interface + * implementations: they must be atomic with respect to the Device Driver code. + * And therefore, should Device Driver code succeed but the U/K implementation + * fail afterwards (but before return to user-space), then the U/K + * implementation must cause appropriate cleanup actions to preserve the + * atomicity of the interface. + * + * @{ + */ + + +/** @defgroup _mali_uk_context U/K Context management + * + * These functions allow for initialisation of the user-kernel interface once per process. + * + * Generally the context will store the OS specific object to communicate with the kernel device driver and further + * state information required by the specific implementation. The context is shareable among all threads in the caller process. + * + * On IOCTL systems, this is likely to be a file descriptor as a result of opening the kernel device driver. + * + * On a bare-metal/RTOS system with no distinction between kernel and + * user-space, the U/K interface simply calls the _mali_ukk variant of the + * function by direct function call. In this case, the context returned is the + * mali_session_data from _mali_ukk_open(). + * + * The kernel side implementations of the U/K interface expect the first member of the argument structure to + * be the context created by _mali_uku_open(). On some OS implementations, the meaning of this context + * will be different between user-side and kernel-side. In which case, the kernel-side will need to replace this context + * with the kernel-side equivalent, because user-side will not have access to kernel-side data. The context parameter + * in the argument structure therefore has to be of type input/output. + * + * It should be noted that the caller cannot reuse the \c ctx member of U/K + * argument structure after a U/K call, because it may be overwritten. Instead, + * the context handle must always be stored elsewhere, and copied into + * the appropriate U/K argument structure for each user-side call to + * the U/K interface. This is not usually a problem, since U/K argument + * structures are usually placed on the stack. + * + * @{ */ + +/** @brief Begin a new Mali Device Driver session + * + * This is used to obtain a per-process context handle for all future U/K calls. + * + * @param context pointer to storage to return a (void*)context handle. + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_open(void **context); + +/** @brief End a Mali Device Driver session + * + * This should be called when the process no longer requires use of the Mali Device Driver. + * + * The context handle must not be used after it has been closed. + * + * @param context pointer to a stored (void*)context handle. + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_close(void **context); + +/** @} */ /* end group _mali_uk_context */ + + +/** @addtogroup _mali_uk_core U/K Core + * + * The core functions provide the following functionality: + * - verify that the user and kernel API are compatible + * - retrieve information about the cores and memory banks in the system + * - wait for the result of jobs started on a core + * + * @{ */ + +/** @brief Waits for a job notification. + * + * Sleeps until notified or a timeout occurs. Returns information about the notification. + * + * @param args see _mali_uk_wait_for_notification_s in "mali_utgard_uk_types.h" + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_wait_for_notification(_mali_uk_wait_for_notification_s *args); + +/** @brief Post a notification to the notification queue of this application. + * + * @param args see _mali_uk_post_notification_s in "mali_utgard_uk_types.h" + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_post_notification(_mali_uk_post_notification_s *args); + +/** @brief Verifies if the user and kernel side of this API are compatible. + * + * This function is obsolete, but kept to allow old, incompatible user space + * clients to robustly detect the incompatibility. + * + * @param args see _mali_uk_get_api_version_s in "mali_utgard_uk_types.h" + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_get_api_version(_mali_uk_get_api_version_s *args); + +/** @brief Verifies if the user and kernel side of this API are compatible. + * + * @param args see _mali_uk_get_api_version_v2_s in "mali_utgard_uk_types.h" + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_get_api_version_v2(_mali_uk_get_api_version_v2_s *args); + +/** @brief Get the user space settings applicable for calling process. + * + * @param args see _mali_uk_get_user_settings_s in "mali_utgard_uk_types.h" + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_get_user_settings(_mali_uk_get_user_settings_s *args); + +/** @brief Get a user space setting applicable for calling process. + * + * @param args see _mali_uk_get_user_setting_s in "mali_utgard_uk_types.h" + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_get_user_setting(_mali_uk_get_user_setting_s *args); + +/* @brief Grant or deny high priority scheduling for this session. + * + * @param args see _mali_uk_request_high_priority_s in "mali_utgard_uk_types.h" + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_request_high_priority(_mali_uk_request_high_priority_s *args); + +/** @brief Make process sleep if the pending big job in kernel >= MALI_MAX_PENDING_BIG_JOB + * + */ +_mali_osk_errcode_t _mali_ukk_pending_submit(_mali_uk_pending_submit_s *args); + +/** @} */ /* end group _mali_uk_core */ + + +/** @addtogroup _mali_uk_memory U/K Memory + * + * The memory functions provide functionality with and without a Mali-MMU present. + * + * For Mali-MMU based systems, the following functionality is provided: + * - Initialize and terminate MALI virtual address space + * - Allocate/deallocate physical memory to a MALI virtual address range and map into/unmap from the + * current process address space + * - Map/unmap external physical memory into the MALI virtual address range + * + * For Mali-nonMMU based systems: + * - Allocate/deallocate MALI memory + * + * @{ */ + +/** @brief Map Mali Memory into the current user process + * + * Maps Mali memory into the current user process in a generic way. + * + * This function is to be used for Mali-MMU mode. The function is available in both Mali-MMU and Mali-nonMMU modes, + * but should not be called by a user process in Mali-nonMMU mode. + * + * The implementation and operation of _mali_ukk_mem_mmap() is dependant on whether the driver is built for Mali-MMU + * or Mali-nonMMU: + * - In the nonMMU case, _mali_ukk_mem_mmap() requires a physical address to be specified. For this reason, an OS U/K + * implementation should not allow this to be called from user-space. In any case, nonMMU implementations are + * inherently insecure, and so the overall impact is minimal. Mali-MMU mode should be used if security is desired. + * - In the MMU case, _mali_ukk_mem_mmap() the _mali_uk_mem_mmap_s::phys_addr + * member is used for the \em Mali-virtual address desired for the mapping. The + * implementation of _mali_ukk_mem_mmap() will allocate both the CPU-virtual + * and CPU-physical addresses, and can cope with mapping a contiguous virtual + * address range to a sequence of non-contiguous physical pages. In this case, + * the CPU-physical addresses are not communicated back to the user-side, as + * they are unnecsessary; the \em Mali-virtual address range must be used for + * programming Mali structures. + * + * In the second (MMU) case, _mali_ukk_mem_mmap() handles management of + * CPU-virtual and CPU-physical ranges, but the \em caller must manage the + * \em Mali-virtual address range from the user-side. + * + * @note Mali-virtual address ranges are entirely separate between processes. + * It is not possible for a process to accidentally corrupt another process' + * \em Mali-virtual address space. + * + * @param args see _mali_uk_mem_mmap_s in "mali_utgard_uk_types.h" + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_mem_mmap(_mali_uk_mem_mmap_s *args); + +/** @brief Unmap Mali Memory from the current user process + * + * Unmaps Mali memory from the current user process in a generic way. This only operates on Mali memory supplied + * from _mali_ukk_mem_mmap(). + * + * @param args see _mali_uk_mem_munmap_s in "mali_utgard_uk_types.h" + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_mem_munmap(_mali_uk_mem_munmap_s *args); + +/** @brief Determine the buffer size necessary for an MMU page table dump. + * @param args see _mali_uk_query_mmu_page_table_dump_size_s in mali_utgard_uk_types.h + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_query_mmu_page_table_dump_size(_mali_uk_query_mmu_page_table_dump_size_s *args); +/** @brief Dump MMU Page tables. + * @param args see _mali_uk_dump_mmu_page_table_s in mali_utgard_uk_types.h + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_dump_mmu_page_table(_mali_uk_dump_mmu_page_table_s *args); + +/** @brief Write user data to specified Mali memory without causing segfaults. + * @param args see _mali_uk_mem_write_safe_s in mali_utgard_uk_types.h + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_mem_write_safe(_mali_uk_mem_write_safe_s *args); + +/** @} */ /* end group _mali_uk_memory */ + + +/** @addtogroup _mali_uk_pp U/K Fragment Processor + * + * The Fragment Processor (aka PP (Pixel Processor)) functions provide the following functionality: + * - retrieving version of the fragment processors + * - determine number of fragment processors + * - starting a job on a fragment processor + * + * @{ */ + +/** @brief Issue a request to start a new job on a Fragment Processor. + * + * If the request fails args->status is set to _MALI_UK_START_JOB_NOT_STARTED_DO_REQUEUE and you can + * try to start the job again. + * + * An existing job could be returned for requeueing if the new job has a higher priority than a previously started job + * which the hardware hasn't actually started processing yet. In this case the new job will be started instead and the + * existing one returned, otherwise the new job is started and the status field args->status is set to + * _MALI_UK_START_JOB_STARTED. + * + * Job completion can be awaited with _mali_ukk_wait_for_notification(). + * + * @param ctx user-kernel context (mali_session) + * @param uargs see _mali_uk_pp_start_job_s in "mali_utgard_uk_types.h". Use _mali_osk_copy_from_user to retrieve data! + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_pp_start_job(void *ctx, _mali_uk_pp_start_job_s *uargs); + +/** + * @brief Issue a request to start new jobs on both Vertex Processor and Fragment Processor. + * + * @note Will call into @ref _mali_ukk_pp_start_job and @ref _mali_ukk_gp_start_job. + * + * @param ctx user-kernel context (mali_session) + * @param uargs see _mali_uk_pp_and_gp_start_job_s in "mali_utgard_uk_types.h". Use _mali_osk_copy_from_user to retrieve data! + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_pp_and_gp_start_job(void *ctx, _mali_uk_pp_and_gp_start_job_s *uargs); + +/** @brief Returns the number of Fragment Processors in the system + * + * @param args see _mali_uk_get_pp_number_of_cores_s in "mali_utgard_uk_types.h" + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_get_pp_number_of_cores(_mali_uk_get_pp_number_of_cores_s *args); + +/** @brief Returns the version that all Fragment Processor cores are compatible with. + * + * This function may only be called when _mali_ukk_get_pp_number_of_cores() indicated at least one Fragment + * Processor core is available. + * + * @param args see _mali_uk_get_pp_core_version_s in "mali_utgard_uk_types.h" + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_get_pp_core_version(_mali_uk_get_pp_core_version_s *args); + +/** @brief Disable Write-back unit(s) on specified job + * + * @param args see _mali_uk_get_pp_core_version_s in "mali_utgard_uk_types.h" + */ +void _mali_ukk_pp_job_disable_wb(_mali_uk_pp_disable_wb_s *args); + + +/** @} */ /* end group _mali_uk_pp */ + + +/** @addtogroup _mali_uk_gp U/K Vertex Processor + * + * The Vertex Processor (aka GP (Geometry Processor)) functions provide the following functionality: + * - retrieving version of the Vertex Processors + * - determine number of Vertex Processors available + * - starting a job on a Vertex Processor + * + * @{ */ + +/** @brief Issue a request to start a new job on a Vertex Processor. + * + * If the request fails args->status is set to _MALI_UK_START_JOB_NOT_STARTED_DO_REQUEUE and you can + * try to start the job again. + * + * An existing job could be returned for requeueing if the new job has a higher priority than a previously started job + * which the hardware hasn't actually started processing yet. In this case the new job will be started and the + * existing one returned, otherwise the new job is started and the status field args->status is set to + * _MALI_UK_START_JOB_STARTED. + * + * Job completion can be awaited with _mali_ukk_wait_for_notification(). + * + * @param ctx user-kernel context (mali_session) + * @param uargs see _mali_uk_gp_start_job_s in "mali_utgard_uk_types.h". Use _mali_osk_copy_from_user to retrieve data! + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_gp_start_job(void *ctx, _mali_uk_gp_start_job_s *uargs); + +/** @brief Returns the number of Vertex Processors in the system. + * + * @param args see _mali_uk_get_gp_number_of_cores_s in "mali_utgard_uk_types.h" + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_get_gp_number_of_cores(_mali_uk_get_gp_number_of_cores_s *args); + +/** @brief Returns the version that all Vertex Processor cores are compatible with. + * + * This function may only be called when _mali_uk_get_gp_number_of_cores() indicated at least one Vertex + * Processor core is available. + * + * @param args see _mali_uk_get_gp_core_version_s in "mali_utgard_uk_types.h" + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_get_gp_core_version(_mali_uk_get_gp_core_version_s *args); + +/** @brief Resume or abort suspended Vertex Processor jobs. + * + * After receiving notification that a Vertex Processor job was suspended from + * _mali_ukk_wait_for_notification() you can use this function to resume or abort the job. + * + * @param args see _mali_uk_gp_suspend_response_s in "mali_utgard_uk_types.h" + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_gp_suspend_response(_mali_uk_gp_suspend_response_s *args); + +/** @} */ /* end group _mali_uk_gp */ + +#if defined(CONFIG_MALI400_PROFILING) +/** @addtogroup _mali_uk_profiling U/K Timeline profiling module + * @{ */ + +/** @brief Add event to profiling buffer. + * + * @param args see _mali_uk_profiling_add_event_s in "mali_utgard_uk_types.h" + */ +_mali_osk_errcode_t _mali_ukk_profiling_add_event(_mali_uk_profiling_add_event_s *args); + +/** @brief Get profiling stream fd. + * + * @param args see _mali_uk_profiling_stream_fd_get_s in "mali_utgard_uk_types.h" + */ +_mali_osk_errcode_t _mali_ukk_profiling_stream_fd_get(_mali_uk_profiling_stream_fd_get_s *args); + +/** @brief Profiling control set. + * + * @param args see _mali_uk_profiling_control_set_s in "mali_utgard_uk_types.h" + */ +_mali_osk_errcode_t _mali_ukk_profiling_control_set(_mali_uk_profiling_control_set_s *args); + +/** @} */ /* end group _mali_uk_profiling */ +#endif + +/** @addtogroup _mali_uk_vsync U/K VSYNC reporting module + * @{ */ + +/** @brief Report events related to vsync. + * + * @note Events should be reported when starting to wait for vsync and when the + * waiting is finished. This information can then be used in kernel space to + * complement the GPU utilization metric. + * + * @param args see _mali_uk_vsync_event_report_s in "mali_utgard_uk_types.h" + */ +_mali_osk_errcode_t _mali_ukk_vsync_event_report(_mali_uk_vsync_event_report_s *args); + +/** @} */ /* end group _mali_uk_vsync */ + +/** @addtogroup _mali_sw_counters_report U/K Software counter reporting + * @{ */ + +/** @brief Report software counters. + * + * @param args see _mali_uk_sw_counters_report_s in "mali_uk_types.h" + */ +_mali_osk_errcode_t _mali_ukk_sw_counters_report(_mali_uk_sw_counters_report_s *args); + +/** @} */ /* end group _mali_sw_counters_report */ + +/** @} */ /* end group u_k_api */ + +/** @} */ /* end group uddapi */ + +u32 _mali_ukk_report_memory_usage(void); + +u32 _mali_ukk_report_total_memory_size(void); + +u32 _mali_ukk_utilization_gp_pp(void); + +u32 _mali_ukk_utilization_gp(void); + +u32 _mali_ukk_utilization_pp(void); + +#ifdef __cplusplus +} +#endif + +#endif /* __MALI_UKK_H__ */ diff --git a/drivers/gpu/arm/utgard/common/mali_user_settings_db.c b/drivers/gpu/arm/utgard/common/mali_user_settings_db.c new file mode 100644 index 00000000000000..54e1580fad1a93 --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_user_settings_db.c @@ -0,0 +1,147 @@ +/** + * Copyright (C) 2012-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_kernel_common.h" +#include "mali_osk.h" +#include "mali_ukk.h" +#include "mali_uk_types.h" +#include "mali_user_settings_db.h" +#include "mali_session.h" + +static u32 mali_user_settings[_MALI_UK_USER_SETTING_MAX]; +const char *_mali_uk_user_setting_descriptions[] = _MALI_UK_USER_SETTING_DESCRIPTIONS; + +static void mali_user_settings_notify(_mali_uk_user_setting_t setting, u32 value) +{ + mali_bool done = MALI_FALSE; + + /* + * This function gets a bit complicated because we can't hold the session lock while + * allocating notification objects. + */ + + while (!done) { + u32 i; + u32 num_sessions_alloc; + u32 num_sessions_with_lock; + u32 used_notification_objects = 0; + _mali_osk_notification_t **notobjs; + + /* Pre allocate the number of notifications objects we need right now (might change after lock has been taken) */ + num_sessions_alloc = mali_session_get_count(); + if (0 == num_sessions_alloc) { + /* No sessions to report to */ + return; + } + + notobjs = (_mali_osk_notification_t **)_mali_osk_malloc(sizeof(_mali_osk_notification_t *) * num_sessions_alloc); + if (NULL == notobjs) { + MALI_PRINT_ERROR(("Failed to notify user space session about num PP core change (alloc failure)\n")); + return; + } + + for (i = 0; i < num_sessions_alloc; i++) { + notobjs[i] = _mali_osk_notification_create(_MALI_NOTIFICATION_SETTINGS_CHANGED, + sizeof(_mali_uk_settings_changed_s)); + if (NULL != notobjs[i]) { + _mali_uk_settings_changed_s *data; + data = notobjs[i]->result_buffer; + + data->setting = setting; + data->value = value; + } else { + MALI_PRINT_ERROR(("Failed to notify user space session about setting change (alloc failure %u)\n", i)); + } + } + + mali_session_lock(); + + /* number of sessions will not change while we hold the lock */ + num_sessions_with_lock = mali_session_get_count(); + + if (num_sessions_alloc >= num_sessions_with_lock) { + /* We have allocated enough notification objects for all the sessions atm */ + struct mali_session_data *session, *tmp; + MALI_SESSION_FOREACH(session, tmp, link) { + MALI_DEBUG_ASSERT(used_notification_objects < num_sessions_alloc); + if (NULL != notobjs[used_notification_objects]) { + mali_session_send_notification(session, notobjs[used_notification_objects]); + notobjs[used_notification_objects] = NULL; /* Don't track this notification object any more */ + } + used_notification_objects++; + } + done = MALI_TRUE; + } + + mali_session_unlock(); + + /* Delete any remaining/unused notification objects */ + for (; used_notification_objects < num_sessions_alloc; used_notification_objects++) { + if (NULL != notobjs[used_notification_objects]) { + _mali_osk_notification_delete(notobjs[used_notification_objects]); + } + } + + _mali_osk_free(notobjs); + } +} + +void mali_set_user_setting(_mali_uk_user_setting_t setting, u32 value) +{ + mali_bool notify = MALI_FALSE; + + if (setting >= _MALI_UK_USER_SETTING_MAX) { + MALI_DEBUG_PRINT_ERROR(("Invalid user setting %ud\n")); + return; + } + + if (mali_user_settings[setting] != value) { + notify = MALI_TRUE; + } + + mali_user_settings[setting] = value; + + if (notify) { + mali_user_settings_notify(setting, value); + } +} + +u32 mali_get_user_setting(_mali_uk_user_setting_t setting) +{ + if (setting >= _MALI_UK_USER_SETTING_MAX) { + return 0; + } + + return mali_user_settings[setting]; +} + +_mali_osk_errcode_t _mali_ukk_get_user_setting(_mali_uk_get_user_setting_s *args) +{ + _mali_uk_user_setting_t setting; + MALI_DEBUG_ASSERT_POINTER(args); + + setting = args->setting; + + if (_MALI_UK_USER_SETTING_MAX > setting) { + args->value = mali_user_settings[setting]; + return _MALI_OSK_ERR_OK; + } else { + return _MALI_OSK_ERR_INVALID_ARGS; + } +} + +_mali_osk_errcode_t _mali_ukk_get_user_settings(_mali_uk_get_user_settings_s *args) +{ + MALI_DEBUG_ASSERT_POINTER(args); + + _mali_osk_memcpy(args->settings, mali_user_settings, sizeof(mali_user_settings)); + + return _MALI_OSK_ERR_OK; +} diff --git a/drivers/gpu/arm/utgard/common/mali_user_settings_db.h b/drivers/gpu/arm/utgard/common/mali_user_settings_db.h new file mode 100644 index 00000000000000..0732c3e56e2ad0 --- /dev/null +++ b/drivers/gpu/arm/utgard/common/mali_user_settings_db.h @@ -0,0 +1,39 @@ +/** + * Copyright (C) 2012-2013, 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_USER_SETTINGS_DB_H__ +#define __MALI_USER_SETTINGS_DB_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "mali_uk_types.h" + +/** @brief Set Mali user setting in DB + * + * Update the DB with a new value for \a setting. If the value is different from theprevious set value running sessions will be notified of the change. + * + * @param setting the setting to be changed + * @param value the new value to set + */ +void mali_set_user_setting(_mali_uk_user_setting_t setting, u32 value); + +/** @brief Get current Mali user setting value from DB + * + * @param setting the setting to extract + * @return the value of the selected setting + */ +u32 mali_get_user_setting(_mali_uk_user_setting_t setting); + +#ifdef __cplusplus +} +#endif +#endif /* __MALI_KERNEL_USER_SETTING__ */ diff --git a/drivers/gpu/arm/utgard/include/linux/mali/mali_utgard.h b/drivers/gpu/arm/utgard/include/linux/mali/mali_utgard.h new file mode 100644 index 00000000000000..4ea02fe65cb57e --- /dev/null +++ b/drivers/gpu/arm/utgard/include/linux/mali/mali_utgard.h @@ -0,0 +1,507 @@ +/* + * Copyright (C) 2012-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_utgard.h + * Defines types and interface exposed by the Mali Utgard device driver + */ + +#ifndef __MALI_UTGARD_H__ +#define __MALI_UTGARD_H__ + +#include "mali_osk_types.h" + +#define MALI_GPU_NAME_UTGARD "mali-utgard" + + +#define MALI_OFFSET_GP 0x00000 +#define MALI_OFFSET_GP_MMU 0x03000 + +#define MALI_OFFSET_PP0 0x08000 +#define MALI_OFFSET_PP0_MMU 0x04000 +#define MALI_OFFSET_PP1 0x0A000 +#define MALI_OFFSET_PP1_MMU 0x05000 +#define MALI_OFFSET_PP2 0x0C000 +#define MALI_OFFSET_PP2_MMU 0x06000 +#define MALI_OFFSET_PP3 0x0E000 +#define MALI_OFFSET_PP3_MMU 0x07000 + +#define MALI_OFFSET_PP4 0x28000 +#define MALI_OFFSET_PP4_MMU 0x1C000 +#define MALI_OFFSET_PP5 0x2A000 +#define MALI_OFFSET_PP5_MMU 0x1D000 +#define MALI_OFFSET_PP6 0x2C000 +#define MALI_OFFSET_PP6_MMU 0x1E000 +#define MALI_OFFSET_PP7 0x2E000 +#define MALI_OFFSET_PP7_MMU 0x1F000 + +#define MALI_OFFSET_L2_RESOURCE0 0x01000 +#define MALI_OFFSET_L2_RESOURCE1 0x10000 +#define MALI_OFFSET_L2_RESOURCE2 0x11000 + +#define MALI400_OFFSET_L2_CACHE0 MALI_OFFSET_L2_RESOURCE0 +#define MALI450_OFFSET_L2_CACHE0 MALI_OFFSET_L2_RESOURCE1 +#define MALI450_OFFSET_L2_CACHE1 MALI_OFFSET_L2_RESOURCE0 +#define MALI450_OFFSET_L2_CACHE2 MALI_OFFSET_L2_RESOURCE2 +#define MALI470_OFFSET_L2_CACHE1 MALI_OFFSET_L2_RESOURCE0 + +#define MALI_OFFSET_BCAST 0x13000 +#define MALI_OFFSET_DLBU 0x14000 + +#define MALI_OFFSET_PP_BCAST 0x16000 +#define MALI_OFFSET_PP_BCAST_MMU 0x15000 + +#define MALI_OFFSET_PMU 0x02000 +#define MALI_OFFSET_DMA 0x12000 + +/* Mali-300 */ + +#define MALI_GPU_RESOURCES_MALI300(base_addr, gp_irq, gp_mmu_irq, pp_irq, pp_mmu_irq) \ + MALI_GPU_RESOURCES_MALI400_MP1(base_addr, gp_irq, gp_mmu_irq, pp_irq, pp_mmu_irq) + +#define MALI_GPU_RESOURCES_MALI300_PMU(base_addr, gp_irq, gp_mmu_irq, pp_irq, pp_mmu_irq) \ + MALI_GPU_RESOURCES_MALI400_MP1_PMU(base_addr, gp_irq, gp_mmu_irq, pp_irq, pp_mmu_irq) + +/* Mali-400 */ + +#define MALI_GPU_RESOURCES_MALI400_MP1(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI400_OFFSET_L2_CACHE0) \ + MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) + +#define MALI_GPU_RESOURCES_MALI400_MP1_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq) \ + MALI_GPU_RESOURCES_MALI400_MP1(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq) \ + MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) + +#define MALI_GPU_RESOURCES_MALI400_MP2(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI400_OFFSET_L2_CACHE0) \ + MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) + +#define MALI_GPU_RESOURCES_MALI400_MP2_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq) \ + MALI_GPU_RESOURCES_MALI400_MP2(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq) \ + MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) + +#define MALI_GPU_RESOURCES_MALI400_MP3(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI400_OFFSET_L2_CACHE0) \ + MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(2, base_addr + MALI_OFFSET_PP2, pp2_irq, base_addr + MALI_OFFSET_PP2_MMU, pp2_mmu_irq) + +#define MALI_GPU_RESOURCES_MALI400_MP3_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq) \ + MALI_GPU_RESOURCES_MALI400_MP3(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq) \ + MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) + +#define MALI_GPU_RESOURCES_MALI400_MP4(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI400_OFFSET_L2_CACHE0) \ + MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(2, base_addr + MALI_OFFSET_PP2, pp2_irq, base_addr + MALI_OFFSET_PP2_MMU, pp2_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(3, base_addr + MALI_OFFSET_PP3, pp3_irq, base_addr + MALI_OFFSET_PP3_MMU, pp3_mmu_irq) + +#define MALI_GPU_RESOURCES_MALI400_MP4_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq) \ + MALI_GPU_RESOURCES_MALI400_MP4(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq) \ + MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ + + /* Mali-450 */ +#define MALI_GPU_RESOURCES_MALI450_MP2(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE0) \ + MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE1) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ + MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ + MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ + MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) \ + MALI_GPU_RESOURCE_DMA(base_addr + MALI_OFFSET_DMA) + +#define MALI_GPU_RESOURCES_MALI450_MP2_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCES_MALI450_MP2(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ + +#define MALI_GPU_RESOURCES_MALI450_MP3(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE0) \ + MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE1) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(2, base_addr + MALI_OFFSET_PP2, pp2_irq, base_addr + MALI_OFFSET_PP2_MMU, pp2_mmu_irq) \ + MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ + MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ + MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) + +#define MALI_GPU_RESOURCES_MALI450_MP3_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCES_MALI450_MP3(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ + +#define MALI_GPU_RESOURCES_MALI450_MP4(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE0) \ + MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE1) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(2, base_addr + MALI_OFFSET_PP2, pp2_irq, base_addr + MALI_OFFSET_PP2_MMU, pp2_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(3, base_addr + MALI_OFFSET_PP3, pp3_irq, base_addr + MALI_OFFSET_PP3_MMU, pp3_mmu_irq) \ + MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ + MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ + MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) \ + MALI_GPU_RESOURCE_DMA(base_addr + MALI_OFFSET_DMA) + +#define MALI_GPU_RESOURCES_MALI450_MP4_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCES_MALI450_MP4(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ + +#define MALI_GPU_RESOURCES_MALI450_MP6(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp4_irq, pp4_mmu_irq, pp5_irq, pp5_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE0) \ + MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE1) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(2, base_addr + MALI_OFFSET_PP2, pp2_irq, base_addr + MALI_OFFSET_PP2_MMU, pp2_mmu_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE2) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(3, base_addr + MALI_OFFSET_PP4, pp3_irq, base_addr + MALI_OFFSET_PP4_MMU, pp3_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(4, base_addr + MALI_OFFSET_PP5, pp4_irq, base_addr + MALI_OFFSET_PP5_MMU, pp4_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(5, base_addr + MALI_OFFSET_PP6, pp5_irq, base_addr + MALI_OFFSET_PP6_MMU, pp5_mmu_irq) \ + MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ + MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ + MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) \ + MALI_GPU_RESOURCE_DMA(base_addr + MALI_OFFSET_DMA) + +#define MALI_GPU_RESOURCES_MALI450_MP6_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp4_irq, pp4_mmu_irq, pp5_irq, pp5_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCES_MALI450_MP6(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp4_irq, pp4_mmu_irq, pp5_irq, pp5_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ + +#define MALI_GPU_RESOURCES_MALI450_MP8(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp4_irq, pp4_mmu_irq, pp5_irq, pp5_mmu_irq, pp6_irq, pp6_mmu_irq, pp7_irq, pp7_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE0) \ + MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE1) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(2, base_addr + MALI_OFFSET_PP2, pp2_irq, base_addr + MALI_OFFSET_PP2_MMU, pp2_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(3, base_addr + MALI_OFFSET_PP3, pp3_irq, base_addr + MALI_OFFSET_PP3_MMU, pp3_mmu_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE2) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(4, base_addr + MALI_OFFSET_PP4, pp4_irq, base_addr + MALI_OFFSET_PP4_MMU, pp4_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(5, base_addr + MALI_OFFSET_PP5, pp5_irq, base_addr + MALI_OFFSET_PP5_MMU, pp5_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(6, base_addr + MALI_OFFSET_PP6, pp6_irq, base_addr + MALI_OFFSET_PP6_MMU, pp6_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(7, base_addr + MALI_OFFSET_PP7, pp7_irq, base_addr + MALI_OFFSET_PP7_MMU, pp7_mmu_irq) \ + MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ + MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ + MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) \ + MALI_GPU_RESOURCE_DMA(base_addr + MALI_OFFSET_DMA) + +#define MALI_GPU_RESOURCES_MALI450_MP8_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp4_irq, pp4_mmu_irq, pp5_irq, pp5_mmu_irq, pp6_irq, pp6_mmu_irq, pp7_irq, pp7_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCES_MALI450_MP8(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp4_irq, pp4_mmu_irq, pp5_irq, pp5_mmu_irq, pp6_irq, pp6_mmu_irq, pp7_irq, pp7_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ + + /* Mali - 470 */ +#define MALI_GPU_RESOURCES_MALI470_MP1(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI470_OFFSET_L2_CACHE1) \ + MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ + MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ + MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ + MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) + +#define MALI_GPU_RESOURCES_MALI470_MP1_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCES_MALI470_MP1(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ + +#define MALI_GPU_RESOURCES_MALI470_MP2(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI470_OFFSET_L2_CACHE1) \ + MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ + MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ + MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ + MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) + +#define MALI_GPU_RESOURCES_MALI470_MP2_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCES_MALI470_MP2(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ + +#define MALI_GPU_RESOURCES_MALI470_MP3(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI470_OFFSET_L2_CACHE1) \ + MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(2, base_addr + MALI_OFFSET_PP2, pp2_irq, base_addr + MALI_OFFSET_PP2_MMU, pp2_mmu_irq) \ + MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ + MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ + MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) + +#define MALI_GPU_RESOURCES_MALI470_MP3_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCES_MALI470_MP3(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ + +#define MALI_GPU_RESOURCES_MALI470_MP4(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI470_OFFSET_L2_CACHE1) \ + MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(2, base_addr + MALI_OFFSET_PP2, pp2_irq, base_addr + MALI_OFFSET_PP2_MMU, pp2_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(3, base_addr + MALI_OFFSET_PP3, pp3_irq, base_addr + MALI_OFFSET_PP3_MMU, pp3_mmu_irq) \ + MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ + MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ + MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) + +#define MALI_GPU_RESOURCES_MALI470_MP4_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCES_MALI470_MP4(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ + +#define MALI_GPU_RESOURCE_L2(addr) \ + { \ + .name = "Mali_L2", \ + .flags = IORESOURCE_MEM, \ + .start = addr, \ + .end = addr + 0x200, \ + }, + +#define MALI_GPU_RESOURCE_GP(gp_addr, gp_irq) \ + { \ + .name = "Mali_GP", \ + .flags = IORESOURCE_MEM, \ + .start = gp_addr, \ + .end = gp_addr + 0x100, \ + }, \ + { \ + .name = "Mali_GP_IRQ", \ + .flags = IORESOURCE_IRQ, \ + .start = gp_irq, \ + .end = gp_irq, \ + }, \ + +#define MALI_GPU_RESOURCE_GP_WITH_MMU(gp_addr, gp_irq, gp_mmu_addr, gp_mmu_irq) \ + { \ + .name = "Mali_GP", \ + .flags = IORESOURCE_MEM, \ + .start = gp_addr, \ + .end = gp_addr + 0x100, \ + }, \ + { \ + .name = "Mali_GP_IRQ", \ + .flags = IORESOURCE_IRQ, \ + .start = gp_irq, \ + .end = gp_irq, \ + }, \ + { \ + .name = "Mali_GP_MMU", \ + .flags = IORESOURCE_MEM, \ + .start = gp_mmu_addr, \ + .end = gp_mmu_addr + 0x100, \ + }, \ + { \ + .name = "Mali_GP_MMU_IRQ", \ + .flags = IORESOURCE_IRQ, \ + .start = gp_mmu_irq, \ + .end = gp_mmu_irq, \ + }, + +#define MALI_GPU_RESOURCE_PP(pp_addr, pp_irq) \ + { \ + .name = "Mali_PP", \ + .flags = IORESOURCE_MEM, \ + .start = pp_addr, \ + .end = pp_addr + 0x1100, \ + }, \ + { \ + .name = "Mali_PP_IRQ", \ + .flags = IORESOURCE_IRQ, \ + .start = pp_irq, \ + .end = pp_irq, \ + }, \ + +#define MALI_GPU_RESOURCE_PP_WITH_MMU(id, pp_addr, pp_irq, pp_mmu_addr, pp_mmu_irq) \ + { \ + .name = "Mali_PP" #id, \ + .flags = IORESOURCE_MEM, \ + .start = pp_addr, \ + .end = pp_addr + 0x1100, \ + }, \ + { \ + .name = "Mali_PP" #id "_IRQ", \ + .flags = IORESOURCE_IRQ, \ + .start = pp_irq, \ + .end = pp_irq, \ + }, \ + { \ + .name = "Mali_PP" #id "_MMU", \ + .flags = IORESOURCE_MEM, \ + .start = pp_mmu_addr, \ + .end = pp_mmu_addr + 0x100, \ + }, \ + { \ + .name = "Mali_PP" #id "_MMU_IRQ", \ + .flags = IORESOURCE_IRQ, \ + .start = pp_mmu_irq, \ + .end = pp_mmu_irq, \ + }, + +#define MALI_GPU_RESOURCE_MMU(mmu_addr, mmu_irq) \ + { \ + .name = "Mali_MMU", \ + .flags = IORESOURCE_MEM, \ + .start = mmu_addr, \ + .end = mmu_addr + 0x100, \ + }, \ + { \ + .name = "Mali_MMU_IRQ", \ + .flags = IORESOURCE_IRQ, \ + .start = mmu_irq, \ + .end = mmu_irq, \ + }, + +#define MALI_GPU_RESOURCE_PMU(pmu_addr) \ + { \ + .name = "Mali_PMU", \ + .flags = IORESOURCE_MEM, \ + .start = pmu_addr, \ + .end = pmu_addr + 0x100, \ + }, + +#define MALI_GPU_RESOURCE_DMA(dma_addr) \ + { \ + .name = "Mali_DMA", \ + .flags = IORESOURCE_MEM, \ + .start = dma_addr, \ + .end = dma_addr + 0x100, \ + }, + +#define MALI_GPU_RESOURCE_DLBU(dlbu_addr) \ + { \ + .name = "Mali_DLBU", \ + .flags = IORESOURCE_MEM, \ + .start = dlbu_addr, \ + .end = dlbu_addr + 0x100, \ + }, + +#define MALI_GPU_RESOURCE_BCAST(bcast_addr) \ + { \ + .name = "Mali_Broadcast", \ + .flags = IORESOURCE_MEM, \ + .start = bcast_addr, \ + .end = bcast_addr + 0x100, \ + }, + +#define MALI_GPU_RESOURCE_PP_BCAST(pp_addr, pp_irq) \ + { \ + .name = "Mali_PP_Broadcast", \ + .flags = IORESOURCE_MEM, \ + .start = pp_addr, \ + .end = pp_addr + 0x1100, \ + }, \ + { \ + .name = "Mali_PP_Broadcast_IRQ", \ + .flags = IORESOURCE_IRQ, \ + .start = pp_irq, \ + .end = pp_irq, \ + }, \ + +#define MALI_GPU_RESOURCE_PP_MMU_BCAST(pp_mmu_bcast_addr) \ + { \ + .name = "Mali_PP_MMU_Broadcast", \ + .flags = IORESOURCE_MEM, \ + .start = pp_mmu_bcast_addr, \ + .end = pp_mmu_bcast_addr + 0x100, \ + }, + + struct mali_gpu_utilization_data { + unsigned int utilization_gpu; /* Utilization for GP and all PP cores combined, 0 = no utilization, 256 = full utilization */ + unsigned int utilization_gp; /* Utilization for GP core only, 0 = no utilization, 256 = full utilization */ + unsigned int utilization_pp; /* Utilization for all PP cores combined, 0 = no utilization, 256 = full utilization */ + }; + + struct mali_gpu_clk_item { + unsigned int clock; /* unit(MHz) */ + unsigned int vol; + }; + + struct mali_gpu_clock { + struct mali_gpu_clk_item *item; + unsigned int num_of_steps; + }; + + struct mali_gpu_device_data { + /* Shared GPU memory */ + unsigned long shared_mem_size; + + /* + * Mali PMU switch delay. + * Only needed if the power gates are connected to the PMU in a high fanout + * network. This value is the number of Mali clock cycles it takes to + * enable the power gates and turn on the power mesh. + * This value will have no effect if a daisy chain implementation is used. + */ + u32 pmu_switch_delay; + + /* Mali Dynamic power domain configuration in sequence from 0-11 + * GP PP0 PP1 PP2 PP3 PP4 PP5 PP6 PP7, L2$0 L2$1 L2$2 + */ + u16 pmu_domain_config[12]; + + /* Dedicated GPU memory range (physical). */ + unsigned long dedicated_mem_start; + unsigned long dedicated_mem_size; + + /* Frame buffer memory to be accessible by Mali GPU (physical) */ + unsigned long fb_start; + unsigned long fb_size; + + /* Max runtime [ms] for jobs */ + int max_job_runtime; + + /* Report GPU utilization and related control in this interval (specified in ms) */ + unsigned long control_interval; + + /* Function that will receive periodic GPU utilization numbers */ + void (*utilization_callback)(struct mali_gpu_utilization_data *data); + + /* Fuction that platform callback for freq setting, needed when CONFIG_MALI_DVFS enabled */ + int (*set_freq)(int setting_clock_step); + /* Function that platfrom report it's clock info which driver can set, needed when CONFIG_MALI_DVFS enabled */ + void (*get_clock_info)(struct mali_gpu_clock **data); + /* Function that get the current clock info, needed when CONFIG_MALI_DVFS enabled */ + int (*get_freq)(void); + }; + + /** + * Pause the scheduling and power state changes of Mali device driver. + * mali_dev_resume() must always be called as soon as possible after this function + * in order to resume normal operation of the Mali driver. + */ + void mali_dev_pause(void); + + /** + * Resume scheduling and allow power changes in Mali device driver. + * This must always be called after mali_dev_pause(). + */ + void mali_dev_resume(void); + + /** @brief Set the desired number of PP cores to use. + * + * The internal Mali PMU will be used, if present, to physically power off the PP cores. + * + * @param num_cores The number of desired cores + * @return 0 on success, otherwise error. -EINVAL means an invalid number of cores was specified. + */ + int mali_perf_set_num_pp_cores(unsigned int num_cores); + +#endif diff --git a/drivers/gpu/arm/utgard/include/linux/mali/mali_utgard_ioctl.h b/drivers/gpu/arm/utgard/include/linux/mali/mali_utgard_ioctl.h new file mode 100644 index 00000000000000..6a6e69ab697891 --- /dev/null +++ b/drivers/gpu/arm/utgard/include/linux/mali/mali_utgard_ioctl.h @@ -0,0 +1,100 @@ +/* + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + + * Class Path Exception + * Linking this library statically or dynamically with other modules is making a combined work based on this library. + * Thus, the terms and conditions of the GNU General Public License cover the whole combination. + * As a special exception, the copyright holders of this library give you permission to link this library with independent modules + * to produce an executable, regardless of the license terms of these independent modules, and to copy and distribute the resulting + * executable under terms of your choice, provided that you also meet, for each linked independent module, the terms and conditions + * of the license of that module. An independent module is a module which is not derived from or based on this library. If you modify + * this library, you may extend this exception to your version of the library, but you are not obligated to do so. + * If you do not wish to do so, delete this exception statement from your version. + */ + +#ifndef __MALI_UTGARD_IOCTL_H__ +#define __MALI_UTGARD_IOCTL_H__ + +#include +#include +#include /* file system operations */ + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @file mali_kernel_ioctl.h + * Interface to the Linux device driver. + * This file describes the interface needed to use the Linux device driver. + * Its interface is designed to used by the HAL implementation through a thin arch layer. + */ + +/** + * ioctl commands + */ + +#define MALI_IOC_BASE 0x82 +#define MALI_IOC_CORE_BASE (_MALI_UK_CORE_SUBSYSTEM + MALI_IOC_BASE) +#define MALI_IOC_MEMORY_BASE (_MALI_UK_MEMORY_SUBSYSTEM + MALI_IOC_BASE) +#define MALI_IOC_PP_BASE (_MALI_UK_PP_SUBSYSTEM + MALI_IOC_BASE) +#define MALI_IOC_GP_BASE (_MALI_UK_GP_SUBSYSTEM + MALI_IOC_BASE) +#define MALI_IOC_PROFILING_BASE (_MALI_UK_PROFILING_SUBSYSTEM + MALI_IOC_BASE) +#define MALI_IOC_VSYNC_BASE (_MALI_UK_VSYNC_SUBSYSTEM + MALI_IOC_BASE) + +#define MALI_IOC_WAIT_FOR_NOTIFICATION _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_WAIT_FOR_NOTIFICATION, _mali_uk_wait_for_notification_s) +#define MALI_IOC_GET_API_VERSION _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_GET_API_VERSION, u32) +#define MALI_IOC_GET_API_VERSION_V2 _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_GET_API_VERSION, _mali_uk_get_api_version_v2_s) +#define MALI_IOC_POST_NOTIFICATION _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_POST_NOTIFICATION, _mali_uk_post_notification_s) +#define MALI_IOC_GET_USER_SETTING _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_GET_USER_SETTING, _mali_uk_get_user_setting_s) +#define MALI_IOC_GET_USER_SETTINGS _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_GET_USER_SETTINGS, _mali_uk_get_user_settings_s) +#define MALI_IOC_REQUEST_HIGH_PRIORITY _IOW (MALI_IOC_CORE_BASE, _MALI_UK_REQUEST_HIGH_PRIORITY, _mali_uk_request_high_priority_s) +#define MALI_IOC_TIMELINE_GET_LATEST_POINT _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_TIMELINE_GET_LATEST_POINT, _mali_uk_timeline_get_latest_point_s) +#define MALI_IOC_TIMELINE_WAIT _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_TIMELINE_WAIT, _mali_uk_timeline_wait_s) +#define MALI_IOC_TIMELINE_CREATE_SYNC_FENCE _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_TIMELINE_CREATE_SYNC_FENCE, _mali_uk_timeline_create_sync_fence_s) +#define MALI_IOC_SOFT_JOB_START _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_SOFT_JOB_START, _mali_uk_soft_job_start_s) +#define MALI_IOC_SOFT_JOB_SIGNAL _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_SOFT_JOB_SIGNAL, _mali_uk_soft_job_signal_s) +#define MALI_IOC_PENDING_SUBMIT _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_PENDING_SUBMIT, _mali_uk_pending_submit_s) + +#define MALI_IOC_MEM_ALLOC _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_ALLOC_MEM, _mali_uk_alloc_mem_s) +#define MALI_IOC_MEM_FREE _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_FREE_MEM, _mali_uk_free_mem_s) +#define MALI_IOC_MEM_BIND _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_BIND_MEM, _mali_uk_bind_mem_s) +#define MALI_IOC_MEM_UNBIND _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_UNBIND_MEM, _mali_uk_unbind_mem_s) +#define MALI_IOC_MEM_COW _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_COW_MEM, _mali_uk_cow_mem_s) +#define MALI_IOC_MEM_COW_MODIFY_RANGE _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_COW_MODIFY_RANGE, _mali_uk_cow_modify_range_s) +#define MALI_IOC_MEM_RESIZE _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_RESIZE_MEM, _mali_uk_mem_resize_s) +#define MALI_IOC_MEM_DMA_BUF_GET_SIZE _IOR(MALI_IOC_MEMORY_BASE, _MALI_UK_DMA_BUF_GET_SIZE, _mali_uk_dma_buf_get_size_s) +#define MALI_IOC_MEM_QUERY_MMU_PAGE_TABLE_DUMP_SIZE _IOR (MALI_IOC_MEMORY_BASE, _MALI_UK_QUERY_MMU_PAGE_TABLE_DUMP_SIZE, _mali_uk_query_mmu_page_table_dump_size_s) +#define MALI_IOC_MEM_DUMP_MMU_PAGE_TABLE _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_DUMP_MMU_PAGE_TABLE, _mali_uk_dump_mmu_page_table_s) +#define MALI_IOC_MEM_WRITE_SAFE _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_MEM_WRITE_SAFE, _mali_uk_mem_write_safe_s) + +#define MALI_IOC_PP_START_JOB _IOWR(MALI_IOC_PP_BASE, _MALI_UK_PP_START_JOB, _mali_uk_pp_start_job_s) +#define MALI_IOC_PP_AND_GP_START_JOB _IOWR(MALI_IOC_PP_BASE, _MALI_UK_PP_AND_GP_START_JOB, _mali_uk_pp_and_gp_start_job_s) +#define MALI_IOC_PP_NUMBER_OF_CORES_GET _IOR (MALI_IOC_PP_BASE, _MALI_UK_GET_PP_NUMBER_OF_CORES, _mali_uk_get_pp_number_of_cores_s) +#define MALI_IOC_PP_CORE_VERSION_GET _IOR (MALI_IOC_PP_BASE, _MALI_UK_GET_PP_CORE_VERSION, _mali_uk_get_pp_core_version_s) +#define MALI_IOC_PP_DISABLE_WB _IOW (MALI_IOC_PP_BASE, _MALI_UK_PP_DISABLE_WB, _mali_uk_pp_disable_wb_s) + +#define MALI_IOC_GP2_START_JOB _IOWR(MALI_IOC_GP_BASE, _MALI_UK_GP_START_JOB, _mali_uk_gp_start_job_s) +#define MALI_IOC_GP2_NUMBER_OF_CORES_GET _IOR (MALI_IOC_GP_BASE, _MALI_UK_GET_GP_NUMBER_OF_CORES, _mali_uk_get_gp_number_of_cores_s) +#define MALI_IOC_GP2_CORE_VERSION_GET _IOR (MALI_IOC_GP_BASE, _MALI_UK_GET_GP_CORE_VERSION, _mali_uk_get_gp_core_version_s) +#define MALI_IOC_GP2_SUSPEND_RESPONSE _IOW (MALI_IOC_GP_BASE, _MALI_UK_GP_SUSPEND_RESPONSE,_mali_uk_gp_suspend_response_s) + +#define MALI_IOC_PROFILING_ADD_EVENT _IOWR(MALI_IOC_PROFILING_BASE, _MALI_UK_PROFILING_ADD_EVENT, _mali_uk_profiling_add_event_s) +#define MALI_IOC_PROFILING_REPORT_SW_COUNTERS _IOW (MALI_IOC_PROFILING_BASE, _MALI_UK_PROFILING_REPORT_SW_COUNTERS, _mali_uk_sw_counters_report_s) +#define MALI_IOC_PROFILING_MEMORY_USAGE_GET _IOR(MALI_IOC_PROFILING_BASE, _MALI_UK_PROFILING_MEMORY_USAGE_GET, _mali_uk_profiling_memory_usage_get_s) +#define MALI_IOC_PROFILING_STREAM_FD_GET _IOR(MALI_IOC_PROFILING_BASE, _MALI_UK_PROFILING_STREAM_FD_GET, _mali_uk_profiling_stream_fd_get_s) +#define MALI_IOC_PROILING_CONTROL_SET _IOR(MALI_IOC_PROFILING_BASE, _MALI_UK_PROFILING_CONTROL_SET, _mali_uk_profiling_control_set_s) + +#define MALI_IOC_VSYNC_EVENT_REPORT _IOW (MALI_IOC_VSYNC_BASE, _MALI_UK_VSYNC_EVENT_REPORT, _mali_uk_vsync_event_report_s) + +#ifdef __cplusplus +} +#endif + +#endif /* __MALI_UTGARD_IOCTL_H__ */ diff --git a/drivers/gpu/arm/utgard/include/linux/mali/mali_utgard_profiling_events.h b/drivers/gpu/arm/utgard/include/linux/mali/mali_utgard_profiling_events.h new file mode 100644 index 00000000000000..279bf8ee38a4ec --- /dev/null +++ b/drivers/gpu/arm/utgard/include/linux/mali/mali_utgard_profiling_events.h @@ -0,0 +1,200 @@ +/* + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + + * Class Path Exception + * Linking this library statically or dynamically with other modules is making a combined work based on this library. + * Thus, the terms and conditions of the GNU General Public License cover the whole combination. + * As a special exception, the copyright holders of this library give you permission to link this library with independent modules + * to produce an executable, regardless of the license terms of these independent modules, and to copy and distribute the resulting + * executable under terms of your choice, provided that you also meet, for each linked independent module, the terms and conditions + * of the license of that module. An independent module is a module which is not derived from or based on this library. If you modify + * this library, you may extend this exception to your version of the library, but you are not obligated to do so. + * If you do not wish to do so, delete this exception statement from your version. + */ + +#ifndef _MALI_UTGARD_PROFILING_EVENTS_H_ +#define _MALI_UTGARD_PROFILING_EVENTS_H_ + +/* + * The event ID is a 32 bit value consisting of different fields + * reserved, 4 bits, for future use + * event type, 4 bits, cinstr_profiling_event_type_t + * event channel, 8 bits, the source of the event. + * event data, 16 bit field, data depending on event type + */ + +/** + * Specifies what kind of event this is + */ +typedef enum { + MALI_PROFILING_EVENT_TYPE_SINGLE = 0 << 24, + MALI_PROFILING_EVENT_TYPE_START = 1 << 24, + MALI_PROFILING_EVENT_TYPE_STOP = 2 << 24, + MALI_PROFILING_EVENT_TYPE_SUSPEND = 3 << 24, + MALI_PROFILING_EVENT_TYPE_RESUME = 4 << 24, +} cinstr_profiling_event_type_t; + + +/** + * Secifies the channel/source of the event + */ +typedef enum { + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE = 0 << 16, + MALI_PROFILING_EVENT_CHANNEL_GP0 = 1 << 16, + MALI_PROFILING_EVENT_CHANNEL_PP0 = 5 << 16, + MALI_PROFILING_EVENT_CHANNEL_PP1 = 6 << 16, + MALI_PROFILING_EVENT_CHANNEL_PP2 = 7 << 16, + MALI_PROFILING_EVENT_CHANNEL_PP3 = 8 << 16, + MALI_PROFILING_EVENT_CHANNEL_PP4 = 9 << 16, + MALI_PROFILING_EVENT_CHANNEL_PP5 = 10 << 16, + MALI_PROFILING_EVENT_CHANNEL_PP6 = 11 << 16, + MALI_PROFILING_EVENT_CHANNEL_PP7 = 12 << 16, + MALI_PROFILING_EVENT_CHANNEL_GPU = 21 << 16, +} cinstr_profiling_event_channel_t; + + +#define MALI_PROFILING_MAKE_EVENT_CHANNEL_GP(num) (((MALI_PROFILING_EVENT_CHANNEL_GP0 >> 16) + (num)) << 16) +#define MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(num) (((MALI_PROFILING_EVENT_CHANNEL_PP0 >> 16) + (num)) << 16) + +/** + * These events are applicable when the type MALI_PROFILING_EVENT_TYPE_SINGLE is used from software channel + */ +typedef enum { + MALI_PROFILING_EVENT_REASON_SINGLE_SW_NONE = 0, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_EGL_NEW_FRAME = 1, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_FLUSH = 2, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_EGL_SWAP_BUFFERS = 3, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_FB_EVENT = 4, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_GP_ENQUEUE = 5, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_PP_ENQUEUE = 6, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_READBACK = 7, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_WRITEBACK = 8, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_ENTER_API_FUNC = 10, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_LEAVE_API_FUNC = 11, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_DISCARD_ATTACHMENTS = 13, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_UMP_TRY_LOCK = 53, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_UMP_LOCK = 54, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_UMP_UNLOCK = 55, + MALI_PROFILING_EVENT_REASON_SINGLE_LOCK_CONTENDED = 56, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_EGL_MALI_FENCE_DUP = 57, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_EGL_SET_PP_JOB_FENCE = 58, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_EGL_WAIT_SYNC = 59, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_EGL_CREATE_FENCE_SYNC = 60, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_EGL_CREATE_NATIVE_FENCE_SYNC = 61, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_EGL_FENCE_FLUSH = 62, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_EGL_FLUSH_SERVER_WAITS = 63, +} cinstr_profiling_event_reason_single_sw_t; + +/** + * These events are applicable when the type MALI_PROFILING_EVENT_TYPE_START/STOP is used from software channel + * to inform whether the core is physical or virtual + */ +typedef enum { + MALI_PROFILING_EVENT_REASON_START_STOP_HW_PHYSICAL = 0, + MALI_PROFILING_EVENT_REASON_START_STOP_HW_VIRTUAL = 1, +} cinstr_profiling_event_reason_start_stop_hw_t; + +/** + * These events are applicable when the type MALI_PROFILING_EVENT_TYPE_START/STOP is used from software channel + */ +typedef enum { + /*MALI_PROFILING_EVENT_REASON_START_STOP_SW_NONE = 0,*/ + MALI_PROFILING_EVENT_REASON_START_STOP_SW_MALI = 1, + MALI_PROFILING_EVENT_REASON_START_STOP_SW_CALLBACK_THREAD = 2, + MALI_PROFILING_EVENT_REASON_START_STOP_SW_WORKER_THREAD = 3, + MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF = 4, + MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF = 5, +} cinstr_profiling_event_reason_start_stop_sw_t; + +/** + * These events are applicable when the type MALI_PROFILING_EVENT_TYPE_SUSPEND/RESUME is used from software channel + */ +typedef enum { + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_NONE = 0, /* used */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_PIPELINE_FULL = 1, /* NOT used */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_VSYNC = 26, /* used in some build configurations */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_FB_IFRAME_WAIT = 27, /* USED */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_FB_IFRAME_SYNC = 28, /* USED */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_VG_WAIT_FILTER_CLEANUP = 29, /* used */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_VG_WAIT_TEXTURE = 30, /* used */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_GLES_WAIT_MIPLEVEL = 31, /* used */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_GLES_WAIT_READPIXELS = 32, /* used */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_EGL_WAIT_SWAP_IMMEDIATE = 33, /* NOT used */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_ICS_QUEUE_BUFFER = 34, /* USED */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_ICS_DEQUEUE_BUFFER = 35, /* USED */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_UMP_LOCK = 36, /* Not currently used */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_X11_GLOBAL_LOCK = 37, /* Not currently used */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_X11_SWAP = 38, /* Not currently used */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_MALI_EGL_IMAGE_SYNC_WAIT = 39, /* USED */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_GP_JOB_HANDLING = 40, /* USED */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_PP_JOB_HANDLING = 41, /* USED */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_EGL_MALI_FENCE_MERGE = 42, /* USED */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_EGL_MALI_FENCE_DUP = 43, + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_EGL_FLUSH_SERVER_WAITS = 44, + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_EGL_WAIT_SYNC = 45, /* USED */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_FB_JOBS_WAIT = 46, /* USED */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_FB_NOFRAMES_WAIT = 47, /* USED */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_FB_NOJOBS_WAIT = 48, /* USED */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_SUBMIT_LIMITER_WAIT = 49, /* USED */ +} cinstr_profiling_event_reason_suspend_resume_sw_t; + +/** + * These events are applicable when the type MALI_PROFILING_EVENT_TYPE_SINGLE is used from a HW channel (GPx+PPx) + */ +typedef enum { + MALI_PROFILING_EVENT_REASON_SINGLE_HW_NONE = 0, + MALI_PROFILING_EVENT_REASON_SINGLE_HW_INTERRUPT = 1, + MALI_PROFILING_EVENT_REASON_SINGLE_HW_FLUSH = 2, +} cinstr_profiling_event_reason_single_hw_t; + +/** + * These events are applicable when the type MALI_PROFILING_EVENT_TYPE_SINGLE is used from the GPU channel + */ +typedef enum { + MALI_PROFILING_EVENT_REASON_SINGLE_GPU_NONE = 0, + MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE = 1, + MALI_PROFILING_EVENT_REASON_SINGLE_GPU_L20_COUNTERS = 2, + MALI_PROFILING_EVENT_REASON_SINGLE_GPU_L21_COUNTERS = 3, + MALI_PROFILING_EVENT_REASON_SINGLE_GPU_L22_COUNTERS = 4, +} cinstr_profiling_event_reason_single_gpu_t; + +/** + * These values are applicable for the 3rd data parameter when + * the type MALI_PROFILING_EVENT_TYPE_START is used from the software channel + * with the MALI_PROFILING_EVENT_REASON_START_STOP_BOTTOM_HALF reason. + */ +typedef enum { + MALI_PROFILING_EVENT_DATA_CORE_GP0 = 1, + MALI_PROFILING_EVENT_DATA_CORE_PP0 = 5, + MALI_PROFILING_EVENT_DATA_CORE_PP1 = 6, + MALI_PROFILING_EVENT_DATA_CORE_PP2 = 7, + MALI_PROFILING_EVENT_DATA_CORE_PP3 = 8, + MALI_PROFILING_EVENT_DATA_CORE_PP4 = 9, + MALI_PROFILING_EVENT_DATA_CORE_PP5 = 10, + MALI_PROFILING_EVENT_DATA_CORE_PP6 = 11, + MALI_PROFILING_EVENT_DATA_CORE_PP7 = 12, + MALI_PROFILING_EVENT_DATA_CORE_GP0_MMU = 22, /* GP0 + 21 */ + MALI_PROFILING_EVENT_DATA_CORE_PP0_MMU = 26, /* PP0 + 21 */ + MALI_PROFILING_EVENT_DATA_CORE_PP1_MMU = 27, /* PP1 + 21 */ + MALI_PROFILING_EVENT_DATA_CORE_PP2_MMU = 28, /* PP2 + 21 */ + MALI_PROFILING_EVENT_DATA_CORE_PP3_MMU = 29, /* PP3 + 21 */ + MALI_PROFILING_EVENT_DATA_CORE_PP4_MMU = 30, /* PP4 + 21 */ + MALI_PROFILING_EVENT_DATA_CORE_PP5_MMU = 31, /* PP5 + 21 */ + MALI_PROFILING_EVENT_DATA_CORE_PP6_MMU = 32, /* PP6 + 21 */ + MALI_PROFILING_EVENT_DATA_CORE_PP7_MMU = 33, /* PP7 + 21 */ + +} cinstr_profiling_event_data_core_t; + +#define MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP(num) (MALI_PROFILING_EVENT_DATA_CORE_GP0 + (num)) +#define MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP_MMU(num) (MALI_PROFILING_EVENT_DATA_CORE_GP0_MMU + (num)) +#define MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP(num) (MALI_PROFILING_EVENT_DATA_CORE_PP0 + (num)) +#define MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP_MMU(num) (MALI_PROFILING_EVENT_DATA_CORE_PP0_MMU + (num)) + + +#endif /*_MALI_UTGARD_PROFILING_EVENTS_H_*/ diff --git a/drivers/gpu/arm/utgard/include/linux/mali/mali_utgard_profiling_gator_api.h b/drivers/gpu/arm/utgard/include/linux/mali/mali_utgard_profiling_gator_api.h new file mode 100644 index 00000000000000..b922187a0b88b6 --- /dev/null +++ b/drivers/gpu/arm/utgard/include/linux/mali/mali_utgard_profiling_gator_api.h @@ -0,0 +1,315 @@ +/* + * Copyright (C) 2013, 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + + * Class Path Exception + * Linking this library statically or dynamically with other modules is making a combined work based on this library. + * Thus, the terms and conditions of the GNU General Public License cover the whole combination. + * As a special exception, the copyright holders of this library give you permission to link this library with independent modules + * to produce an executable, regardless of the license terms of these independent modules, and to copy and distribute the resulting + * executable under terms of your choice, provided that you also meet, for each linked independent module, the terms and conditions + * of the license of that module. An independent module is a module which is not derived from or based on this library. If you modify + * this library, you may extend this exception to your version of the library, but you are not obligated to do so. + * If you do not wish to do so, delete this exception statement from your version. + */ + +#ifndef __MALI_UTGARD_PROFILING_GATOR_API_H__ +#define __MALI_UTGARD_PROFILING_GATOR_API_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +#define MALI_PROFILING_API_VERSION 4 + +#define MAX_NUM_L2_CACHE_CORES 3 +#define MAX_NUM_FP_CORES 8 +#define MAX_NUM_VP_CORES 1 + +#define _MALI_SPCIAL_COUNTER_DESCRIPTIONS \ + { \ + "Filmstrip_cnt0", \ + "Frequency", \ + "Voltage", \ + "vertex", \ + "fragment", \ + "Total_alloc_pages", \ + }; + +#define _MALI_MEM_COUTNER_DESCRIPTIONS \ + { \ + "untyped_memory", \ + "vertex_index_buffer", \ + "texture_buffer", \ + "varying_buffer", \ + "render_target", \ + "pbuffer_buffer", \ + "plbu_heap", \ + "pointer_array_buffer", \ + "slave_tilelist", \ + "untyped_gp_cmdlist", \ + "polygon_cmdlist", \ + "texture_descriptor", \ + "render_state_word", \ + "shader", \ + "stream_buffer", \ + "fragment_stack", \ + "uniform", \ + "untyped_frame_pool", \ + "untyped_surface", \ + }; + +/** The list of events supported by the Mali DDK. */ +typedef enum { + /* Vertex processor activity */ + ACTIVITY_VP_0 = 0, + + /* Fragment processor activity */ + ACTIVITY_FP_0, + ACTIVITY_FP_1, + ACTIVITY_FP_2, + ACTIVITY_FP_3, + ACTIVITY_FP_4, + ACTIVITY_FP_5, + ACTIVITY_FP_6, + ACTIVITY_FP_7, + + /* L2 cache counters */ + COUNTER_L2_0_C0, + COUNTER_L2_0_C1, + COUNTER_L2_1_C0, + COUNTER_L2_1_C1, + COUNTER_L2_2_C0, + COUNTER_L2_2_C1, + + /* Vertex processor counters */ + COUNTER_VP_0_C0, + COUNTER_VP_0_C1, + + /* Fragment processor counters */ + COUNTER_FP_0_C0, + COUNTER_FP_0_C1, + COUNTER_FP_1_C0, + COUNTER_FP_1_C1, + COUNTER_FP_2_C0, + COUNTER_FP_2_C1, + COUNTER_FP_3_C0, + COUNTER_FP_3_C1, + COUNTER_FP_4_C0, + COUNTER_FP_4_C1, + COUNTER_FP_5_C0, + COUNTER_FP_5_C1, + COUNTER_FP_6_C0, + COUNTER_FP_6_C1, + COUNTER_FP_7_C0, + COUNTER_FP_7_C1, + + /* + * If more hardware counters are added, the _mali_osk_hw_counter_table + * below should also be updated. + */ + + /* EGL software counters */ + COUNTER_EGL_BLIT_TIME, + + /* GLES software counters */ + COUNTER_GLES_DRAW_ELEMENTS_CALLS, + COUNTER_GLES_DRAW_ELEMENTS_NUM_INDICES, + COUNTER_GLES_DRAW_ELEMENTS_NUM_TRANSFORMED, + COUNTER_GLES_DRAW_ARRAYS_CALLS, + COUNTER_GLES_DRAW_ARRAYS_NUM_TRANSFORMED, + COUNTER_GLES_DRAW_POINTS, + COUNTER_GLES_DRAW_LINES, + COUNTER_GLES_DRAW_LINE_LOOP, + COUNTER_GLES_DRAW_LINE_STRIP, + COUNTER_GLES_DRAW_TRIANGLES, + COUNTER_GLES_DRAW_TRIANGLE_STRIP, + COUNTER_GLES_DRAW_TRIANGLE_FAN, + COUNTER_GLES_NON_VBO_DATA_COPY_TIME, + COUNTER_GLES_UNIFORM_BYTES_COPIED_TO_MALI, + COUNTER_GLES_UPLOAD_TEXTURE_TIME, + COUNTER_GLES_UPLOAD_VBO_TIME, + COUNTER_GLES_NUM_FLUSHES, + COUNTER_GLES_NUM_VSHADERS_GENERATED, + COUNTER_GLES_NUM_FSHADERS_GENERATED, + COUNTER_GLES_VSHADER_GEN_TIME, + COUNTER_GLES_FSHADER_GEN_TIME, + COUNTER_GLES_INPUT_TRIANGLES, + COUNTER_GLES_VXCACHE_HIT, + COUNTER_GLES_VXCACHE_MISS, + COUNTER_GLES_VXCACHE_COLLISION, + COUNTER_GLES_CULLED_TRIANGLES, + COUNTER_GLES_CULLED_LINES, + COUNTER_GLES_BACKFACE_TRIANGLES, + COUNTER_GLES_GBCLIP_TRIANGLES, + COUNTER_GLES_GBCLIP_LINES, + COUNTER_GLES_TRIANGLES_DRAWN, + COUNTER_GLES_DRAWCALL_TIME, + COUNTER_GLES_TRIANGLES_COUNT, + COUNTER_GLES_INDEPENDENT_TRIANGLES_COUNT, + COUNTER_GLES_STRIP_TRIANGLES_COUNT, + COUNTER_GLES_FAN_TRIANGLES_COUNT, + COUNTER_GLES_LINES_COUNT, + COUNTER_GLES_INDEPENDENT_LINES_COUNT, + COUNTER_GLES_STRIP_LINES_COUNT, + COUNTER_GLES_LOOP_LINES_COUNT, + + /* Special counter */ + + /* Framebuffer capture pseudo-counter */ + COUNTER_FILMSTRIP, + COUNTER_FREQUENCY, + COUNTER_VOLTAGE, + COUNTER_VP_ACTIVITY, + COUNTER_FP_ACTIVITY, + COUNTER_TOTAL_ALLOC_PAGES, + + /* Memory usage counter */ + COUNTER_MEM_UNTYPED, + COUNTER_MEM_VB_IB, + COUNTER_MEM_TEXTURE, + COUNTER_MEM_VARYING, + COUNTER_MEM_RT, + COUNTER_MEM_PBUFFER, + /* memory usages for gp command */ + COUNTER_MEM_PLBU_HEAP, + COUNTER_MEM_POINTER_ARRAY, + COUNTER_MEM_SLAVE_TILELIST, + COUNTER_MEM_UNTYPE_GP_CMDLIST, + /* memory usages for polygon list command */ + COUNTER_MEM_POLYGON_CMDLIST, + /* memory usages for pp command */ + COUNTER_MEM_TD, + COUNTER_MEM_RSW, + /* other memory usages */ + COUNTER_MEM_SHADER, + COUNTER_MEM_STREAMS, + COUNTER_MEM_FRAGMENT_STACK, + COUNTER_MEM_UNIFORM, + /* Special mem usage, which is used for mem pool allocation */ + COUNTER_MEM_UNTYPE_MEM_POOL, + COUNTER_MEM_UNTYPE_SURFACE, + + NUMBER_OF_EVENTS +} _mali_osk_counter_id; + +#define FIRST_ACTIVITY_EVENT ACTIVITY_VP_0 +#define LAST_ACTIVITY_EVENT ACTIVITY_FP_7 + +#define FIRST_HW_COUNTER COUNTER_L2_0_C0 +#define LAST_HW_COUNTER COUNTER_FP_7_C1 + +#define FIRST_SW_COUNTER COUNTER_EGL_BLIT_TIME +#define LAST_SW_COUNTER COUNTER_GLES_LOOP_LINES_COUNT + +#define FIRST_SPECIAL_COUNTER COUNTER_FILMSTRIP +#define LAST_SPECIAL_COUNTER COUNTER_TOTAL_ALLOC_PAGES + +#define FIRST_MEM_COUNTER COUNTER_MEM_UNTYPED +#define LAST_MEM_COUNTER COUNTER_MEM_UNTYPE_SURFACE + +#define MALI_PROFILING_MEM_COUNTERS_NUM (LAST_MEM_COUNTER - FIRST_MEM_COUNTER + 1) +#define MALI_PROFILING_SPECIAL_COUNTERS_NUM (LAST_SPECIAL_COUNTER - FIRST_SPECIAL_COUNTER + 1) +#define MALI_PROFILING_SW_COUNTERS_NUM (LAST_SW_COUNTER - FIRST_SW_COUNTER + 1) + +/** + * Define the stream header type for porfiling stream. + */ +#define STREAM_HEADER_FRAMEBUFFER 0x05 /* The stream packet header type for framebuffer dumping. */ +#define STREAM_HEADER_COUNTER_VALUE 0x09 /* The stream packet header type for hw/sw/memory counter sampling. */ +#define STREAM_HEADER_CORE_ACTIVITY 0x0a /* The stream packet header type for activity counter sampling. */ +#define STREAM_HEADER_SIZE 5 + +/** + * Define the packet header type of profiling control packet. + */ +#define PACKET_HEADER_ERROR 0x80 /* The response packet header type if error. */ +#define PACKET_HEADER_ACK 0x81 /* The response packet header type if OK. */ +#define PACKET_HEADER_COUNTERS_REQUEST 0x82 /* The control packet header type to request counter information from ddk. */ +#define PACKET_HEADER_COUNTERS_ACK 0x83 /* The response packet header type to send out counter information. */ +#define PACKET_HEADER_COUNTERS_ENABLE 0x84 /* The control packet header type to enable counters. */ +#define PACKET_HEADER_START_CAPTURE_VALUE 0x85 /* The control packet header type to start capture values. */ + +#define PACKET_HEADER_SIZE 5 + +/** + * Structure to pass performance counter data of a Mali core + */ +typedef struct _mali_profiling_core_counters { + u32 source0; + u32 value0; + u32 source1; + u32 value1; +} _mali_profiling_core_counters; + +/** + * Structure to pass performance counter data of Mali L2 cache cores + */ +typedef struct _mali_profiling_l2_counter_values { + struct _mali_profiling_core_counters cores[MAX_NUM_L2_CACHE_CORES]; +} _mali_profiling_l2_counter_values; + +/** + * Structure to pass data defining Mali instance in use: + * + * mali_product_id - Mali product id + * mali_version_major - Mali version major number + * mali_version_minor - Mali version minor number + * num_of_l2_cores - number of L2 cache cores + * num_of_fp_cores - number of fragment processor cores + * num_of_vp_cores - number of vertex processor cores + */ +typedef struct _mali_profiling_mali_version { + u32 mali_product_id; + u32 mali_version_major; + u32 mali_version_minor; + u32 num_of_l2_cores; + u32 num_of_fp_cores; + u32 num_of_vp_cores; +} _mali_profiling_mali_version; + +/** + * Structure to define the mali profiling counter struct. + */ +typedef struct mali_profiling_counter { + char counter_name[40]; + u32 counter_id; + u32 counter_event; + u32 prev_counter_value; + u32 current_counter_value; + u32 key; + int enabled; +} mali_profiling_counter; + +/* + * List of possible actions to be controlled by Streamline. + * The following numbers are used by gator to control the frame buffer dumping and s/w counter reporting. + * We cannot use the enums in mali_uk_types.h because they are unknown inside gator. + */ +#define FBDUMP_CONTROL_ENABLE (1) +#define FBDUMP_CONTROL_RATE (2) +#define SW_COUNTER_ENABLE (3) +#define FBDUMP_CONTROL_RESIZE_FACTOR (4) +#define MEM_COUNTER_ENABLE (5) +#define ANNOTATE_PROFILING_ENABLE (6) + +void _mali_profiling_control(u32 action, u32 value); + +u32 _mali_profiling_get_l2_counters(_mali_profiling_l2_counter_values *values); + +int _mali_profiling_set_event(u32 counter_id, s32 event_id); + +u32 _mali_profiling_get_api_version(void); + +void _mali_profiling_get_mali_version(struct _mali_profiling_mali_version *values); + +#ifdef __cplusplus +} +#endif + +#endif /* __MALI_UTGARD_PROFILING_GATOR_API_H__ */ diff --git a/drivers/gpu/arm/utgard/include/linux/mali/mali_utgard_uk_types.h b/drivers/gpu/arm/utgard/include/linux/mali/mali_utgard_uk_types.h new file mode 100644 index 00000000000000..a291dfba2c0877 --- /dev/null +++ b/drivers/gpu/arm/utgard/include/linux/mali/mali_utgard_uk_types.h @@ -0,0 +1,1106 @@ +/* + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + + * Class Path Exception + * Linking this library statically or dynamically with other modules is making a combined work based on this library. + * Thus, the terms and conditions of the GNU General Public License cover the whole combination. + * As a special exception, the copyright holders of this library give you permission to link this library with independent modules + * to produce an executable, regardless of the license terms of these independent modules, and to copy and distribute the resulting + * executable under terms of your choice, provided that you also meet, for each linked independent module, the terms and conditions + * of the license of that module. An independent module is a module which is not derived from or based on this library. If you modify + * this library, you may extend this exception to your version of the library, but you are not obligated to do so. + * If you do not wish to do so, delete this exception statement from your version. + */ + +/** + * @file mali_uk_types.h + * Defines the types and constants used in the user-kernel interface + */ + +#ifndef __MALI_UTGARD_UK_TYPES_H__ +#define __MALI_UTGARD_UK_TYPES_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +/* Iteration functions depend on these values being consecutive. */ +#define MALI_UK_TIMELINE_GP 0 +#define MALI_UK_TIMELINE_PP 1 +#define MALI_UK_TIMELINE_SOFT 2 +#define MALI_UK_TIMELINE_MAX 3 + +#define MALI_UK_BIG_VARYING_SIZE (1024*1024*2) + +typedef struct { + u32 points[MALI_UK_TIMELINE_MAX]; + s32 sync_fd; +} _mali_uk_fence_t; + +/** + * @addtogroup uddapi Unified Device Driver (UDD) APIs + * + * @{ + */ + +/** + * @addtogroup u_k_api UDD User/Kernel Interface (U/K) APIs + * + * @{ + */ + +/** @defgroup _mali_uk_core U/K Core + * @{ */ + +/** Definition of subsystem numbers, to assist in creating a unique identifier + * for each U/K call. + * + * @see _mali_uk_functions */ +typedef enum { + _MALI_UK_CORE_SUBSYSTEM, /**< Core Group of U/K calls */ + _MALI_UK_MEMORY_SUBSYSTEM, /**< Memory Group of U/K calls */ + _MALI_UK_PP_SUBSYSTEM, /**< Fragment Processor Group of U/K calls */ + _MALI_UK_GP_SUBSYSTEM, /**< Vertex Processor Group of U/K calls */ + _MALI_UK_PROFILING_SUBSYSTEM, /**< Profiling Group of U/K calls */ + _MALI_UK_VSYNC_SUBSYSTEM, /**< VSYNC Group of U/K calls */ +} _mali_uk_subsystem_t; + +/** Within a function group each function has its unique sequence number + * to assist in creating a unique identifier for each U/K call. + * + * An ordered pair of numbers selected from + * ( \ref _mali_uk_subsystem_t,\ref _mali_uk_functions) will uniquely identify the + * U/K call across all groups of functions, and all functions. */ +typedef enum { + /** Core functions */ + + _MALI_UK_OPEN = 0, /**< _mali_ukk_open() */ + _MALI_UK_CLOSE, /**< _mali_ukk_close() */ + _MALI_UK_WAIT_FOR_NOTIFICATION, /**< _mali_ukk_wait_for_notification() */ + _MALI_UK_GET_API_VERSION, /**< _mali_ukk_get_api_version() */ + _MALI_UK_POST_NOTIFICATION, /**< _mali_ukk_post_notification() */ + _MALI_UK_GET_USER_SETTING, /**< _mali_ukk_get_user_setting() *//**< [out] */ + _MALI_UK_GET_USER_SETTINGS, /**< _mali_ukk_get_user_settings() *//**< [out] */ + _MALI_UK_REQUEST_HIGH_PRIORITY, /**< _mali_ukk_request_high_priority() */ + _MALI_UK_TIMELINE_GET_LATEST_POINT, /**< _mali_ukk_timeline_get_latest_point() */ + _MALI_UK_TIMELINE_WAIT, /**< _mali_ukk_timeline_wait() */ + _MALI_UK_TIMELINE_CREATE_SYNC_FENCE, /**< _mali_ukk_timeline_create_sync_fence() */ + _MALI_UK_SOFT_JOB_START, /**< _mali_ukk_soft_job_start() */ + _MALI_UK_SOFT_JOB_SIGNAL, /**< _mali_ukk_soft_job_signal() */ + _MALI_UK_PENDING_SUBMIT, /**< _mali_ukk_pending_submit() */ + + /** Memory functions */ + + _MALI_UK_ALLOC_MEM = 0, /**< _mali_ukk_alloc_mem() */ + _MALI_UK_FREE_MEM, /**< _mali_ukk_free_mem() */ + _MALI_UK_BIND_MEM, /**< _mali_ukk_mem_bind() */ + _MALI_UK_UNBIND_MEM, /**< _mali_ukk_mem_unbind() */ + _MALI_UK_COW_MEM, /**< _mali_ukk_mem_cow() */ + _MALI_UK_COW_MODIFY_RANGE, /**< _mali_ukk_mem_cow_modify_range() */ + _MALI_UK_RESIZE_MEM, /**<._mali_ukk_mem_resize() */ + _MALI_UK_QUERY_MMU_PAGE_TABLE_DUMP_SIZE, /**< _mali_ukk_mem_get_mmu_page_table_dump_size() */ + _MALI_UK_DUMP_MMU_PAGE_TABLE, /**< _mali_ukk_mem_dump_mmu_page_table() */ + _MALI_UK_DMA_BUF_GET_SIZE, /**< _mali_ukk_dma_buf_get_size() */ + _MALI_UK_MEM_WRITE_SAFE, /**< _mali_uku_mem_write_safe() */ + + /** Common functions for each core */ + + _MALI_UK_START_JOB = 0, /**< Start a Fragment/Vertex Processor Job on a core */ + _MALI_UK_GET_NUMBER_OF_CORES, /**< Get the number of Fragment/Vertex Processor cores */ + _MALI_UK_GET_CORE_VERSION, /**< Get the Fragment/Vertex Processor version compatible with all cores */ + + /** Fragment Processor Functions */ + + _MALI_UK_PP_START_JOB = _MALI_UK_START_JOB, /**< _mali_ukk_pp_start_job() */ + _MALI_UK_GET_PP_NUMBER_OF_CORES = _MALI_UK_GET_NUMBER_OF_CORES, /**< _mali_ukk_get_pp_number_of_cores() */ + _MALI_UK_GET_PP_CORE_VERSION = _MALI_UK_GET_CORE_VERSION, /**< _mali_ukk_get_pp_core_version() */ + _MALI_UK_PP_DISABLE_WB, /**< _mali_ukk_pp_job_disable_wb() */ + _MALI_UK_PP_AND_GP_START_JOB, /**< _mali_ukk_pp_and_gp_start_job() */ + + /** Vertex Processor Functions */ + + _MALI_UK_GP_START_JOB = _MALI_UK_START_JOB, /**< _mali_ukk_gp_start_job() */ + _MALI_UK_GET_GP_NUMBER_OF_CORES = _MALI_UK_GET_NUMBER_OF_CORES, /**< _mali_ukk_get_gp_number_of_cores() */ + _MALI_UK_GET_GP_CORE_VERSION = _MALI_UK_GET_CORE_VERSION, /**< _mali_ukk_get_gp_core_version() */ + _MALI_UK_GP_SUSPEND_RESPONSE, /**< _mali_ukk_gp_suspend_response() */ + + /** Profiling functions */ + + _MALI_UK_PROFILING_ADD_EVENT = 0, /**< __mali_uku_profiling_add_event() */ + _MALI_UK_PROFILING_REPORT_SW_COUNTERS,/**< __mali_uku_profiling_report_sw_counters() */ + _MALI_UK_PROFILING_MEMORY_USAGE_GET, /**< __mali_uku_profiling_memory_usage_get() */ + _MALI_UK_PROFILING_STREAM_FD_GET, /** < __mali_uku_profiling_stream_fd_get() */ + _MALI_UK_PROFILING_CONTROL_SET, /** < __mali_uku_profiling_control_set() */ + + /** VSYNC reporting fuctions */ + _MALI_UK_VSYNC_EVENT_REPORT = 0, /**< _mali_ukk_vsync_event_report() */ +} _mali_uk_functions; + +/** @defgroup _mali_uk_getsysteminfo U/K Get System Info + * @{ */ + +/** + * Type definition for the core version number. + * Used when returning the version number read from a core + * + * Its format is that of the 32-bit Version register for a particular core. + * Refer to the "Mali200 and MaliGP2 3D Graphics Processor Technical Reference + * Manual", ARM DDI 0415C, for more information. + */ +typedef u32 _mali_core_version; + +/** @} */ /* end group _mali_uk_core */ + + +/** @defgroup _mali_uk_gp U/K Vertex Processor + * @{ */ + +/** @defgroup _mali_uk_gp_suspend_response_s Vertex Processor Suspend Response + * @{ */ + +/** @brief Arguments for _mali_ukk_gp_suspend_response() + * + * When _mali_wait_for_notification() receives notification that a + * Vertex Processor job was suspended, you need to send a response to indicate + * what needs to happen with this job. You can either abort or resume the job. + * + * - set @c code to indicate response code. This is either @c _MALIGP_JOB_ABORT or + * @c _MALIGP_JOB_RESUME_WITH_NEW_HEAP to indicate you will provide a new heap + * for the job that will resolve the out of memory condition for the job. + * - copy the @c cookie value from the @c _mali_uk_gp_job_suspended_s notification; + * this is an identifier for the suspended job + * - set @c arguments[0] and @c arguments[1] to zero if you abort the job. If + * you resume it, @c argument[0] should specify the Mali start address for the new + * heap and @c argument[1] the Mali end address of the heap. + * - pass in the user-kernel context @c ctx that was returned from _mali_ukk_open() + * + */ +typedef enum _maligp_job_suspended_response_code { + _MALIGP_JOB_ABORT, /**< Abort the Vertex Processor job */ + _MALIGP_JOB_RESUME_WITH_NEW_HEAP /**< Resume the Vertex Processor job with a new heap */ +} _maligp_job_suspended_response_code; + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 cookie; /**< [in] cookie from the _mali_uk_gp_job_suspended_s notification */ + _maligp_job_suspended_response_code code; /**< [in] abort or resume response code, see \ref _maligp_job_suspended_response_code */ + u32 arguments[2]; /**< [in] 0 when aborting a job. When resuming a job, the Mali start and end address for a new heap to resume the job with */ +} _mali_uk_gp_suspend_response_s; + +/** @} */ /* end group _mali_uk_gp_suspend_response_s */ + +/** @defgroup _mali_uk_gpstartjob_s Vertex Processor Start Job + * @{ */ + +/** @brief Status indicating the result of the execution of a Vertex or Fragment processor job */ +typedef enum { + _MALI_UK_JOB_STATUS_END_SUCCESS = 1 << (16 + 0), + _MALI_UK_JOB_STATUS_END_OOM = 1 << (16 + 1), + _MALI_UK_JOB_STATUS_END_ABORT = 1 << (16 + 2), + _MALI_UK_JOB_STATUS_END_TIMEOUT_SW = 1 << (16 + 3), + _MALI_UK_JOB_STATUS_END_HANG = 1 << (16 + 4), + _MALI_UK_JOB_STATUS_END_SEG_FAULT = 1 << (16 + 5), + _MALI_UK_JOB_STATUS_END_ILLEGAL_JOB = 1 << (16 + 6), + _MALI_UK_JOB_STATUS_END_UNKNOWN_ERR = 1 << (16 + 7), + _MALI_UK_JOB_STATUS_END_SHUTDOWN = 1 << (16 + 8), + _MALI_UK_JOB_STATUS_END_SYSTEM_UNUSABLE = 1 << (16 + 9) +} _mali_uk_job_status; + +#define MALIGP2_NUM_REGS_FRAME (6) + +/** @brief Arguments for _mali_ukk_gp_start_job() + * + * To start a Vertex Processor job + * - associate the request with a reference to a @c mali_gp_job_info by setting + * user_job_ptr to the address of the @c mali_gp_job_info of the job. + * - set @c priority to the priority of the @c mali_gp_job_info + * - specify a timeout for the job by setting @c watchdog_msecs to the number of + * milliseconds the job is allowed to run. Specifying a value of 0 selects the + * default timeout in use by the device driver. + * - copy the frame registers from the @c mali_gp_job_info into @c frame_registers. + * - set the @c perf_counter_flag, @c perf_counter_src0 and @c perf_counter_src1 to zero + * for a non-instrumented build. For an instrumented build you can use up + * to two performance counters. Set the corresponding bit in @c perf_counter_flag + * to enable them. @c perf_counter_src0 and @c perf_counter_src1 specify + * the source of what needs to get counted (e.g. number of vertex loader + * cache hits). For source id values, see ARM DDI0415A, Table 3-60. + * - pass in the user-kernel context @c ctx that was returned from _mali_ukk_open() + * + * When @c _mali_ukk_gp_start_job() returns @c _MALI_OSK_ERR_OK, status contains the + * result of the request (see \ref _mali_uk_start_job_status). If the job could + * not get started (@c _MALI_UK_START_JOB_NOT_STARTED_DO_REQUEUE) it should be + * tried again. + * + * After the job has started, @c _mali_wait_for_notification() will be notified + * that the job finished or got suspended. It may get suspended due to + * resource shortage. If it finished (see _mali_ukk_wait_for_notification()) + * the notification will contain a @c _mali_uk_gp_job_finished_s result. If + * it got suspended the notification will contain a @c _mali_uk_gp_job_suspended_s + * result. + * + * The @c _mali_uk_gp_job_finished_s contains the job status (see \ref _mali_uk_job_status), + * the number of milliseconds the job took to render, and values of core registers + * when the job finished (irq status, performance counters, renderer list + * address). A job has finished succesfully when its status is + * @c _MALI_UK_JOB_STATUS_FINISHED. If the hardware detected a timeout while rendering + * the job, or software detected the job is taking more than watchdog_msecs to + * complete, the status will indicate @c _MALI_UK_JOB_STATUS_HANG. + * If the hardware detected a bus error while accessing memory associated with the + * job, status will indicate @c _MALI_UK_JOB_STATUS_SEG_FAULT. + * status will indicate @c _MALI_UK_JOB_STATUS_NOT_STARTED if the driver had to + * stop the job but the job didn't start on the hardware yet, e.g. when the + * driver shutdown. + * + * In case the job got suspended, @c _mali_uk_gp_job_suspended_s contains + * the @c user_job_ptr identifier used to start the job with, the @c reason + * why the job stalled (see \ref _maligp_job_suspended_reason) and a @c cookie + * to identify the core on which the job stalled. This @c cookie will be needed + * when responding to this nofication by means of _mali_ukk_gp_suspend_response(). + * (see _mali_ukk_gp_suspend_response()). The response is either to abort or + * resume the job. If the job got suspended due to an out of memory condition + * you may be able to resolve this by providing more memory and resuming the job. + * + */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u64 user_job_ptr; /**< [in] identifier for the job in user space, a @c mali_gp_job_info* */ + u32 priority; /**< [in] job priority. A lower number means higher priority */ + u32 frame_registers[MALIGP2_NUM_REGS_FRAME]; /**< [in] core specific registers associated with this job */ + u32 heap_grow_size; /** <[in] the grow size of the plbu heap when out of memory */ + u32 perf_counter_flag; /**< [in] bitmask indicating which performance counters to enable, see \ref _MALI_PERFORMANCE_COUNTER_FLAG_SRC0_ENABLE and related macro definitions */ + u32 perf_counter_src0; /**< [in] source id for performance counter 0 (see ARM DDI0415A, Table 3-60) */ + u32 perf_counter_src1; /**< [in] source id for performance counter 1 (see ARM DDI0415A, Table 3-60) */ + u32 frame_builder_id; /**< [in] id of the originating frame builder */ + u32 flush_id; /**< [in] flush id within the originating frame builder */ + _mali_uk_fence_t fence; /**< [in] fence this job must wait on */ + u64 timeline_point_ptr; /**< [in,out] pointer to u32: location where point on gp timeline for this job will be written */ + u32 varying_memsize; /** < [in] size of varying memory to use deffer bind*/ + u32 varying_alloc_num; + u64 varying_alloc_list; /** < [in] memory hanlde list of varying buffer to use deffer bind */ +} _mali_uk_gp_start_job_s; + +#define _MALI_PERFORMANCE_COUNTER_FLAG_SRC0_ENABLE (1<<0) /**< Enable performance counter SRC0 for a job */ +#define _MALI_PERFORMANCE_COUNTER_FLAG_SRC1_ENABLE (1<<1) /**< Enable performance counter SRC1 for a job */ +#define _MALI_PERFORMANCE_COUNTER_FLAG_HEATMAP_ENABLE (1<<2) /**< Enable per tile (aka heatmap) generation with for a job (using the enabled counter sources) */ + +/** @} */ /* end group _mali_uk_gpstartjob_s */ + +typedef struct { + u64 user_job_ptr; /**< [out] identifier for the job in user space */ + _mali_uk_job_status status; /**< [out] status of finished job */ + u32 heap_current_addr; /**< [out] value of the GP PLB PL heap start address register */ + u32 perf_counter0; /**< [out] value of performance counter 0 (see ARM DDI0415A) */ + u32 perf_counter1; /**< [out] value of performance counter 1 (see ARM DDI0415A) */ + u32 pending_big_job_num; +} _mali_uk_gp_job_finished_s; + +typedef struct { + u64 user_job_ptr; /**< [out] identifier for the job in user space */ + u32 cookie; /**< [out] identifier for the core in kernel space on which the job stalled */ + u32 heap_added_size; +} _mali_uk_gp_job_suspended_s; + +/** @} */ /* end group _mali_uk_gp */ + + +/** @defgroup _mali_uk_pp U/K Fragment Processor + * @{ */ + +#define _MALI_PP_MAX_SUB_JOBS 8 + +#define _MALI_PP_MAX_FRAME_REGISTERS ((0x058/4)+1) + +#define _MALI_PP_MAX_WB_REGISTERS ((0x02C/4)+1) + +#define _MALI_DLBU_MAX_REGISTERS 4 + +/** Flag for _mali_uk_pp_start_job_s */ +#define _MALI_PP_JOB_FLAG_NO_NOTIFICATION (1<<0) +#define _MALI_PP_JOB_FLAG_IS_WINDOW_SURFACE (1<<1) + +/** @defgroup _mali_uk_ppstartjob_s Fragment Processor Start Job + * @{ */ + +/** @brief Arguments for _mali_ukk_pp_start_job() + * + * To start a Fragment Processor job + * - associate the request with a reference to a mali_pp_job by setting + * @c user_job_ptr to the address of the @c mali_pp_job of the job. + * - set @c priority to the priority of the mali_pp_job + * - specify a timeout for the job by setting @c watchdog_msecs to the number of + * milliseconds the job is allowed to run. Specifying a value of 0 selects the + * default timeout in use by the device driver. + * - copy the frame registers from the @c mali_pp_job into @c frame_registers. + * For MALI200 you also need to copy the write back 0,1 and 2 registers. + * - set the @c perf_counter_flag, @c perf_counter_src0 and @c perf_counter_src1 to zero + * for a non-instrumented build. For an instrumented build you can use up + * to two performance counters. Set the corresponding bit in @c perf_counter_flag + * to enable them. @c perf_counter_src0 and @c perf_counter_src1 specify + * the source of what needs to get counted (e.g. number of vertex loader + * cache hits). For source id values, see ARM DDI0415A, Table 3-60. + * - pass in the user-kernel context in @c ctx that was returned from _mali_ukk_open() + * + * When _mali_ukk_pp_start_job() returns @c _MALI_OSK_ERR_OK, @c status contains the + * result of the request (see \ref _mali_uk_start_job_status). If the job could + * not get started (@c _MALI_UK_START_JOB_NOT_STARTED_DO_REQUEUE) it should be + * tried again. + * + * After the job has started, _mali_wait_for_notification() will be notified + * when the job finished. The notification will contain a + * @c _mali_uk_pp_job_finished_s result. It contains the @c user_job_ptr + * identifier used to start the job with, the job @c status (see \ref _mali_uk_job_status), + * the number of milliseconds the job took to render, and values of core registers + * when the job finished (irq status, performance counters, renderer list + * address). A job has finished succesfully when its status is + * @c _MALI_UK_JOB_STATUS_FINISHED. If the hardware detected a timeout while rendering + * the job, or software detected the job is taking more than @c watchdog_msecs to + * complete, the status will indicate @c _MALI_UK_JOB_STATUS_HANG. + * If the hardware detected a bus error while accessing memory associated with the + * job, status will indicate @c _MALI_UK_JOB_STATUS_SEG_FAULT. + * status will indicate @c _MALI_UK_JOB_STATUS_NOT_STARTED if the driver had to + * stop the job but the job didn't start on the hardware yet, e.g. when the + * driver shutdown. + * + */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u64 user_job_ptr; /**< [in] identifier for the job in user space */ + u32 priority; /**< [in] job priority. A lower number means higher priority */ + u32 frame_registers[_MALI_PP_MAX_FRAME_REGISTERS]; /**< [in] core specific registers associated with first sub job, see ARM DDI0415A */ + u32 frame_registers_addr_frame[_MALI_PP_MAX_SUB_JOBS - 1]; /**< [in] ADDR_FRAME registers for sub job 1-7 */ + u32 frame_registers_addr_stack[_MALI_PP_MAX_SUB_JOBS - 1]; /**< [in] ADDR_STACK registers for sub job 1-7 */ + u32 wb0_registers[_MALI_PP_MAX_WB_REGISTERS]; + u32 wb1_registers[_MALI_PP_MAX_WB_REGISTERS]; + u32 wb2_registers[_MALI_PP_MAX_WB_REGISTERS]; + u32 dlbu_registers[_MALI_DLBU_MAX_REGISTERS]; /**< [in] Dynamic load balancing unit registers */ + u32 num_cores; /**< [in] Number of cores to set up (valid range: 1-8(M450) or 4(M400)) */ + u32 perf_counter_flag; /**< [in] bitmask indicating which performance counters to enable, see \ref _MALI_PERFORMANCE_COUNTER_FLAG_SRC0_ENABLE and related macro definitions */ + u32 perf_counter_src0; /**< [in] source id for performance counter 0 (see ARM DDI0415A, Table 3-60) */ + u32 perf_counter_src1; /**< [in] source id for performance counter 1 (see ARM DDI0415A, Table 3-60) */ + u32 frame_builder_id; /**< [in] id of the originating frame builder */ + u32 flush_id; /**< [in] flush id within the originating frame builder */ + u32 flags; /**< [in] See _MALI_PP_JOB_FLAG_* for a list of avaiable flags */ + u32 tilesx; /**< [in] number of tiles in the x direction (needed for heatmap generation */ + u32 tilesy; /**< [in] number of tiles in y direction (needed for reading the heatmap memory) */ + u32 heatmap_mem; /**< [in] memory address to store counter values per tile (aka heatmap) */ + u32 num_memory_cookies; /**< [in] number of memory cookies attached to job */ + u64 memory_cookies; /**< [in] pointer to array of u32 memory cookies attached to job */ + _mali_uk_fence_t fence; /**< [in] fence this job must wait on */ + u64 timeline_point_ptr; /**< [in,out] pointer to location of u32 where point on pp timeline for this job will be written */ +} _mali_uk_pp_start_job_s; + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u64 gp_args; /**< [in,out] GP uk arguments (see _mali_uk_gp_start_job_s) */ + u64 pp_args; /**< [in,out] PP uk arguments (see _mali_uk_pp_start_job_s) */ +} _mali_uk_pp_and_gp_start_job_s; + +/** @} */ /* end group _mali_uk_ppstartjob_s */ + +typedef struct { + u64 user_job_ptr; /**< [out] identifier for the job in user space */ + _mali_uk_job_status status; /**< [out] status of finished job */ + u32 perf_counter0[_MALI_PP_MAX_SUB_JOBS]; /**< [out] value of perfomance counter 0 (see ARM DDI0415A), one for each sub job */ + u32 perf_counter1[_MALI_PP_MAX_SUB_JOBS]; /**< [out] value of perfomance counter 1 (see ARM DDI0415A), one for each sub job */ + u32 perf_counter_src0; + u32 perf_counter_src1; +} _mali_uk_pp_job_finished_s; + +typedef struct { + u32 number_of_enabled_cores; /**< [out] the new number of enabled cores */ +} _mali_uk_pp_num_cores_changed_s; + + + +/** + * Flags to indicate write-back units + */ +typedef enum { + _MALI_UK_PP_JOB_WB0 = 1, + _MALI_UK_PP_JOB_WB1 = 2, + _MALI_UK_PP_JOB_WB2 = 4, +} _mali_uk_pp_job_wbx_flag; + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 fb_id; /**< [in] Frame builder ID of job to disable WB units for */ + u32 wb0_memory; + u32 wb1_memory; + u32 wb2_memory; +} _mali_uk_pp_disable_wb_s; + + +/** @} */ /* end group _mali_uk_pp */ + +/** @defgroup _mali_uk_soft_job U/K Soft Job + * @{ */ + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u64 user_job; /**< [in] identifier for the job in user space */ + u64 job_id_ptr; /**< [in,out] pointer to location of u32 where job id will be written */ + _mali_uk_fence_t fence; /**< [in] fence this job must wait on */ + u32 point; /**< [out] point on soft timeline for this job */ + u32 type; /**< [in] type of soft job */ +} _mali_uk_soft_job_start_s; + +typedef struct { + u64 user_job; /**< [out] identifier for the job in user space */ +} _mali_uk_soft_job_activated_s; + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 job_id; /**< [in] id for soft job */ +} _mali_uk_soft_job_signal_s; + +/** @} */ /* end group _mali_uk_soft_job */ + +typedef struct { + u32 counter_id; + u32 key; + int enable; +} _mali_uk_annotate_profiling_mem_counter_s; + +typedef struct { + u32 sampling_rate; + int enable; +} _mali_uk_annotate_profiling_enable_s; + + +/** @addtogroup _mali_uk_core U/K Core + * @{ */ + +/** @defgroup _mali_uk_waitfornotification_s Wait For Notification + * @{ */ + +/** @brief Notification type encodings + * + * Each Notification type is an ordered pair of (subsystem,id), and is unique. + * + * The encoding of subsystem,id into a 32-bit word is: + * encoding = (( subsystem << _MALI_NOTIFICATION_SUBSYSTEM_SHIFT ) & _MALI_NOTIFICATION_SUBSYSTEM_MASK) + * | (( id << _MALI_NOTIFICATION_ID_SHIFT ) & _MALI_NOTIFICATION_ID_MASK) + * + * @see _mali_uk_wait_for_notification_s + */ +typedef enum { + /** core notifications */ + + _MALI_NOTIFICATION_CORE_SHUTDOWN_IN_PROGRESS = (_MALI_UK_CORE_SUBSYSTEM << 16) | 0x20, + _MALI_NOTIFICATION_APPLICATION_QUIT = (_MALI_UK_CORE_SUBSYSTEM << 16) | 0x40, + _MALI_NOTIFICATION_SETTINGS_CHANGED = (_MALI_UK_CORE_SUBSYSTEM << 16) | 0x80, + _MALI_NOTIFICATION_SOFT_ACTIVATED = (_MALI_UK_CORE_SUBSYSTEM << 16) | 0x100, + + /** Fragment Processor notifications */ + + _MALI_NOTIFICATION_PP_FINISHED = (_MALI_UK_PP_SUBSYSTEM << 16) | 0x10, + _MALI_NOTIFICATION_PP_NUM_CORE_CHANGE = (_MALI_UK_PP_SUBSYSTEM << 16) | 0x20, + + /** Vertex Processor notifications */ + + _MALI_NOTIFICATION_GP_FINISHED = (_MALI_UK_GP_SUBSYSTEM << 16) | 0x10, + _MALI_NOTIFICATION_GP_STALLED = (_MALI_UK_GP_SUBSYSTEM << 16) | 0x20, + + /** Profiling notifications */ + _MALI_NOTIFICATION_ANNOTATE_PROFILING_MEM_COUNTER = (_MALI_UK_PROFILING_SUBSYSTEM << 16) | 0x10, + _MALI_NOTIFICATION_ANNOTATE_PROFILING_ENABLE = (_MALI_UK_PROFILING_SUBSYSTEM << 16) | 0x20, +} _mali_uk_notification_type; + +/** to assist in splitting up 32-bit notification value in subsystem and id value */ +#define _MALI_NOTIFICATION_SUBSYSTEM_MASK 0xFFFF0000 +#define _MALI_NOTIFICATION_SUBSYSTEM_SHIFT 16 +#define _MALI_NOTIFICATION_ID_MASK 0x0000FFFF +#define _MALI_NOTIFICATION_ID_SHIFT 0 + + +/** @brief Enumeration of possible settings which match mali_setting_t in user space + * + * + */ +typedef enum { + _MALI_UK_USER_SETTING_SW_EVENTS_ENABLE = 0, + _MALI_UK_USER_SETTING_COLORBUFFER_CAPTURE_ENABLED, + _MALI_UK_USER_SETTING_DEPTHBUFFER_CAPTURE_ENABLED, + _MALI_UK_USER_SETTING_STENCILBUFFER_CAPTURE_ENABLED, + _MALI_UK_USER_SETTING_PER_TILE_COUNTERS_CAPTURE_ENABLED, + _MALI_UK_USER_SETTING_BUFFER_CAPTURE_COMPOSITOR, + _MALI_UK_USER_SETTING_BUFFER_CAPTURE_WINDOW, + _MALI_UK_USER_SETTING_BUFFER_CAPTURE_OTHER, + _MALI_UK_USER_SETTING_BUFFER_CAPTURE_N_FRAMES, + _MALI_UK_USER_SETTING_BUFFER_CAPTURE_RESIZE_FACTOR, + _MALI_UK_USER_SETTING_SW_COUNTER_ENABLED, + _MALI_UK_USER_SETTING_MAX, +} _mali_uk_user_setting_t; + +/* See mali_user_settings_db.c */ +extern const char *_mali_uk_user_setting_descriptions[]; +#define _MALI_UK_USER_SETTING_DESCRIPTIONS \ + { \ + "sw_events_enable", \ + "colorbuffer_capture_enable", \ + "depthbuffer_capture_enable", \ + "stencilbuffer_capture_enable", \ + "per_tile_counters_enable", \ + "buffer_capture_compositor", \ + "buffer_capture_window", \ + "buffer_capture_other", \ + "buffer_capture_n_frames", \ + "buffer_capture_resize_factor", \ + "sw_counters_enable", \ + }; + +/** @brief struct to hold the value to a particular setting as seen in the kernel space + */ +typedef struct { + _mali_uk_user_setting_t setting; + u32 value; +} _mali_uk_settings_changed_s; + +/** @brief Arguments for _mali_ukk_wait_for_notification() + * + * On successful return from _mali_ukk_wait_for_notification(), the members of + * this structure will indicate the reason for notification. + * + * Specifically, the source of the notification can be identified by the + * subsystem and id fields of the mali_uk_notification_type in the code.type + * member. The type member is encoded in a way to divide up the types into a + * subsystem field, and a per-subsystem ID field. See + * _mali_uk_notification_type for more information. + * + * Interpreting the data union member depends on the notification type: + * + * - type == _MALI_NOTIFICATION_CORE_SHUTDOWN_IN_PROGRESS + * - The kernel side is shutting down. No further + * _mali_uk_wait_for_notification() calls should be made. + * - In this case, the value of the data union member is undefined. + * - This is used to indicate to the user space client that it should close + * the connection to the Mali Device Driver. + * - type == _MALI_NOTIFICATION_PP_FINISHED + * - The notification data is of type _mali_uk_pp_job_finished_s. It contains the user_job_ptr + * identifier used to start the job with, the job status, the number of milliseconds the job took to render, + * and values of core registers when the job finished (irq status, performance counters, renderer list + * address). + * - A job has finished succesfully when its status member is _MALI_UK_JOB_STATUS_FINISHED. + * - If the hardware detected a timeout while rendering the job, or software detected the job is + * taking more than watchdog_msecs (see _mali_ukk_pp_start_job()) to complete, the status member will + * indicate _MALI_UK_JOB_STATUS_HANG. + * - If the hardware detected a bus error while accessing memory associated with the job, status will + * indicate _MALI_UK_JOB_STATUS_SEG_FAULT. + * - Status will indicate MALI_UK_JOB_STATUS_NOT_STARTED if the driver had to stop the job but the job + * didn't start the hardware yet, e.g. when the driver closes. + * - type == _MALI_NOTIFICATION_GP_FINISHED + * - The notification data is of type _mali_uk_gp_job_finished_s. The notification is similar to that of + * type == _MALI_NOTIFICATION_PP_FINISHED, except that several other GP core register values are returned. + * The status values have the same meaning for type == _MALI_NOTIFICATION_PP_FINISHED. + * - type == _MALI_NOTIFICATION_GP_STALLED + * - The nofication data is of type _mali_uk_gp_job_suspended_s. It contains the user_job_ptr + * identifier used to start the job with, the reason why the job stalled and a cookie to identify the core on + * which the job stalled. + * - The reason member of gp_job_suspended is set to _MALIGP_JOB_SUSPENDED_OUT_OF_MEMORY + * when the polygon list builder unit has run out of memory. + */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + _mali_uk_notification_type type; /**< [out] Type of notification available */ + union { + _mali_uk_gp_job_suspended_s gp_job_suspended;/**< [out] Notification data for _MALI_NOTIFICATION_GP_STALLED notification type */ + _mali_uk_gp_job_finished_s gp_job_finished; /**< [out] Notification data for _MALI_NOTIFICATION_GP_FINISHED notification type */ + _mali_uk_pp_job_finished_s pp_job_finished; /**< [out] Notification data for _MALI_NOTIFICATION_PP_FINISHED notification type */ + _mali_uk_settings_changed_s setting_changed;/**< [out] Notification data for _MALI_NOTIFICAATION_SETTINGS_CHANGED notification type */ + _mali_uk_soft_job_activated_s soft_job_activated; /**< [out] Notification data for _MALI_NOTIFICATION_SOFT_ACTIVATED notification type */ + _mali_uk_annotate_profiling_mem_counter_s profiling_mem_counter; + _mali_uk_annotate_profiling_enable_s profiling_enable; + } data; +} _mali_uk_wait_for_notification_s; + +/** @brief Arguments for _mali_ukk_post_notification() + * + * Posts the specified notification to the notification queue for this application. + * This is used to send a quit message to the callback thread. + */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + _mali_uk_notification_type type; /**< [in] Type of notification to post */ +} _mali_uk_post_notification_s; + +/** @} */ /* end group _mali_uk_waitfornotification_s */ + +/** @defgroup _mali_uk_getapiversion_s Get API Version + * @{ */ + +/** helpers for Device Driver API version handling */ + +/** @brief Encode a version ID from a 16-bit input + * + * @note the input is assumed to be 16 bits. It must not exceed 16 bits. */ +#define _MAKE_VERSION_ID(x) (((x) << 16UL) | (x)) + +/** @brief Check whether a 32-bit value is likely to be Device Driver API + * version ID. */ +#define _IS_VERSION_ID(x) (((x) & 0xFFFF) == (((x) >> 16UL) & 0xFFFF)) + +/** @brief Decode a 16-bit version number from a 32-bit Device Driver API version + * ID */ +#define _GET_VERSION(x) (((x) >> 16UL) & 0xFFFF) + +/** @brief Determine whether two 32-bit encoded version IDs match */ +#define _IS_API_MATCH(x, y) (IS_VERSION_ID((x)) && IS_VERSION_ID((y)) && (GET_VERSION((x)) == GET_VERSION((y)))) + +/** + * API version define. + * Indicates the version of the kernel API + * The version is a 16bit integer incremented on each API change. + * The 16bit integer is stored twice in a 32bit integer + * For example, for version 1 the value would be 0x00010001 + */ +#define _MALI_API_VERSION 800 +#define _MALI_UK_API_VERSION _MAKE_VERSION_ID(_MALI_API_VERSION) + +/** + * The API version is a 16-bit integer stored in both the lower and upper 16-bits + * of a 32-bit value. The 16-bit API version value is incremented on each API + * change. Version 1 would be 0x00010001. Used in _mali_uk_get_api_version_s. + */ +typedef u32 _mali_uk_api_version; + +/** @brief Arguments for _mali_uk_get_api_version() + * + * The user-side interface version must be written into the version member, + * encoded using _MAKE_VERSION_ID(). It will be compared to the API version of + * the kernel-side interface. + * + * On successful return, the version member will be the API version of the + * kernel-side interface. _MALI_UK_API_VERSION macro defines the current version + * of the API. + * + * The compatible member must be checked to see if the version of the user-side + * interface is compatible with the kernel-side interface, since future versions + * of the interface may be backwards compatible. + */ +typedef struct { + u32 ctx; /**< [in,out] user-kernel context (trashed on output) */ + _mali_uk_api_version version; /**< [in,out] API version of user-side interface. */ + int compatible; /**< [out] @c 1 when @version is compatible, @c 0 otherwise */ +} _mali_uk_get_api_version_s; + +/** @brief Arguments for _mali_uk_get_api_version_v2() + * + * The user-side interface version must be written into the version member, + * encoded using _MAKE_VERSION_ID(). It will be compared to the API version of + * the kernel-side interface. + * + * On successful return, the version member will be the API version of the + * kernel-side interface. _MALI_UK_API_VERSION macro defines the current version + * of the API. + * + * The compatible member must be checked to see if the version of the user-side + * interface is compatible with the kernel-side interface, since future versions + * of the interface may be backwards compatible. + */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + _mali_uk_api_version version; /**< [in,out] API version of user-side interface. */ + int compatible; /**< [out] @c 1 when @version is compatible, @c 0 otherwise */ +} _mali_uk_get_api_version_v2_s; + +/** @} */ /* end group _mali_uk_getapiversion_s */ + +/** @defgroup _mali_uk_get_user_settings_s Get user space settings */ + +/** @brief struct to keep the matching values of the user space settings within certain context + * + * Each member of the settings array corresponds to a matching setting in the user space and its value is the value + * of that particular setting. + * + * All settings are given reference to the context pointed to by the ctx pointer. + * + */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 settings[_MALI_UK_USER_SETTING_MAX]; /**< [out] The values for all settings */ +} _mali_uk_get_user_settings_s; + +/** @brief struct to hold the value of a particular setting from the user space within a given context + */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + _mali_uk_user_setting_t setting; /**< [in] setting to get */ + u32 value; /**< [out] value of setting */ +} _mali_uk_get_user_setting_s; + +/** @brief Arguments for _mali_ukk_request_high_priority() */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ +} _mali_uk_request_high_priority_s; + +/** @brief Arguments for _mali_ukk_pending_submit() */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ +} _mali_uk_pending_submit_s; + +/** @} */ /* end group _mali_uk_core */ + + +/** @defgroup _mali_uk_memory U/K Memory + * @{ */ + +#define _MALI_MEMORY_ALLOCATE_RESIZEABLE (1<<4) /* BUFFER can trim dow/grow*/ +#define _MALI_MEMORY_ALLOCATE_NO_BIND_GPU (1<<5) /*Not map to GPU when allocate, must call bind later*/ +#define _MALI_MEMORY_ALLOCATE_SWAPPABLE (1<<6) /* Allocate swappale memory. */ +#define _MALI_MEMORY_ALLOCATE_DEFER_BIND (1<<7) /*Not map to GPU when allocate, must call bind later*/ + + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 gpu_vaddr; /**< [in] GPU virtual address */ + u32 vsize; /**< [in] vitrual size of the allocation */ + u32 psize; /**< [in] physical size of the allocation */ + u32 flags; + u64 backend_handle; /**< [out] backend handle */ + struct { + /* buffer types*/ + /* CPU read/write info*/ + } buffer_info; +} _mali_uk_alloc_mem_s; + + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 gpu_vaddr; /**< [in] use as handle to free allocation */ + u32 free_pages_nr; /** < [out] record the number of free pages */ +} _mali_uk_free_mem_s; + + +#define _MALI_MEMORY_BIND_BACKEND_UMP (1<<8) +#define _MALI_MEMORY_BIND_BACKEND_DMA_BUF (1<<9) +#define _MALI_MEMORY_BIND_BACKEND_MALI_MEMORY (1<<10) +#define _MALI_MEMORY_BIND_BACKEND_EXTERNAL_MEMORY (1<<11) +#define _MALI_MEMORY_BIND_BACKEND_EXT_COW (1<<12) +#define _MALI_MEMORY_BIND_BACKEND_HAVE_ALLOCATION (1<<13) + + +#define _MALI_MEMORY_BIND_BACKEND_MASK (_MALI_MEMORY_BIND_BACKEND_UMP| \ + _MALI_MEMORY_BIND_BACKEND_DMA_BUF |\ + _MALI_MEMORY_BIND_BACKEND_MALI_MEMORY |\ + _MALI_MEMORY_BIND_BACKEND_EXTERNAL_MEMORY |\ + _MALI_MEMORY_BIND_BACKEND_EXT_COW |\ + _MALI_MEMORY_BIND_BACKEND_HAVE_ALLOCATION) + + +#define _MALI_MEMORY_GPU_READ_ALLOCATE (1<<16) + + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 vaddr; /**< [in] mali address to map the physical memory to */ + u32 size; /**< [in] size */ + u32 flags; /**< [in] see_MALI_MEMORY_BIND_BACKEND_* */ + u32 padding; /** padding for 32/64 struct alignment */ + union { + struct { + u32 secure_id; /**< [in] secure id */ + u32 rights; /**< [in] rights necessary for accessing memory */ + u32 flags; /**< [in] flags, see \ref _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE */ + } bind_ump; + struct { + u32 mem_fd; /**< [in] Memory descriptor */ + u32 rights; /**< [in] rights necessary for accessing memory */ + u32 flags; /**< [in] flags, see \ref _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE */ + } bind_dma_buf; + struct { + /**/ + } bind_mali_memory; + struct { + u32 phys_addr; /**< [in] physical address */ + u32 rights; /**< [in] rights necessary for accessing memory */ + u32 flags; /**< [in] flags, see \ref _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE */ + } bind_ext_memory; + } mem_union; +} _mali_uk_bind_mem_s; + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 flags; /**< [in] see_MALI_MEMORY_BIND_BACKEND_* */ + u32 vaddr; /**< [in] identifier for mapped memory object in kernel space */ +} _mali_uk_unbind_mem_s; + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 target_handle; /**< [in] handle of allocation need to do COW */ + u32 target_offset; /**< [in] offset in target allocation to do COW(for support COW a memory allocated from memory_bank, PAGE_SIZE align)*/ + u32 target_size; /**< [in] size of target allocation to do COW (for support memory bank, PAGE_SIZE align)(in byte) */ + u32 range_start; /**< [in] re allocate range start offset, offset from the start of allocation (PAGE_SIZE align)*/ + u32 range_size; /**< [in] re allocate size (PAGE_SIZE align)*/ + u32 vaddr; /**< [in] mali address for the new allocaiton */ + u32 backend_handle; /**< [out] backend handle */ + u32 flags; +} _mali_uk_cow_mem_s; + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 range_start; /**< [in] re allocate range start offset, offset from the start of allocation */ + u32 size; /**< [in] re allocate size*/ + u32 vaddr; /**< [in] mali address for the new allocaiton */ + s32 change_pages_nr; /**< [out] record the page number change for cow operation */ +} _mali_uk_cow_modify_range_s; + + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 mem_fd; /**< [in] Memory descriptor */ + u32 size; /**< [out] size */ +} _mali_uk_dma_buf_get_size_s; + +/** Flag for _mali_uk_map_external_mem_s, _mali_uk_attach_ump_mem_s and _mali_uk_attach_dma_buf_s */ +#define _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE (1<<0) + + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u64 vaddr; /* the buffer to do resize*/ + u32 psize; /* wanted physical size of this memory */ +} _mali_uk_mem_resize_s; + +/** + * @brief Arguments for _mali_uk[uk]_mem_write_safe() + */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u64 src; /**< [in] Pointer to source data */ + u64 dest; /**< [in] Destination Mali buffer */ + u32 size; /**< [in,out] Number of bytes to write/copy on input, number of bytes actually written/copied on output */ +} _mali_uk_mem_write_safe_s; + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 size; /**< [out] size of MMU page table information (registers + page tables) */ +} _mali_uk_query_mmu_page_table_dump_size_s; + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 size; /**< [in] size of buffer to receive mmu page table information */ + u64 buffer; /**< [in,out] buffer to receive mmu page table information */ + u32 register_writes_size; /**< [out] size of MMU register dump */ + u64 register_writes; /**< [out] pointer within buffer where MMU register dump is stored */ + u32 page_table_dump_size; /**< [out] size of MMU page table dump */ + u64 page_table_dump; /**< [out] pointer within buffer where MMU page table dump is stored */ +} _mali_uk_dump_mmu_page_table_s; + +/** @} */ /* end group _mali_uk_memory */ + + +/** @addtogroup _mali_uk_pp U/K Fragment Processor + * @{ */ + +/** @brief Arguments for _mali_ukk_get_pp_number_of_cores() + * + * - pass in the user-kernel context @c ctx that was returned from _mali_ukk_open() + * - Upon successful return from _mali_ukk_get_pp_number_of_cores(), @c number_of_cores + * will contain the number of Fragment Processor cores in the system. + */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 number_of_total_cores; /**< [out] Total number of Fragment Processor cores in the system */ + u32 number_of_enabled_cores; /**< [out] Number of enabled Fragment Processor cores */ +} _mali_uk_get_pp_number_of_cores_s; + +/** @brief Arguments for _mali_ukk_get_pp_core_version() + * + * - pass in the user-kernel context @c ctx that was returned from _mali_ukk_open() + * - Upon successful return from _mali_ukk_get_pp_core_version(), @c version contains + * the version that all Fragment Processor cores are compatible with. + */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + _mali_core_version version; /**< [out] version returned from core, see \ref _mali_core_version */ + u32 padding; +} _mali_uk_get_pp_core_version_s; + +/** @} */ /* end group _mali_uk_pp */ + + +/** @addtogroup _mali_uk_gp U/K Vertex Processor + * @{ */ + +/** @brief Arguments for _mali_ukk_get_gp_number_of_cores() + * + * - pass in the user-kernel context @c ctx that was returned from _mali_ukk_open() + * - Upon successful return from _mali_ukk_get_gp_number_of_cores(), @c number_of_cores + * will contain the number of Vertex Processor cores in the system. + */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 number_of_cores; /**< [out] number of Vertex Processor cores in the system */ +} _mali_uk_get_gp_number_of_cores_s; + +/** @brief Arguments for _mali_ukk_get_gp_core_version() + * + * - pass in the user-kernel context @c ctx that was returned from _mali_ukk_open() + * - Upon successful return from _mali_ukk_get_gp_core_version(), @c version contains + * the version that all Vertex Processor cores are compatible with. + */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + _mali_core_version version; /**< [out] version returned from core, see \ref _mali_core_version */ +} _mali_uk_get_gp_core_version_s; + +/** @} */ /* end group _mali_uk_gp */ + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 event_id; /**< [in] event id to register (see enum mali_profiling_events for values) */ + u32 data[5]; /**< [in] event specific data */ +} _mali_uk_profiling_add_event_s; + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 memory_usage; /**< [out] total memory usage */ + u32 vaddr; /**< [in] mali address for the cow allocaiton */ + s32 change_pages_nr; /**< [out] record the page number change for cow operation */ +} _mali_uk_profiling_memory_usage_get_s; + + +/** @addtogroup _mali_uk_memory U/K Memory + * @{ */ + +/** @brief Arguments to _mali_ukk_mem_mmap() + * + * Use of the phys_addr member depends on whether the driver is compiled for + * Mali-MMU or nonMMU: + * - in the nonMMU case, this is the physical address of the memory as seen by + * the CPU (which may be a constant offset from that used by Mali) + * - in the MMU case, this is the Mali Virtual base address of the memory to + * allocate, and the particular physical pages used to back the memory are + * entirely determined by _mali_ukk_mem_mmap(). The details of the physical pages + * are not reported to user-space for security reasons. + * + * The cookie member must be stored for use later when freeing the memory by + * calling _mali_ukk_mem_munmap(). In the Mali-MMU case, the cookie is secure. + * + * The ukk_private word must be set to zero when calling from user-space. On + * Kernel-side, the OS implementation of the U/K interface can use it to + * communicate data to the OS implementation of the OSK layer. In particular, + * _mali_ukk_get_big_block() directly calls _mali_ukk_mem_mmap directly, and + * will communicate its own ukk_private word through the ukk_private member + * here. The common code itself will not inspect or modify the ukk_private + * word, and so it may be safely used for whatever purposes necessary to + * integrate Mali Memory handling into the OS. + * + * The uku_private member is currently reserved for use by the user-side + * implementation of the U/K interface. Its value must be zero. + */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + void *mapping; /**< [out] Returns user-space virtual address for the mapping */ + u32 size; /**< [in] Size of the requested mapping */ + u32 phys_addr; /**< [in] Physical address - could be offset, depending on caller+callee convention */ + mali_bool writeable; +} _mali_uk_mem_mmap_s; + +/** @brief Arguments to _mali_ukk_mem_munmap() + * + * The cookie and mapping members must be that returned from the same previous + * call to _mali_ukk_mem_mmap(). The size member must correspond to cookie + * and mapping - that is, it must be the value originally supplied to a call to + * _mali_ukk_mem_mmap that returned the values of mapping and cookie. + * + * An error will be returned if an attempt is made to unmap only part of the + * originally obtained range, or to unmap more than was originally obtained. + */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + void *mapping; /**< [in] The mapping returned from mmap call */ + u32 size; /**< [in] The size passed to mmap call */ +} _mali_uk_mem_munmap_s; +/** @} */ /* end group _mali_uk_memory */ + +/** @defgroup _mali_uk_vsync U/K VSYNC Wait Reporting Module + * @{ */ + +/** @brief VSYNC events + * + * These events are reported when DDK starts to wait for vsync and when the + * vsync has occured and the DDK can continue on the next frame. + */ +typedef enum _mali_uk_vsync_event { + _MALI_UK_VSYNC_EVENT_BEGIN_WAIT = 0, + _MALI_UK_VSYNC_EVENT_END_WAIT +} _mali_uk_vsync_event; + +/** @brief Arguments to _mali_ukk_vsync_event() + * + */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + _mali_uk_vsync_event event; /**< [in] VSYNCH event type */ +} _mali_uk_vsync_event_report_s; + +/** @} */ /* end group _mali_uk_vsync */ + +/** @defgroup _mali_uk_sw_counters_report U/K Software Counter Reporting + * @{ */ + +/** @brief Software counter values + * + * Values recorded for each of the software counters during a single renderpass. + */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u64 counters; /**< [in] The array of u32 counter values */ + u32 num_counters; /**< [in] The number of elements in counters array */ +} _mali_uk_sw_counters_report_s; + +/** @} */ /* end group _mali_uk_sw_counters_report */ + +/** @defgroup _mali_uk_timeline U/K Mali Timeline + * @{ */ + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 timeline; /**< [in] timeline id */ + u32 point; /**< [out] latest point on timeline */ +} _mali_uk_timeline_get_latest_point_s; + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + _mali_uk_fence_t fence; /**< [in] fence */ + u32 timeout; /**< [in] timeout (0 for no wait, -1 for blocking) */ + u32 status; /**< [out] status of fence (1 if signaled, 0 if timeout) */ +} _mali_uk_timeline_wait_s; + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + _mali_uk_fence_t fence; /**< [in] mali fence to create linux sync fence from */ + s32 sync_fd; /**< [out] file descriptor for new linux sync fence */ +} _mali_uk_timeline_create_sync_fence_s; + +/** @} */ /* end group _mali_uk_timeline */ + +/** @} */ /* end group u_k_api */ + +/** @} */ /* end group uddapi */ + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + s32 stream_fd; /**< [in] The profiling kernel base stream fd handle */ +} _mali_uk_profiling_stream_fd_get_s; + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u64 control_packet_data; /**< [in] the control packet data for control settings */ + u32 control_packet_size; /**< [in] The control packet size */ + u64 response_packet_data; /** < [out] The response packet data */ + u32 response_packet_size; /** < [in,out] The response packet data */ +} _mali_uk_profiling_control_set_s; + +#ifdef __cplusplus +} +#endif + +#endif /* __MALI_UTGARD_UK_TYPES_H__ */ diff --git a/drivers/gpu/arm/utgard/linux/license/gpl/mali_kernel_license.h b/drivers/gpu/arm/utgard/linux/license/gpl/mali_kernel_license.h new file mode 100644 index 00000000000000..c88c992e859b81 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/license/gpl/mali_kernel_license.h @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2010, 2013, 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_kernel_license.h + * Defines for the macro MODULE_LICENSE. + */ + +#ifndef __MALI_KERNEL_LICENSE_H__ +#define __MALI_KERNEL_LICENSE_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +#define MALI_KERNEL_LINUX_LICENSE "GPL" +#define MALI_LICENSE_IS_GPL 1 + +#ifdef __cplusplus +} +#endif + +#endif /* __MALI_KERNEL_LICENSE_H__ */ diff --git a/drivers/gpu/arm/utgard/linux/mali_device_pause_resume.c b/drivers/gpu/arm/utgard/linux/mali_device_pause_resume.c new file mode 100644 index 00000000000000..37076a2eaf671b --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_device_pause_resume.c @@ -0,0 +1,36 @@ +/** + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_device_pause_resume.c + * Implementation of the Mali pause/resume functionality + */ + +#include +#include +#include "mali_pm.h" + +void mali_dev_pause(void) +{ + /* + * Deactive all groups to prevent hardware being touched + * during the period of mali device pausing + */ + mali_pm_os_suspend(MALI_FALSE); +} + +EXPORT_SYMBOL(mali_dev_pause); + +void mali_dev_resume(void) +{ + mali_pm_os_resume(); +} + +EXPORT_SYMBOL(mali_dev_resume); diff --git a/drivers/gpu/arm/utgard/linux/mali_kernel_linux.c b/drivers/gpu/arm/utgard/linux/mali_kernel_linux.c new file mode 100644 index 00000000000000..34cb3d7ca95a47 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_kernel_linux.c @@ -0,0 +1,941 @@ +/** + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_kernel_linux.c + * Implementation of the Linux device driver entrypoints + */ +#include /* kernel module definitions */ +#include /* file system operations */ +#include /* character device definitions */ +#include /* memory manager definitions */ +#include +#include +#include +#include "mali_kernel_license.h" +#include +#include +#include +#include + +#include +#include "mali_kernel_common.h" +#include "mali_session.h" +#include "mali_kernel_core.h" +#include "mali_osk.h" +#include "mali_kernel_linux.h" +#include "mali_ukk.h" +#include "mali_ukk_wrappers.h" +#include "mali_kernel_sysfs.h" +#include "mali_pm.h" +#include "mali_kernel_license.h" +#include "mali_memory.h" +#include "mali_memory_dma_buf.h" +#include "mali_memory_manager.h" +#include "mali_memory_swap_alloc.h" +#if defined(CONFIG_MALI400_INTERNAL_PROFILING) +#include "mali_profiling_internal.h" +#endif +#if defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_MALI_DVFS) +#include "mali_osk_profiling.h" +#include "mali_dvfs_policy.h" + +static int is_first_resume = 1; +/*Store the clk and vol for boot/insmod and mali_resume*/ +static struct mali_gpu_clk_item mali_gpu_clk[2]; +#endif + +/* Streamline support for the Mali driver */ +#if defined(CONFIG_TRACEPOINTS) && defined(CONFIG_MALI400_PROFILING) +/* Ask Linux to create the tracepoints */ +#define CREATE_TRACE_POINTS +#include "mali_linux_trace.h" + +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_event); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_hw_counter); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_sw_counters); +#endif /* CONFIG_TRACEPOINTS */ + +/* from the __malidrv_build_info.c file that is generated during build */ +extern const char *__malidrv_build_info(void); + +/* Module parameter to control log level */ +int mali_debug_level = 2; +module_param(mali_debug_level, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH); /* rw-rw-r-- */ +MODULE_PARM_DESC(mali_debug_level, "Higher number, more dmesg output"); + +extern int mali_max_job_runtime; +module_param(mali_max_job_runtime, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH); +MODULE_PARM_DESC(mali_max_job_runtime, "Maximum allowed job runtime in msecs.\nJobs will be killed after this no matter what"); + +extern int mali_l2_max_reads; +module_param(mali_l2_max_reads, int, S_IRUSR | S_IRGRP | S_IROTH); +MODULE_PARM_DESC(mali_l2_max_reads, "Maximum reads for Mali L2 cache"); + +extern unsigned int mali_dedicated_mem_start; +module_param(mali_dedicated_mem_start, uint, S_IRUSR | S_IRGRP | S_IROTH); +MODULE_PARM_DESC(mali_dedicated_mem_start, "Physical start address of dedicated Mali GPU memory."); + +extern unsigned int mali_dedicated_mem_size; +module_param(mali_dedicated_mem_size, uint, S_IRUSR | S_IRGRP | S_IROTH); +MODULE_PARM_DESC(mali_dedicated_mem_size, "Size of dedicated Mali GPU memory."); + +extern unsigned int mali_shared_mem_size; +module_param(mali_shared_mem_size, uint, S_IRUSR | S_IRGRP | S_IROTH); +MODULE_PARM_DESC(mali_shared_mem_size, "Size of shared Mali GPU memory."); + +#if defined(CONFIG_MALI400_PROFILING) +extern int mali_boot_profiling; +module_param(mali_boot_profiling, int, S_IRUSR | S_IRGRP | S_IROTH); +MODULE_PARM_DESC(mali_boot_profiling, "Start profiling as a part of Mali driver initialization"); +#endif + +extern int mali_max_pp_cores_group_1; +module_param(mali_max_pp_cores_group_1, int, S_IRUSR | S_IRGRP | S_IROTH); +MODULE_PARM_DESC(mali_max_pp_cores_group_1, "Limit the number of PP cores to use from first PP group."); + +extern int mali_max_pp_cores_group_2; +module_param(mali_max_pp_cores_group_2, int, S_IRUSR | S_IRGRP | S_IROTH); +MODULE_PARM_DESC(mali_max_pp_cores_group_2, "Limit the number of PP cores to use from second PP group (Mali-450 only)."); + +extern unsigned int mali_mem_swap_out_threshold_value; +module_param(mali_mem_swap_out_threshold_value, uint, S_IRUSR | S_IRGRP | S_IROTH); +MODULE_PARM_DESC(mali_mem_swap_out_threshold_value, "Threshold value used to limit how much swappable memory cached in Mali driver."); + +#if defined(CONFIG_MALI_DVFS) +/** the max fps the same as display vsync default 60, can set by module insert parameter */ +extern int mali_max_system_fps; +module_param(mali_max_system_fps, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH); +MODULE_PARM_DESC(mali_max_system_fps, "Max system fps the same as display VSYNC."); + +/** a lower limit on their desired FPS default 58, can set by module insert parameter*/ +extern int mali_desired_fps; +module_param(mali_desired_fps, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH); +MODULE_PARM_DESC(mali_desired_fps, "A bit lower than max_system_fps which user desired fps"); +#endif + +#if MALI_ENABLE_CPU_CYCLES +#include +#include +#include +static struct timer_list mali_init_cpu_clock_timers[8]; +static u32 mali_cpu_clock_last_value[8] = {0,}; +#endif + +/* Export symbols from common code: mali_user_settings.c */ +#include "mali_user_settings_db.h" +EXPORT_SYMBOL(mali_set_user_setting); +EXPORT_SYMBOL(mali_get_user_setting); + +static char mali_dev_name[] = "mali"; /* should be const, but the functions we call requires non-cost */ + +/* This driver only supports one Mali device, and this variable stores this single platform device */ +struct platform_device *mali_platform_device = NULL; + +/* This driver only supports one Mali device, and this variable stores the exposed misc device (/dev/mali) */ +static struct miscdevice mali_miscdevice = { 0, }; + +static int mali_miscdevice_register(struct platform_device *pdev); +static void mali_miscdevice_unregister(void); + +static int mali_open(struct inode *inode, struct file *filp); +static int mali_release(struct inode *inode, struct file *filp); +#ifdef HAVE_UNLOCKED_IOCTL +static long mali_ioctl(struct file *filp, unsigned int cmd, unsigned long arg); +#else +static int mali_ioctl(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg); +#endif + +static int mali_probe(struct platform_device *pdev); +static int mali_remove(struct platform_device *pdev); + +static int mali_driver_suspend_scheduler(struct device *dev); +static int mali_driver_resume_scheduler(struct device *dev); + +#ifdef CONFIG_PM_RUNTIME +static int mali_driver_runtime_suspend(struct device *dev); +static int mali_driver_runtime_resume(struct device *dev); +static int mali_driver_runtime_idle(struct device *dev); +#endif + +#if defined(MALI_FAKE_PLATFORM_DEVICE) +#if defined(CONFIG_MALI_DT) +extern int mali_platform_device_init(struct platform_device *device); +extern int mali_platform_device_deinit(struct platform_device *device); +#else +extern int mali_platform_device_register(void); +extern int mali_platform_device_unregister(void); +#endif +#endif + +/* Linux power management operations provided by the Mali device driver */ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 29)) +struct pm_ext_ops mali_dev_ext_pm_ops = { + .base = + { + .suspend = mali_driver_suspend_scheduler, + .resume = mali_driver_resume_scheduler, + .freeze = mali_driver_suspend_scheduler, + .thaw = mali_driver_resume_scheduler, + }, +}; +#else +static const struct dev_pm_ops mali_dev_pm_ops = { +#ifdef CONFIG_PM_RUNTIME + .runtime_suspend = mali_driver_runtime_suspend, + .runtime_resume = mali_driver_runtime_resume, + .runtime_idle = mali_driver_runtime_idle, +#endif + .suspend = mali_driver_suspend_scheduler, + .resume = mali_driver_resume_scheduler, + .freeze = mali_driver_suspend_scheduler, + .thaw = mali_driver_resume_scheduler, + .poweroff = mali_driver_suspend_scheduler, +}; +#endif + +#ifdef CONFIG_MALI_DT +static struct of_device_id base_dt_ids[] = { + {.compatible = "arm,mali-300"}, + {.compatible = "arm,mali-400"}, + {.compatible = "arm,mali-450"}, + {.compatible = "arm,mali-470"}, + {}, +}; + +MODULE_DEVICE_TABLE(of, base_dt_ids); +#endif + +/* The Mali device driver struct */ +static struct platform_driver mali_platform_driver = { + .probe = mali_probe, + .remove = mali_remove, +#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 29)) + .pm = &mali_dev_ext_pm_ops, +#endif + .driver = + { + .name = MALI_GPU_NAME_UTGARD, + .owner = THIS_MODULE, + .bus = &platform_bus_type, +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29)) + .pm = &mali_dev_pm_ops, +#endif +#ifdef CONFIG_MALI_DT + .of_match_table = of_match_ptr(base_dt_ids), +#endif + }, +}; + +/* Linux misc device operations (/dev/mali) */ +struct file_operations mali_fops = { + .owner = THIS_MODULE, + .open = mali_open, + .release = mali_release, +#ifdef HAVE_UNLOCKED_IOCTL + .unlocked_ioctl = mali_ioctl, +#else + .ioctl = mali_ioctl, +#endif + .compat_ioctl = mali_ioctl, + .mmap = mali_mmap +}; + +#if MALI_ENABLE_CPU_CYCLES +void mali_init_cpu_time_counters(int reset, int enable_divide_by_64) +{ + /* The CPU assembly reference used is: ARM Architecture Reference Manual ARMv7-AR C.b */ + u32 write_value; + + /* See B4.1.116 PMCNTENSET, Performance Monitors Count Enable Set register, VMSA */ + /* setting p15 c9 c12 1 to 0x8000000f==CPU_CYCLE_ENABLE |EVENT_3_ENABLE|EVENT_2_ENABLE|EVENT_1_ENABLE|EVENT_0_ENABLE */ + asm volatile("mcr p15, 0, %0, c9, c12, 1" :: "r"(0x8000000f)); + + + /* See B4.1.117 PMCR, Performance Monitors Control Register. Writing to p15, c9, c12, 0 */ + write_value = 1 << 0; /* Bit 0 set. Enable counters */ + if (reset) { + write_value |= 1 << 1; /* Reset event counters */ + write_value |= 1 << 2; /* Reset cycle counter */ + } + if (enable_divide_by_64) { + write_value |= 1 << 3; /* Enable the Clock divider by 64 */ + } + write_value |= 1 << 4; /* Export enable. Not needed */ + asm volatile("MCR p15, 0, %0, c9, c12, 0\t\n" :: "r"(write_value)); + + /* PMOVSR Overflow Flag Status Register - Clear Clock and Event overflows */ + asm volatile("MCR p15, 0, %0, c9, c12, 3\t\n" :: "r"(0x8000000f)); + + + /* See B4.1.124 PMUSERENR - setting p15 c9 c14 to 1" */ + /* User mode access to the Performance Monitors enabled. */ + /* Lets User space read cpu clock cycles */ + asm volatile("mcr p15, 0, %0, c9, c14, 0" :: "r"(1)); +} + +/** A timer function that configures the cycle clock counter on current CPU. + * The function \a mali_init_cpu_time_counters_on_all_cpus sets up this + * function to trigger on all Cpus during module load. + */ +static void mali_init_cpu_clock_timer_func(unsigned long data) +{ + int reset_counters, enable_divide_clock_counter_by_64; + int current_cpu = raw_smp_processor_id(); + unsigned int sample0; + unsigned int sample1; + + MALI_IGNORE(data); + + reset_counters = 1; + enable_divide_clock_counter_by_64 = 0; + mali_init_cpu_time_counters(reset_counters, enable_divide_clock_counter_by_64); + + sample0 = mali_get_cpu_cyclecount(); + sample1 = mali_get_cpu_cyclecount(); + + MALI_DEBUG_PRINT(3, ("Init Cpu %d cycle counter- First two samples: %08x %08x \n", current_cpu, sample0, sample1)); +} + +/** A timer functions for storing current time on all cpus. + * Used for checking if the clocks have similar values or if they are drifting. + */ +static void mali_print_cpu_clock_timer_func(unsigned long data) +{ + int current_cpu = raw_smp_processor_id(); + unsigned int sample0; + + MALI_IGNORE(data); + sample0 = mali_get_cpu_cyclecount(); + if (current_cpu < 8) { + mali_cpu_clock_last_value[current_cpu] = sample0; + } +} + +/** Init the performance registers on all CPUs to count clock cycles. + * For init \a print_only should be 0. + * If \a print_only is 1, it will intead print the current clock value of all CPUs. + */ +void mali_init_cpu_time_counters_on_all_cpus(int print_only) +{ + int i = 0; + int cpu_number; + int jiffies_trigger; + int jiffies_wait; + + jiffies_wait = 2; + jiffies_trigger = jiffies + jiffies_wait; + + for (i = 0 ; i < 8 ; i++) { + init_timer(&mali_init_cpu_clock_timers[i]); + if (print_only) mali_init_cpu_clock_timers[i].function = mali_print_cpu_clock_timer_func; + else mali_init_cpu_clock_timers[i].function = mali_init_cpu_clock_timer_func; + mali_init_cpu_clock_timers[i].expires = jiffies_trigger ; + } + cpu_number = cpumask_first(cpu_online_mask); + for (i = 0 ; i < 8 ; i++) { + int next_cpu; + add_timer_on(&mali_init_cpu_clock_timers[i], cpu_number); + next_cpu = cpumask_next(cpu_number, cpu_online_mask); + if (next_cpu >= nr_cpu_ids) break; + cpu_number = next_cpu; + } + + while (jiffies_wait) jiffies_wait = schedule_timeout_uninterruptible(jiffies_wait); + + for (i = 0 ; i < 8 ; i++) { + del_timer_sync(&mali_init_cpu_clock_timers[i]); + } + + if (print_only) { + if ((0 == mali_cpu_clock_last_value[2]) && (0 == mali_cpu_clock_last_value[3])) { + /* Diff can be printed if we want to check if the clocks are in sync + int diff = mali_cpu_clock_last_value[0] - mali_cpu_clock_last_value[1];*/ + MALI_DEBUG_PRINT(2, ("CPU cycle counters readout all: %08x %08x\n", mali_cpu_clock_last_value[0], mali_cpu_clock_last_value[1])); + } else { + MALI_DEBUG_PRINT(2, ("CPU cycle counters readout all: %08x %08x %08x %08x\n", mali_cpu_clock_last_value[0], mali_cpu_clock_last_value[1], mali_cpu_clock_last_value[2], mali_cpu_clock_last_value[3])); + } + } +} +#endif + +int mali_module_init(void) +{ + int err = 0; + + MALI_DEBUG_PRINT(2, ("Inserting Mali v%d device driver. \n", _MALI_API_VERSION)); + MALI_DEBUG_PRINT(2, ("Compiled: %s, time: %s.\n", __DATE__, __TIME__)); + MALI_DEBUG_PRINT(2, ("Driver revision: %s\n", SVN_REV_STRING)); + +#if MALI_ENABLE_CPU_CYCLES + mali_init_cpu_time_counters_on_all_cpus(0); + MALI_DEBUG_PRINT(2, ("CPU cycle counter setup complete\n")); + /* Printing the current cpu counters */ + mali_init_cpu_time_counters_on_all_cpus(1); +#endif + + /* Initialize module wide settings */ +#ifdef MALI_FAKE_PLATFORM_DEVICE +#ifndef CONFIG_MALI_DT + MALI_DEBUG_PRINT(2, ("mali_module_init() registering device\n")); + err = mali_platform_device_register(); + if (0 != err) { + return err; + } +#endif +#endif + + MALI_DEBUG_PRINT(2, ("mali_module_init() registering driver\n")); + + err = platform_driver_register(&mali_platform_driver); + + if (0 != err) { + MALI_DEBUG_PRINT(2, ("mali_module_init() Failed to register driver (%d)\n", err)); +#ifdef MALI_FAKE_PLATFORM_DEVICE +#ifndef CONFIG_MALI_DT + mali_platform_device_unregister(); +#endif +#endif + mali_platform_device = NULL; + return err; + } + +#if defined(CONFIG_MALI400_INTERNAL_PROFILING) + err = _mali_internal_profiling_init(mali_boot_profiling ? MALI_TRUE : MALI_FALSE); + if (0 != err) { + /* No biggie if we wheren't able to initialize the profiling */ + MALI_PRINT_ERROR(("Failed to initialize profiling, feature will be unavailable\n")); + } +#endif + + /* Tracing the current frequency and voltage from boot/insmod*/ +#if defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_MALI_DVFS) + /* Just call mali_get_current_gpu_clk_item(),to record current clk info.*/ + mali_get_current_gpu_clk_item(&mali_gpu_clk[0]); + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | + MALI_PROFILING_EVENT_CHANNEL_GPU | + MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE, + mali_gpu_clk[0].clock, + mali_gpu_clk[0].vol / 1000, + 0, 0, 0); +#endif + + MALI_PRINT(("Mali device driver loaded\n")); + + return 0; /* Success */ +} + +void mali_module_exit(void) +{ + MALI_DEBUG_PRINT(2, ("Unloading Mali v%d device driver.\n", _MALI_API_VERSION)); + + MALI_DEBUG_PRINT(2, ("mali_module_exit() unregistering driver\n")); + + platform_driver_unregister(&mali_platform_driver); + +#if defined(MALI_FAKE_PLATFORM_DEVICE) +#ifndef CONFIG_MALI_DT + MALI_DEBUG_PRINT(2, ("mali_module_exit() unregistering device\n")); + mali_platform_device_unregister(); +#endif +#endif + + /* Tracing the current frequency and voltage from rmmod*/ + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | + MALI_PROFILING_EVENT_CHANNEL_GPU | + MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE, + 0, + 0, + 0, 0, 0); + +#if defined(CONFIG_MALI400_INTERNAL_PROFILING) + _mali_internal_profiling_term(); +#endif + + MALI_PRINT(("Mali device driver unloaded\n")); +} + +static int mali_probe(struct platform_device *pdev) +{ + int err; + + MALI_DEBUG_PRINT(2, ("mali_probe(): Called for platform device %s\n", pdev->name)); + + if (NULL != mali_platform_device) { + /* Already connected to a device, return error */ + MALI_PRINT_ERROR(("mali_probe(): The Mali driver is already connected with a Mali device.")); + return -EEXIST; + } + + mali_platform_device = pdev; + +#ifdef CONFIG_MALI_DT + /* If we use DT to initialize our DDK, we have to prepare somethings. */ + err = mali_platform_device_init(mali_platform_device); + if (0 != err) { + MALI_PRINT_ERROR(("mali_probe(): Failed to initialize platform device.")); + return -EFAULT; + } +#endif + + if (_MALI_OSK_ERR_OK == _mali_osk_wq_init()) { + /* Initialize the Mali GPU HW specified by pdev */ + if (_MALI_OSK_ERR_OK == mali_initialize_subsystems()) { + /* Register a misc device (so we are accessible from user space) */ + err = mali_miscdevice_register(pdev); + if (0 == err) { + /* Setup sysfs entries */ + err = mali_sysfs_register(mali_dev_name); + + if (0 == err) { + MALI_DEBUG_PRINT(2, ("mali_probe(): Successfully initialized driver for platform device %s\n", pdev->name)); + + return 0; + } else { + MALI_PRINT_ERROR(("mali_probe(): failed to register sysfs entries")); + } + mali_miscdevice_unregister(); + } else { + MALI_PRINT_ERROR(("mali_probe(): failed to register Mali misc device.")); + } + mali_terminate_subsystems(); + } else { + MALI_PRINT_ERROR(("mali_probe(): Failed to initialize Mali device driver.")); + } + _mali_osk_wq_term(); + } + + mali_platform_device = NULL; + return -EFAULT; +} + +static int mali_remove(struct platform_device *pdev) +{ + MALI_DEBUG_PRINT(2, ("mali_remove() called for platform device %s\n", pdev->name)); + mali_sysfs_unregister(); + mali_miscdevice_unregister(); + mali_terminate_subsystems(); + _mali_osk_wq_term(); +#ifdef CONFIG_MALI_DT + mali_platform_device_deinit(mali_platform_device); +#endif + mali_platform_device = NULL; + return 0; +} + +static int mali_miscdevice_register(struct platform_device *pdev) +{ + int err; + + mali_miscdevice.minor = MISC_DYNAMIC_MINOR; + mali_miscdevice.name = mali_dev_name; + mali_miscdevice.fops = &mali_fops; + mali_miscdevice.parent = get_device(&pdev->dev); + + err = misc_register(&mali_miscdevice); + if (0 != err) { + MALI_PRINT_ERROR(("Failed to register misc device, misc_register() returned %d\n", err)); + } + + return err; +} + +static void mali_miscdevice_unregister(void) +{ + misc_deregister(&mali_miscdevice); +} + +static int mali_driver_suspend_scheduler(struct device *dev) +{ + mali_pm_os_suspend(MALI_TRUE); + /* Tracing the frequency and voltage after mali is suspended */ + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | + MALI_PROFILING_EVENT_CHANNEL_GPU | + MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE, + 0, + 0, + 0, 0, 0); + return 0; +} + +static int mali_driver_resume_scheduler(struct device *dev) +{ + /* Tracing the frequency and voltage after mali is resumed */ +#if defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_MALI_DVFS) + /* Just call mali_get_current_gpu_clk_item() once,to record current clk info.*/ + if (is_first_resume == 1) { + mali_get_current_gpu_clk_item(&mali_gpu_clk[1]); + is_first_resume = 0; + } + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | + MALI_PROFILING_EVENT_CHANNEL_GPU | + MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE, + mali_gpu_clk[1].clock, + mali_gpu_clk[1].vol / 1000, + 0, 0, 0); +#endif + mali_pm_os_resume(); + return 0; +} + +#ifdef CONFIG_PM_RUNTIME +static int mali_driver_runtime_suspend(struct device *dev) +{ + if (MALI_TRUE == mali_pm_runtime_suspend()) { + /* Tracing the frequency and voltage after mali is suspended */ + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | + MALI_PROFILING_EVENT_CHANNEL_GPU | + MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE, + 0, + 0, + 0, 0, 0); + + return 0; + } else { + return -EBUSY; + } +} + +static int mali_driver_runtime_resume(struct device *dev) +{ + /* Tracing the frequency and voltage after mali is resumed */ +#if defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_MALI_DVFS) + /* Just call mali_get_current_gpu_clk_item() once,to record current clk info.*/ + if (is_first_resume == 1) { + mali_get_current_gpu_clk_item(&mali_gpu_clk[1]); + is_first_resume = 0; + } + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | + MALI_PROFILING_EVENT_CHANNEL_GPU | + MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE, + mali_gpu_clk[1].clock, + mali_gpu_clk[1].vol / 1000, + 0, 0, 0); +#endif + + mali_pm_runtime_resume(); + return 0; +} + +static int mali_driver_runtime_idle(struct device *dev) +{ + /* Nothing to do */ + return 0; +} +#endif + +static int mali_open(struct inode *inode, struct file *filp) +{ + struct mali_session_data *session_data; + _mali_osk_errcode_t err; + + /* input validation */ + if (mali_miscdevice.minor != iminor(inode)) { + MALI_PRINT_ERROR(("mali_open() Minor does not match\n")); + return -ENODEV; + } + + /* allocated struct to track this session */ + err = _mali_ukk_open((void **)&session_data); + if (_MALI_OSK_ERR_OK != err) return map_errcode(err); + + /* initialize file pointer */ + filp->f_pos = 0; + + /* link in our session data */ + filp->private_data = (void *)session_data; + + filp->f_mapping = mali_mem_swap_get_global_swap_file()->f_mapping; + + return 0; +} + +static int mali_release(struct inode *inode, struct file *filp) +{ + _mali_osk_errcode_t err; + + /* input validation */ + if (mali_miscdevice.minor != iminor(inode)) { + MALI_PRINT_ERROR(("mali_release() Minor does not match\n")); + return -ENODEV; + } + + err = _mali_ukk_close((void **)&filp->private_data); + if (_MALI_OSK_ERR_OK != err) return map_errcode(err); + + return 0; +} + +int map_errcode(_mali_osk_errcode_t err) +{ + switch (err) { + case _MALI_OSK_ERR_OK : + return 0; + case _MALI_OSK_ERR_FAULT: + return -EFAULT; + case _MALI_OSK_ERR_INVALID_FUNC: + return -ENOTTY; + case _MALI_OSK_ERR_INVALID_ARGS: + return -EINVAL; + case _MALI_OSK_ERR_NOMEM: + return -ENOMEM; + case _MALI_OSK_ERR_TIMEOUT: + return -ETIMEDOUT; + case _MALI_OSK_ERR_RESTARTSYSCALL: + return -ERESTARTSYS; + case _MALI_OSK_ERR_ITEM_NOT_FOUND: + return -ENOENT; + default: + return -EFAULT; + } +} + +#ifdef HAVE_UNLOCKED_IOCTL +static long mali_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +#else +static int mali_ioctl(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg) +#endif +{ + int err; + struct mali_session_data *session_data; + +#ifndef HAVE_UNLOCKED_IOCTL + /* inode not used */ + (void)inode; +#endif + + MALI_DEBUG_PRINT(7, ("Ioctl received 0x%08X 0x%08lX\n", cmd, arg)); + + session_data = (struct mali_session_data *)filp->private_data; + if (NULL == session_data) { + MALI_DEBUG_PRINT(7, ("filp->private_data was NULL\n")); + return -ENOTTY; + } + + if (NULL == (void *)arg) { + MALI_DEBUG_PRINT(7, ("arg was NULL\n")); + return -ENOTTY; + } + + switch (cmd) { + case MALI_IOC_WAIT_FOR_NOTIFICATION: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_wait_for_notification_s), sizeof(u64))); + err = wait_for_notification_wrapper(session_data, (_mali_uk_wait_for_notification_s __user *)arg); + break; + + case MALI_IOC_GET_API_VERSION_V2: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_api_version_v2_s), sizeof(u64))); + err = get_api_version_v2_wrapper(session_data, (_mali_uk_get_api_version_v2_s __user *)arg); + break; + + case MALI_IOC_GET_API_VERSION: + err = get_api_version_wrapper(session_data, (_mali_uk_get_api_version_s __user *)arg); + break; + + case MALI_IOC_POST_NOTIFICATION: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_post_notification_s), sizeof(u64))); + err = post_notification_wrapper(session_data, (_mali_uk_post_notification_s __user *)arg); + break; + + case MALI_IOC_GET_USER_SETTINGS: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_user_settings_s), sizeof(u64))); + err = get_user_settings_wrapper(session_data, (_mali_uk_get_user_settings_s __user *)arg); + break; + + case MALI_IOC_REQUEST_HIGH_PRIORITY: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_request_high_priority_s), sizeof(u64))); + err = request_high_priority_wrapper(session_data, (_mali_uk_request_high_priority_s __user *)arg); + break; + + case MALI_IOC_PENDING_SUBMIT: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_pending_submit_s), sizeof(u64))); + err = pending_submit_wrapper(session_data, (_mali_uk_pending_submit_s __user *)arg); + break; + +#if defined(CONFIG_MALI400_PROFILING) + case MALI_IOC_PROFILING_ADD_EVENT: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_profiling_add_event_s), sizeof(u64))); + err = profiling_add_event_wrapper(session_data, (_mali_uk_profiling_add_event_s __user *)arg); + break; + + case MALI_IOC_PROFILING_REPORT_SW_COUNTERS: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_sw_counters_report_s), sizeof(u64))); + err = profiling_report_sw_counters_wrapper(session_data, (_mali_uk_sw_counters_report_s __user *)arg); + break; + + case MALI_IOC_PROFILING_STREAM_FD_GET: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_profiling_stream_fd_get_s), sizeof(u64))); + err = profiling_get_stream_fd_wrapper(session_data, (_mali_uk_profiling_stream_fd_get_s __user *)arg); + break; + + case MALI_IOC_PROILING_CONTROL_SET: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_profiling_control_set_s), sizeof(u64))); + err = profiling_control_set_wrapper(session_data, (_mali_uk_profiling_control_set_s __user *)arg); + break; +#else + + case MALI_IOC_PROFILING_ADD_EVENT: /* FALL-THROUGH */ + case MALI_IOC_PROFILING_REPORT_SW_COUNTERS: /* FALL-THROUGH */ + MALI_DEBUG_PRINT(2, ("Profiling not supported\n")); + err = -ENOTTY; + break; +#endif + + case MALI_IOC_PROFILING_MEMORY_USAGE_GET: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_profiling_memory_usage_get_s), sizeof(u64))); + err = mem_usage_get_wrapper(session_data, (_mali_uk_profiling_memory_usage_get_s __user *)arg); + break; + + case MALI_IOC_MEM_ALLOC: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_alloc_mem_s), sizeof(u64))); + err = mem_alloc_wrapper(session_data, (_mali_uk_alloc_mem_s __user *)arg); + break; + + case MALI_IOC_MEM_FREE: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_free_mem_s), sizeof(u64))); + err = mem_free_wrapper(session_data, (_mali_uk_free_mem_s __user *)arg); + break; + + case MALI_IOC_MEM_BIND: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_bind_mem_s), sizeof(u64))); + err = mem_bind_wrapper(session_data, (_mali_uk_bind_mem_s __user *)arg); + break; + + case MALI_IOC_MEM_UNBIND: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_unbind_mem_s), sizeof(u64))); + err = mem_unbind_wrapper(session_data, (_mali_uk_unbind_mem_s __user *)arg); + break; + + case MALI_IOC_MEM_COW: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_cow_mem_s), sizeof(u64))); + err = mem_cow_wrapper(session_data, (_mali_uk_cow_mem_s __user *)arg); + break; + + case MALI_IOC_MEM_COW_MODIFY_RANGE: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_cow_modify_range_s), sizeof(u64))); + err = mem_cow_modify_range_wrapper(session_data, (_mali_uk_cow_modify_range_s __user *)arg); + break; + + case MALI_IOC_MEM_RESIZE: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_mem_resize_s), sizeof(u64))); + err = mem_resize_mem_wrapper(session_data, (_mali_uk_mem_resize_s __user *)arg); + break; + + case MALI_IOC_MEM_WRITE_SAFE: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_mem_write_safe_s), sizeof(u64))); + err = mem_write_safe_wrapper(session_data, (_mali_uk_mem_write_safe_s __user *)arg); + break; + + case MALI_IOC_MEM_QUERY_MMU_PAGE_TABLE_DUMP_SIZE: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_query_mmu_page_table_dump_size_s), sizeof(u64))); + err = mem_query_mmu_page_table_dump_size_wrapper(session_data, (_mali_uk_query_mmu_page_table_dump_size_s __user *)arg); + break; + + case MALI_IOC_MEM_DUMP_MMU_PAGE_TABLE: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_dump_mmu_page_table_s), sizeof(u64))); + err = mem_dump_mmu_page_table_wrapper(session_data, (_mali_uk_dump_mmu_page_table_s __user *)arg); + break; + + case MALI_IOC_MEM_DMA_BUF_GET_SIZE: +#ifdef CONFIG_DMA_SHARED_BUFFER + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_dma_buf_get_size_s), sizeof(u64))); + err = mali_dma_buf_get_size(session_data, (_mali_uk_dma_buf_get_size_s __user *)arg); +#else + MALI_DEBUG_PRINT(2, ("DMA-BUF not supported\n")); + err = -ENOTTY; +#endif + break; + + case MALI_IOC_PP_START_JOB: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_pp_start_job_s), sizeof(u64))); + err = pp_start_job_wrapper(session_data, (_mali_uk_pp_start_job_s __user *)arg); + break; + + case MALI_IOC_PP_AND_GP_START_JOB: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_pp_and_gp_start_job_s), sizeof(u64))); + err = pp_and_gp_start_job_wrapper(session_data, (_mali_uk_pp_and_gp_start_job_s __user *)arg); + break; + + case MALI_IOC_PP_NUMBER_OF_CORES_GET: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_pp_number_of_cores_s), sizeof(u64))); + err = pp_get_number_of_cores_wrapper(session_data, (_mali_uk_get_pp_number_of_cores_s __user *)arg); + break; + + case MALI_IOC_PP_CORE_VERSION_GET: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_pp_core_version_s), sizeof(u64))); + err = pp_get_core_version_wrapper(session_data, (_mali_uk_get_pp_core_version_s __user *)arg); + break; + + case MALI_IOC_PP_DISABLE_WB: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_pp_disable_wb_s), sizeof(u64))); + err = pp_disable_wb_wrapper(session_data, (_mali_uk_pp_disable_wb_s __user *)arg); + break; + + case MALI_IOC_GP2_START_JOB: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_gp_start_job_s), sizeof(u64))); + err = gp_start_job_wrapper(session_data, (_mali_uk_gp_start_job_s __user *)arg); + break; + + case MALI_IOC_GP2_NUMBER_OF_CORES_GET: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_gp_number_of_cores_s), sizeof(u64))); + err = gp_get_number_of_cores_wrapper(session_data, (_mali_uk_get_gp_number_of_cores_s __user *)arg); + break; + + case MALI_IOC_GP2_CORE_VERSION_GET: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_gp_core_version_s), sizeof(u64))); + err = gp_get_core_version_wrapper(session_data, (_mali_uk_get_gp_core_version_s __user *)arg); + break; + + case MALI_IOC_GP2_SUSPEND_RESPONSE: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_gp_suspend_response_s), sizeof(u64))); + err = gp_suspend_response_wrapper(session_data, (_mali_uk_gp_suspend_response_s __user *)arg); + break; + + case MALI_IOC_VSYNC_EVENT_REPORT: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_vsync_event_report_s), sizeof(u64))); + err = vsync_event_report_wrapper(session_data, (_mali_uk_vsync_event_report_s __user *)arg); + break; + + case MALI_IOC_TIMELINE_GET_LATEST_POINT: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_timeline_get_latest_point_s), sizeof(u64))); + err = timeline_get_latest_point_wrapper(session_data, (_mali_uk_timeline_get_latest_point_s __user *)arg); + break; + case MALI_IOC_TIMELINE_WAIT: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_timeline_wait_s), sizeof(u64))); + err = timeline_wait_wrapper(session_data, (_mali_uk_timeline_wait_s __user *)arg); + break; + case MALI_IOC_TIMELINE_CREATE_SYNC_FENCE: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_timeline_create_sync_fence_s), sizeof(u64))); + err = timeline_create_sync_fence_wrapper(session_data, (_mali_uk_timeline_create_sync_fence_s __user *)arg); + break; + case MALI_IOC_SOFT_JOB_START: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_soft_job_start_s), sizeof(u64))); + err = soft_job_start_wrapper(session_data, (_mali_uk_soft_job_start_s __user *)arg); + break; + case MALI_IOC_SOFT_JOB_SIGNAL: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_soft_job_signal_s), sizeof(u64))); + err = soft_job_signal_wrapper(session_data, (_mali_uk_soft_job_signal_s __user *)arg); + break; + + default: + MALI_DEBUG_PRINT(2, ("No handler for ioctl 0x%08X 0x%08lX\n", cmd, arg)); + err = -ENOTTY; + }; + + return err; +} + + +module_init(mali_module_init); +module_exit(mali_module_exit); + +MODULE_LICENSE(MALI_KERNEL_LINUX_LICENSE); +MODULE_AUTHOR("ARM Ltd."); +MODULE_VERSION(SVN_REV_STRING); diff --git a/drivers/gpu/arm/utgard/linux/mali_kernel_linux.h b/drivers/gpu/arm/utgard/linux/mali_kernel_linux.h new file mode 100644 index 00000000000000..f6042712d1e3a7 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_kernel_linux.h @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_KERNEL_LINUX_H__ +#define __MALI_KERNEL_LINUX_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +#include /* character device definitions */ +#include +#include +#include "mali_kernel_license.h" +#include "mali_osk_types.h" + +extern struct platform_device *mali_platform_device; + +#ifdef __cplusplus +} +#endif + +#endif /* __MALI_KERNEL_LINUX_H__ */ diff --git a/drivers/gpu/arm/utgard/linux/mali_kernel_sysfs.c b/drivers/gpu/arm/utgard/linux/mali_kernel_sysfs.c new file mode 100644 index 00000000000000..4f06ca14bf48fb --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_kernel_sysfs.c @@ -0,0 +1,1410 @@ +/** + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + + +/** + * @file mali_kernel_sysfs.c + * Implementation of some sysfs data exports + */ + +#include +#include +#include +#include +#include "mali_kernel_license.h" +#include "mali_kernel_common.h" +#include "mali_ukk.h" + +#if MALI_LICENSE_IS_GPL + +#include +#include +#include +#include +#include +#include "mali_kernel_sysfs.h" +#if defined(CONFIG_MALI400_INTERNAL_PROFILING) +#include +#include "mali_osk_profiling.h" +#endif + +#include +#include "mali_pm.h" +#include "mali_pmu.h" +#include "mali_group.h" +#include "mali_gp.h" +#include "mali_pp.h" +#include "mali_l2_cache.h" +#include "mali_hw_core.h" +#include "mali_kernel_core.h" +#include "mali_user_settings_db.h" +#include "mali_profiling_internal.h" +#include "mali_gp_job.h" +#include "mali_pp_job.h" +#include "mali_executor.h" + +#define PRIVATE_DATA_COUNTER_MAKE_GP(src) (src) +#define PRIVATE_DATA_COUNTER_MAKE_PP(src) ((1 << 24) | src) +#define PRIVATE_DATA_COUNTER_MAKE_PP_SUB_JOB(src, sub_job) ((1 << 24) | (1 << 16) | (sub_job << 8) | src) +#define PRIVATE_DATA_COUNTER_IS_PP(a) ((((a) >> 24) & 0xFF) ? MALI_TRUE : MALI_FALSE) +#define PRIVATE_DATA_COUNTER_GET_SRC(a) (a & 0xFF) +#define PRIVATE_DATA_COUNTER_IS_SUB_JOB(a) ((((a) >> 16) & 0xFF) ? MALI_TRUE : MALI_FALSE) +#define PRIVATE_DATA_COUNTER_GET_SUB_JOB(a) (((a) >> 8) & 0xFF) + +#define POWER_BUFFER_SIZE 3 + +static struct dentry *mali_debugfs_dir = NULL; + +typedef enum { + _MALI_DEVICE_SUSPEND, + _MALI_DEVICE_RESUME, + _MALI_DEVICE_DVFS_PAUSE, + _MALI_DEVICE_DVFS_RESUME, + _MALI_MAX_EVENTS +} _mali_device_debug_power_events; + +static const char *const mali_power_events[_MALI_MAX_EVENTS] = { + [_MALI_DEVICE_SUSPEND] = "suspend", + [_MALI_DEVICE_RESUME] = "resume", + [_MALI_DEVICE_DVFS_PAUSE] = "dvfs_pause", + [_MALI_DEVICE_DVFS_RESUME] = "dvfs_resume", +}; + +static mali_bool power_always_on_enabled = MALI_FALSE; + +static int open_copy_private_data(struct inode *inode, struct file *filp) +{ + filp->private_data = inode->i_private; + return 0; +} + +static ssize_t group_enabled_read(struct file *filp, char __user *buf, size_t count, loff_t *offp) +{ + int r; + char buffer[64]; + struct mali_group *group; + + group = (struct mali_group *)filp->private_data; + MALI_DEBUG_ASSERT_POINTER(group); + + r = snprintf(buffer, 64, "%u\n", + mali_executor_group_is_disabled(group) ? 0 : 1); + + return simple_read_from_buffer(buf, count, offp, buffer, r); +} + +static ssize_t group_enabled_write(struct file *filp, const char __user *buf, size_t count, loff_t *offp) +{ + int r; + char buffer[64]; + unsigned long val; + struct mali_group *group; + + group = (struct mali_group *)filp->private_data; + MALI_DEBUG_ASSERT_POINTER(group); + + if (count >= sizeof(buffer)) { + return -ENOMEM; + } + + if (copy_from_user(&buffer[0], buf, count)) { + return -EFAULT; + } + buffer[count] = '\0'; + + r = kstrtoul(&buffer[0], 10, &val); + if (0 != r) { + return -EINVAL; + } + + switch (val) { + case 1: + mali_executor_group_enable(group); + break; + case 0: + mali_executor_group_disable(group); + break; + default: + return -EINVAL; + break; + } + + *offp += count; + return count; +} + +static const struct file_operations group_enabled_fops = { + .owner = THIS_MODULE, + .open = open_copy_private_data, + .read = group_enabled_read, + .write = group_enabled_write, +}; + +static ssize_t hw_core_base_addr_read(struct file *filp, char __user *buf, size_t count, loff_t *offp) +{ + int r; + char buffer[64]; + struct mali_hw_core *hw_core; + + hw_core = (struct mali_hw_core *)filp->private_data; + MALI_DEBUG_ASSERT_POINTER(hw_core); + + r = snprintf(buffer, 64, "0x%lX\n", hw_core->phys_addr); + + return simple_read_from_buffer(buf, count, offp, buffer, r); +} + +static const struct file_operations hw_core_base_addr_fops = { + .owner = THIS_MODULE, + .open = open_copy_private_data, + .read = hw_core_base_addr_read, +}; + +static ssize_t profiling_counter_src_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) +{ + u32 is_pp = PRIVATE_DATA_COUNTER_IS_PP((uintptr_t)filp->private_data); + u32 src_id = PRIVATE_DATA_COUNTER_GET_SRC((uintptr_t)filp->private_data); + mali_bool is_sub_job = PRIVATE_DATA_COUNTER_IS_SUB_JOB((uintptr_t)filp->private_data); + u32 sub_job = PRIVATE_DATA_COUNTER_GET_SUB_JOB((uintptr_t)filp->private_data); + char buf[64]; + int r; + u32 val; + + if (MALI_TRUE == is_pp) { + /* PP counter */ + if (MALI_TRUE == is_sub_job) { + /* Get counter for a particular sub job */ + if (0 == src_id) { + val = mali_pp_job_get_pp_counter_sub_job_src0(sub_job); + } else { + val = mali_pp_job_get_pp_counter_sub_job_src1(sub_job); + } + } else { + /* Get default counter for all PP sub jobs */ + if (0 == src_id) { + val = mali_pp_job_get_pp_counter_global_src0(); + } else { + val = mali_pp_job_get_pp_counter_global_src1(); + } + } + } else { + /* GP counter */ + if (0 == src_id) { + val = mali_gp_job_get_gp_counter_src0(); + } else { + val = mali_gp_job_get_gp_counter_src1(); + } + } + + if (MALI_HW_CORE_NO_COUNTER == val) { + r = snprintf(buf, 64, "-1\n"); + } else { + r = snprintf(buf, 64, "%u\n", val); + } + + return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); +} + +static ssize_t profiling_counter_src_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) +{ + u32 is_pp = PRIVATE_DATA_COUNTER_IS_PP((uintptr_t)filp->private_data); + u32 src_id = PRIVATE_DATA_COUNTER_GET_SRC((uintptr_t)filp->private_data); + mali_bool is_sub_job = PRIVATE_DATA_COUNTER_IS_SUB_JOB((uintptr_t)filp->private_data); + u32 sub_job = PRIVATE_DATA_COUNTER_GET_SUB_JOB((uintptr_t)filp->private_data); + char buf[64]; + long val; + int ret; + + if (cnt >= sizeof(buf)) { + return -EINVAL; + } + + if (copy_from_user(&buf, ubuf, cnt)) { + return -EFAULT; + } + + buf[cnt] = 0; + + ret = kstrtol(buf, 10, &val); + if (ret < 0) { + return ret; + } + + if (val < 0) { + /* any negative input will disable counter */ + val = MALI_HW_CORE_NO_COUNTER; + } + + if (MALI_TRUE == is_pp) { + /* PP counter */ + if (MALI_TRUE == is_sub_job) { + /* Set counter for a particular sub job */ + if (0 == src_id) { + mali_pp_job_set_pp_counter_sub_job_src0(sub_job, (u32)val); + } else { + mali_pp_job_set_pp_counter_sub_job_src1(sub_job, (u32)val); + } + } else { + /* Set default counter for all PP sub jobs */ + if (0 == src_id) { + mali_pp_job_set_pp_counter_global_src0((u32)val); + } else { + mali_pp_job_set_pp_counter_global_src1((u32)val); + } + } + } else { + /* GP counter */ + if (0 == src_id) { + mali_gp_job_set_gp_counter_src0((u32)val); + } else { + mali_gp_job_set_gp_counter_src1((u32)val); + } + } + + *ppos += cnt; + return cnt; +} + +static const struct file_operations profiling_counter_src_fops = { + .owner = THIS_MODULE, + .open = open_copy_private_data, + .read = profiling_counter_src_read, + .write = profiling_counter_src_write, +}; + +static ssize_t l2_l2x_counter_srcx_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos, u32 src_id) +{ + char buf[64]; + int r; + u32 val; + struct mali_l2_cache_core *l2_core = (struct mali_l2_cache_core *)filp->private_data; + + if (0 == src_id) { + val = mali_l2_cache_core_get_counter_src0(l2_core); + } else { + val = mali_l2_cache_core_get_counter_src1(l2_core); + } + + if (MALI_HW_CORE_NO_COUNTER == val) { + r = snprintf(buf, 64, "-1\n"); + } else { + r = snprintf(buf, 64, "%u\n", val); + } + return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); +} + +static ssize_t l2_l2x_counter_srcx_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos, u32 src_id) +{ + struct mali_l2_cache_core *l2_core = (struct mali_l2_cache_core *)filp->private_data; + char buf[64]; + long val; + int ret; + + if (cnt >= sizeof(buf)) { + return -EINVAL; + } + + if (copy_from_user(&buf, ubuf, cnt)) { + return -EFAULT; + } + + buf[cnt] = 0; + + ret = kstrtol(buf, 10, &val); + if (ret < 0) { + return ret; + } + + if (val < 0) { + /* any negative input will disable counter */ + val = MALI_HW_CORE_NO_COUNTER; + } + + mali_l2_cache_core_set_counter_src(l2_core, src_id, (u32)val); + + *ppos += cnt; + return cnt; +} + +static ssize_t l2_all_counter_srcx_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos, u32 src_id) +{ + char buf[64]; + long val; + int ret; + u32 l2_id; + struct mali_l2_cache_core *l2_cache; + + if (cnt >= sizeof(buf)) { + return -EINVAL; + } + + if (copy_from_user(&buf, ubuf, cnt)) { + return -EFAULT; + } + + buf[cnt] = 0; + + ret = kstrtol(buf, 10, &val); + if (ret < 0) { + return ret; + } + + if (val < 0) { + /* any negative input will disable counter */ + val = MALI_HW_CORE_NO_COUNTER; + } + + l2_id = 0; + l2_cache = mali_l2_cache_core_get_glob_l2_core(l2_id); + while (NULL != l2_cache) { + mali_l2_cache_core_set_counter_src(l2_cache, src_id, (u32)val); + + /* try next L2 */ + l2_id++; + l2_cache = mali_l2_cache_core_get_glob_l2_core(l2_id); + } + + *ppos += cnt; + return cnt; +} + +static ssize_t l2_l2x_counter_src0_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) +{ + return l2_l2x_counter_srcx_read(filp, ubuf, cnt, ppos, 0); +} + +static ssize_t l2_l2x_counter_src1_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) +{ + return l2_l2x_counter_srcx_read(filp, ubuf, cnt, ppos, 1); +} + +static ssize_t l2_l2x_counter_src0_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) +{ + return l2_l2x_counter_srcx_write(filp, ubuf, cnt, ppos, 0); +} + +static ssize_t l2_l2x_counter_src1_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) +{ + return l2_l2x_counter_srcx_write(filp, ubuf, cnt, ppos, 1); +} + +static ssize_t l2_all_counter_src0_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) +{ + return l2_all_counter_srcx_write(filp, ubuf, cnt, ppos, 0); +} + +static ssize_t l2_all_counter_src1_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) +{ + return l2_all_counter_srcx_write(filp, ubuf, cnt, ppos, 1); +} + +static const struct file_operations l2_l2x_counter_src0_fops = { + .owner = THIS_MODULE, + .open = open_copy_private_data, + .read = l2_l2x_counter_src0_read, + .write = l2_l2x_counter_src0_write, +}; + +static const struct file_operations l2_l2x_counter_src1_fops = { + .owner = THIS_MODULE, + .open = open_copy_private_data, + .read = l2_l2x_counter_src1_read, + .write = l2_l2x_counter_src1_write, +}; + +static const struct file_operations l2_all_counter_src0_fops = { + .owner = THIS_MODULE, + .write = l2_all_counter_src0_write, +}; + +static const struct file_operations l2_all_counter_src1_fops = { + .owner = THIS_MODULE, + .write = l2_all_counter_src1_write, +}; + +static ssize_t l2_l2x_counter_valx_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos, u32 src_id) +{ + char buf[64]; + int r; + u32 src0 = 0; + u32 val0 = 0; + u32 src1 = 0; + u32 val1 = 0; + u32 val = -1; + struct mali_l2_cache_core *l2_core = (struct mali_l2_cache_core *)filp->private_data; + + mali_l2_cache_core_get_counter_values(l2_core, &src0, &val0, &src1, &val1); + + if (0 == src_id) { + if (MALI_HW_CORE_NO_COUNTER != val0) { + val = val0; + } + } else { + if (MALI_HW_CORE_NO_COUNTER != val1) { + val = val1; + } + } + + r = snprintf(buf, 64, "%u\n", val); + + return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); +} + +static ssize_t l2_l2x_counter_val0_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) +{ + return l2_l2x_counter_valx_read(filp, ubuf, cnt, ppos, 0); +} + +static ssize_t l2_l2x_counter_val1_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) +{ + return l2_l2x_counter_valx_read(filp, ubuf, cnt, ppos, 1); +} + +static const struct file_operations l2_l2x_counter_val0_fops = { + .owner = THIS_MODULE, + .open = open_copy_private_data, + .read = l2_l2x_counter_val0_read, +}; + +static const struct file_operations l2_l2x_counter_val1_fops = { + .owner = THIS_MODULE, + .open = open_copy_private_data, + .read = l2_l2x_counter_val1_read, +}; + +static ssize_t power_always_on_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) +{ + unsigned long val; + int ret; + char buf[32]; + + cnt = min(cnt, sizeof(buf) - 1); + if (copy_from_user(buf, ubuf, cnt)) { + return -EFAULT; + } + buf[cnt] = '\0'; + + ret = kstrtoul(buf, 10, &val); + if (0 != ret) { + return ret; + } + + /* Update setting (not exactly thread safe) */ + if (1 == val && MALI_FALSE == power_always_on_enabled) { + power_always_on_enabled = MALI_TRUE; + _mali_osk_pm_dev_ref_get_sync(); + } else if (0 == val && MALI_TRUE == power_always_on_enabled) { + power_always_on_enabled = MALI_FALSE; + _mali_osk_pm_dev_ref_put(); + } + + *ppos += cnt; + return cnt; +} + +static ssize_t power_always_on_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) +{ + if (MALI_TRUE == power_always_on_enabled) { + return simple_read_from_buffer(ubuf, cnt, ppos, "1\n", 2); + } else { + return simple_read_from_buffer(ubuf, cnt, ppos, "0\n", 2); + } +} + +static const struct file_operations power_always_on_fops = { + .owner = THIS_MODULE, + .read = power_always_on_read, + .write = power_always_on_write, +}; + +static ssize_t power_power_events_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) +{ + if (!strncmp(ubuf, mali_power_events[_MALI_DEVICE_SUSPEND], strlen(mali_power_events[_MALI_DEVICE_SUSPEND]) - 1)) { + mali_pm_os_suspend(MALI_TRUE); + } else if (!strncmp(ubuf, mali_power_events[_MALI_DEVICE_RESUME], strlen(mali_power_events[_MALI_DEVICE_RESUME]) - 1)) { + mali_pm_os_resume(); + } else if (!strncmp(ubuf, mali_power_events[_MALI_DEVICE_DVFS_PAUSE], strlen(mali_power_events[_MALI_DEVICE_DVFS_PAUSE]) - 1)) { + mali_dev_pause(); + } else if (!strncmp(ubuf, mali_power_events[_MALI_DEVICE_DVFS_RESUME], strlen(mali_power_events[_MALI_DEVICE_DVFS_RESUME]) - 1)) { + mali_dev_resume(); + } + *ppos += cnt; + return cnt; +} + +static loff_t power_power_events_seek(struct file *file, loff_t offset, int orig) +{ + file->f_pos = offset; + return 0; +} + +static const struct file_operations power_power_events_fops = { + .owner = THIS_MODULE, + .write = power_power_events_write, + .llseek = power_power_events_seek, +}; + +#if MALI_STATE_TRACKING +static int mali_seq_internal_state_show(struct seq_file *seq_file, void *v) +{ + u32 len = 0; + u32 size; + char *buf; + + size = seq_get_buf(seq_file, &buf); + + if (!size) { + return -ENOMEM; + } + + /* Create the internal state dump. */ + len = snprintf(buf + len, size - len, "Mali device driver %s\n", SVN_REV_STRING); + len += snprintf(buf + len, size - len, "License: %s\n\n", MALI_KERNEL_LINUX_LICENSE); + + len += _mali_kernel_core_dump_state(buf + len, size - len); + + seq_commit(seq_file, len); + + return 0; +} + +static int mali_seq_internal_state_open(struct inode *inode, struct file *file) +{ + return single_open(file, mali_seq_internal_state_show, NULL); +} + +static const struct file_operations mali_seq_internal_state_fops = { + .owner = THIS_MODULE, + .open = mali_seq_internal_state_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; +#endif /* MALI_STATE_TRACKING */ + +#if defined(CONFIG_MALI400_INTERNAL_PROFILING) +static ssize_t profiling_record_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) +{ + char buf[64]; + int r; + + r = snprintf(buf, 64, "%u\n", _mali_internal_profiling_is_recording() ? 1 : 0); + return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); +} + +static ssize_t profiling_record_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) +{ + char buf[64]; + unsigned long val; + int ret; + + if (cnt >= sizeof(buf)) { + return -EINVAL; + } + + if (copy_from_user(&buf, ubuf, cnt)) { + return -EFAULT; + } + + buf[cnt] = 0; + + ret = kstrtoul(buf, 10, &val); + if (ret < 0) { + return ret; + } + + if (val != 0) { + u32 limit = MALI_PROFILING_MAX_BUFFER_ENTRIES; /* This can be made configurable at a later stage if we need to */ + + /* check if we are already recording */ + if (MALI_TRUE == _mali_internal_profiling_is_recording()) { + MALI_DEBUG_PRINT(3, ("Recording of profiling events already in progress\n")); + return -EFAULT; + } + + /* check if we need to clear out an old recording first */ + if (MALI_TRUE == _mali_internal_profiling_have_recording()) { + if (_MALI_OSK_ERR_OK != _mali_internal_profiling_clear()) { + MALI_DEBUG_PRINT(3, ("Failed to clear existing recording of profiling events\n")); + return -EFAULT; + } + } + + /* start recording profiling data */ + if (_MALI_OSK_ERR_OK != _mali_internal_profiling_start(&limit)) { + MALI_DEBUG_PRINT(3, ("Failed to start recording of profiling events\n")); + return -EFAULT; + } + + MALI_DEBUG_PRINT(3, ("Profiling recording started (max %u events)\n", limit)); + } else { + /* stop recording profiling data */ + u32 count = 0; + if (_MALI_OSK_ERR_OK != _mali_internal_profiling_stop(&count)) { + MALI_DEBUG_PRINT(2, ("Failed to stop recording of profiling events\n")); + return -EFAULT; + } + + MALI_DEBUG_PRINT(2, ("Profiling recording stopped (recorded %u events)\n", count)); + } + + *ppos += cnt; + return cnt; +} + +static const struct file_operations profiling_record_fops = { + .owner = THIS_MODULE, + .read = profiling_record_read, + .write = profiling_record_write, +}; + +static void *profiling_events_start(struct seq_file *s, loff_t *pos) +{ + loff_t *spos; + + /* check if we have data avaiable */ + if (MALI_TRUE != _mali_internal_profiling_have_recording()) { + return NULL; + } + + spos = kmalloc(sizeof(loff_t), GFP_KERNEL); + if (NULL == spos) { + return NULL; + } + + *spos = *pos; + return spos; +} + +static void *profiling_events_next(struct seq_file *s, void *v, loff_t *pos) +{ + loff_t *spos = v; + + /* check if we have data avaiable */ + if (MALI_TRUE != _mali_internal_profiling_have_recording()) { + return NULL; + } + + /* check if the next entry actually is avaiable */ + if (_mali_internal_profiling_get_count() <= (u32)(*spos + 1)) { + return NULL; + } + + *pos = ++*spos; + return spos; +} + +static void profiling_events_stop(struct seq_file *s, void *v) +{ + kfree(v); +} + +static int profiling_events_show(struct seq_file *seq_file, void *v) +{ + loff_t *spos = v; + u32 index; + u64 timestamp; + u32 event_id; + u32 data[5]; + + index = (u32) * spos; + + /* Retrieve all events */ + if (_MALI_OSK_ERR_OK == _mali_internal_profiling_get_event(index, ×tamp, &event_id, data)) { + seq_printf(seq_file, "%llu %u %u %u %u %u %u\n", timestamp, event_id, data[0], data[1], data[2], data[3], data[4]); + return 0; + } + + return 0; +} + +static int profiling_events_show_human_readable(struct seq_file *seq_file, void *v) +{ +#define MALI_EVENT_ID_IS_HW(event_id) (((event_id & 0x00FF0000) >= MALI_PROFILING_EVENT_CHANNEL_GP0) && ((event_id & 0x00FF0000) <= MALI_PROFILING_EVENT_CHANNEL_PP7)) + + static u64 start_time = 0; + loff_t *spos = v; + u32 index; + u64 timestamp; + u32 event_id; + u32 data[5]; + + index = (u32) * spos; + + /* Retrieve all events */ + if (_MALI_OSK_ERR_OK == _mali_internal_profiling_get_event(index, ×tamp, &event_id, data)) { + seq_printf(seq_file, "%llu %u %u %u %u %u %u # ", timestamp, event_id, data[0], data[1], data[2], data[3], data[4]); + + if (0 == index) { + start_time = timestamp; + } + + seq_printf(seq_file, "[%06u] ", index); + + switch (event_id & 0x0F000000) { + case MALI_PROFILING_EVENT_TYPE_SINGLE: + seq_printf(seq_file, "SINGLE | "); + break; + case MALI_PROFILING_EVENT_TYPE_START: + seq_printf(seq_file, "START | "); + break; + case MALI_PROFILING_EVENT_TYPE_STOP: + seq_printf(seq_file, "STOP | "); + break; + case MALI_PROFILING_EVENT_TYPE_SUSPEND: + seq_printf(seq_file, "SUSPEND | "); + break; + case MALI_PROFILING_EVENT_TYPE_RESUME: + seq_printf(seq_file, "RESUME | "); + break; + default: + seq_printf(seq_file, "0x%01X | ", (event_id & 0x0F000000) >> 24); + break; + } + + switch (event_id & 0x00FF0000) { + case MALI_PROFILING_EVENT_CHANNEL_SOFTWARE: + seq_printf(seq_file, "SW | "); + break; + case MALI_PROFILING_EVENT_CHANNEL_GP0: + seq_printf(seq_file, "GP0 | "); + break; + case MALI_PROFILING_EVENT_CHANNEL_PP0: + seq_printf(seq_file, "PP0 | "); + break; + case MALI_PROFILING_EVENT_CHANNEL_PP1: + seq_printf(seq_file, "PP1 | "); + break; + case MALI_PROFILING_EVENT_CHANNEL_PP2: + seq_printf(seq_file, "PP2 | "); + break; + case MALI_PROFILING_EVENT_CHANNEL_PP3: + seq_printf(seq_file, "PP3 | "); + break; + case MALI_PROFILING_EVENT_CHANNEL_PP4: + seq_printf(seq_file, "PP4 | "); + break; + case MALI_PROFILING_EVENT_CHANNEL_PP5: + seq_printf(seq_file, "PP5 | "); + break; + case MALI_PROFILING_EVENT_CHANNEL_PP6: + seq_printf(seq_file, "PP6 | "); + break; + case MALI_PROFILING_EVENT_CHANNEL_PP7: + seq_printf(seq_file, "PP7 | "); + break; + case MALI_PROFILING_EVENT_CHANNEL_GPU: + seq_printf(seq_file, "GPU | "); + break; + default: + seq_printf(seq_file, "0x%02X | ", (event_id & 0x00FF0000) >> 16); + break; + } + + if (MALI_EVENT_ID_IS_HW(event_id)) { + if (((event_id & 0x0F000000) == MALI_PROFILING_EVENT_TYPE_START) || ((event_id & 0x0F000000) == MALI_PROFILING_EVENT_TYPE_STOP)) { + switch (event_id & 0x0000FFFF) { + case MALI_PROFILING_EVENT_REASON_START_STOP_HW_PHYSICAL: + seq_printf(seq_file, "PHYSICAL | "); + break; + case MALI_PROFILING_EVENT_REASON_START_STOP_HW_VIRTUAL: + seq_printf(seq_file, "VIRTUAL | "); + break; + default: + seq_printf(seq_file, "0x%04X | ", event_id & 0x0000FFFF); + break; + } + } else { + seq_printf(seq_file, "0x%04X | ", event_id & 0x0000FFFF); + } + } else { + seq_printf(seq_file, "0x%04X | ", event_id & 0x0000FFFF); + } + + seq_printf(seq_file, "T0 + 0x%016llX\n", timestamp - start_time); + + return 0; + } + + return 0; +} + +static const struct seq_operations profiling_events_seq_ops = { + .start = profiling_events_start, + .next = profiling_events_next, + .stop = profiling_events_stop, + .show = profiling_events_show +}; + +static int profiling_events_open(struct inode *inode, struct file *file) +{ + return seq_open(file, &profiling_events_seq_ops); +} + +static const struct file_operations profiling_events_fops = { + .owner = THIS_MODULE, + .open = profiling_events_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release, +}; + +static const struct seq_operations profiling_events_human_readable_seq_ops = { + .start = profiling_events_start, + .next = profiling_events_next, + .stop = profiling_events_stop, + .show = profiling_events_show_human_readable +}; + +static int profiling_events_human_readable_open(struct inode *inode, struct file *file) +{ + return seq_open(file, &profiling_events_human_readable_seq_ops); +} + +static const struct file_operations profiling_events_human_readable_fops = { + .owner = THIS_MODULE, + .open = profiling_events_human_readable_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release, +}; + +#endif + +static int memory_debugfs_show(struct seq_file *s, void *private_data) +{ +#ifdef MALI_MEM_SWAP_TRACKING + seq_printf(s, " %-25s %-10s %-10s %-15s %-15s %-10s %-10s %-10s \n"\ + "=================================================================================================================================\n", + "Name (:bytes)", "pid", "mali_mem", "max_mali_mem", + "external_mem", "ump_mem", "dma_mem", "swap_mem"); +#else + seq_printf(s, " %-25s %-10s %-10s %-15s %-15s %-10s %-10s \n"\ + "========================================================================================================================\n", + "Name (:bytes)", "pid", "mali_mem", "max_mali_mem", + "external_mem", "ump_mem", "dma_mem"); +#endif + mali_session_memory_tracking(s); + return 0; +} + +static int memory_debugfs_open(struct inode *inode, struct file *file) +{ + return single_open(file, memory_debugfs_show, inode->i_private); +} + +static const struct file_operations memory_usage_fops = { + .owner = THIS_MODULE, + .open = memory_debugfs_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static ssize_t utilization_gp_pp_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) +{ + char buf[64]; + size_t r; + u32 uval = _mali_ukk_utilization_gp_pp(); + + r = snprintf(buf, 64, "%u\n", uval); + return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); +} + +static ssize_t utilization_gp_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) +{ + char buf[64]; + size_t r; + u32 uval = _mali_ukk_utilization_gp(); + + r = snprintf(buf, 64, "%u\n", uval); + return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); +} + +static ssize_t utilization_pp_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) +{ + char buf[64]; + size_t r; + u32 uval = _mali_ukk_utilization_pp(); + + r = snprintf(buf, 64, "%u\n", uval); + return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); +} + + +static const struct file_operations utilization_gp_pp_fops = { + .owner = THIS_MODULE, + .read = utilization_gp_pp_read, +}; + +static const struct file_operations utilization_gp_fops = { + .owner = THIS_MODULE, + .read = utilization_gp_read, +}; + +static const struct file_operations utilization_pp_fops = { + .owner = THIS_MODULE, + .read = utilization_pp_read, +}; + +static ssize_t user_settings_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) +{ + unsigned long val; + int ret; + _mali_uk_user_setting_t setting; + char buf[32]; + + cnt = min(cnt, sizeof(buf) - 1); + if (copy_from_user(buf, ubuf, cnt)) { + return -EFAULT; + } + buf[cnt] = '\0'; + + ret = kstrtoul(buf, 10, &val); + if (0 != ret) { + return ret; + } + + /* Update setting */ + setting = (_mali_uk_user_setting_t)(filp->private_data); + mali_set_user_setting(setting, val); + + *ppos += cnt; + return cnt; +} + +static ssize_t user_settings_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) +{ + char buf[64]; + size_t r; + u32 value; + _mali_uk_user_setting_t setting; + + setting = (_mali_uk_user_setting_t)(filp->private_data); + value = mali_get_user_setting(setting); + + r = snprintf(buf, 64, "%u\n", value); + return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); +} + +static const struct file_operations user_settings_fops = { + .owner = THIS_MODULE, + .open = open_copy_private_data, + .read = user_settings_read, + .write = user_settings_write, +}; + +static int mali_sysfs_user_settings_register(void) +{ + struct dentry *mali_user_settings_dir = debugfs_create_dir("userspace_settings", mali_debugfs_dir); + + if (mali_user_settings_dir != NULL) { + long i; + for (i = 0; i < _MALI_UK_USER_SETTING_MAX; i++) { + debugfs_create_file(_mali_uk_user_setting_descriptions[i], + 0600, mali_user_settings_dir, (void *)i, + &user_settings_fops); + } + } + + return 0; +} + +static ssize_t pp_num_cores_enabled_write(struct file *filp, const char __user *buf, size_t count, loff_t *offp) +{ + int ret; + char buffer[32]; + unsigned long val; + + if (count >= sizeof(buffer)) { + return -ENOMEM; + } + + if (copy_from_user(&buffer[0], buf, count)) { + return -EFAULT; + } + buffer[count] = '\0'; + + ret = kstrtoul(&buffer[0], 10, &val); + if (0 != ret) { + return -EINVAL; + } + + ret = mali_executor_set_perf_level(val, MALI_TRUE); /* override even if core scaling is disabled */ + if (ret) { + return ret; + } + + *offp += count; + return count; +} + +static ssize_t pp_num_cores_enabled_read(struct file *filp, char __user *buf, size_t count, loff_t *offp) +{ + int r; + char buffer[64]; + + r = snprintf(buffer, 64, "%u\n", mali_executor_get_num_cores_enabled()); + + return simple_read_from_buffer(buf, count, offp, buffer, r); +} + +static const struct file_operations pp_num_cores_enabled_fops = { + .owner = THIS_MODULE, + .write = pp_num_cores_enabled_write, + .read = pp_num_cores_enabled_read, + .llseek = default_llseek, +}; + +static ssize_t pp_num_cores_total_read(struct file *filp, char __user *buf, size_t count, loff_t *offp) +{ + int r; + char buffer[64]; + + r = snprintf(buffer, 64, "%u\n", mali_executor_get_num_cores_total()); + + return simple_read_from_buffer(buf, count, offp, buffer, r); +} + +static const struct file_operations pp_num_cores_total_fops = { + .owner = THIS_MODULE, + .read = pp_num_cores_total_read, +}; + +static ssize_t pp_core_scaling_enabled_write(struct file *filp, const char __user *buf, size_t count, loff_t *offp) +{ + int ret; + char buffer[32]; + unsigned long val; + + if (count >= sizeof(buffer)) { + return -ENOMEM; + } + + if (copy_from_user(&buffer[0], buf, count)) { + return -EFAULT; + } + buffer[count] = '\0'; + + ret = kstrtoul(&buffer[0], 10, &val); + if (0 != ret) { + return -EINVAL; + } + + switch (val) { + case 1: + mali_executor_core_scaling_enable(); + break; + case 0: + mali_executor_core_scaling_disable(); + break; + default: + return -EINVAL; + break; + } + + *offp += count; + return count; +} + +static ssize_t pp_core_scaling_enabled_read(struct file *filp, char __user *buf, size_t count, loff_t *offp) +{ + return simple_read_from_buffer(buf, count, offp, mali_executor_core_scaling_is_enabled() ? "1\n" : "0\n", 2); +} +static const struct file_operations pp_core_scaling_enabled_fops = { + .owner = THIS_MODULE, + .write = pp_core_scaling_enabled_write, + .read = pp_core_scaling_enabled_read, + .llseek = default_llseek, +}; + +static ssize_t version_read(struct file *filp, char __user *buf, size_t count, loff_t *offp) +{ + int r = 0; + char buffer[64]; + + switch (mali_kernel_core_get_product_id()) { + case _MALI_PRODUCT_ID_MALI200: + r = snprintf(buffer, 64, "Mali-200\n"); + break; + case _MALI_PRODUCT_ID_MALI300: + r = snprintf(buffer, 64, "Mali-300\n"); + break; + case _MALI_PRODUCT_ID_MALI400: + r = snprintf(buffer, 64, "Mali-400 MP\n"); + break; + case _MALI_PRODUCT_ID_MALI450: + r = snprintf(buffer, 64, "Mali-450 MP\n"); + break; + case _MALI_PRODUCT_ID_MALI470: + r = snprintf(buffer, 64, "Mali-470 MP\n"); + break; + case _MALI_PRODUCT_ID_UNKNOWN: + return -EINVAL; + break; + }; + + return simple_read_from_buffer(buf, count, offp, buffer, r); +} + +static const struct file_operations version_fops = { + .owner = THIS_MODULE, + .read = version_read, +}; + +#if defined(DEBUG) +static int timeline_debugfs_show(struct seq_file *s, void *private_data) +{ + struct mali_session_data *session, *tmp; + u32 session_seq = 1; + + seq_printf(s, "timeline system info: \n=================\n\n"); + + mali_session_lock(); + MALI_SESSION_FOREACH(session, tmp, link) { + seq_printf(s, "session %d <%p> start:\n", session_seq, session); + mali_timeline_debug_print_system(session->timeline_system, s); + seq_printf(s, "session %d end\n\n\n", session_seq++); + } + mali_session_unlock(); + + return 0; +} + +static int timeline_debugfs_open(struct inode *inode, struct file *file) +{ + return single_open(file, timeline_debugfs_show, inode->i_private); +} + +static const struct file_operations timeline_dump_fops = { + .owner = THIS_MODULE, + .open = timeline_debugfs_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release +}; +#endif + +int mali_sysfs_register(const char *mali_dev_name) +{ + mali_debugfs_dir = debugfs_create_dir(mali_dev_name, NULL); + if (ERR_PTR(-ENODEV) == mali_debugfs_dir) { + /* Debugfs not supported. */ + mali_debugfs_dir = NULL; + } else { + if (NULL != mali_debugfs_dir) { + /* Debugfs directory created successfully; create files now */ + struct dentry *mali_power_dir; + struct dentry *mali_gp_dir; + struct dentry *mali_pp_dir; + struct dentry *mali_l2_dir; + struct dentry *mali_profiling_dir; + + debugfs_create_file("version", 0400, mali_debugfs_dir, NULL, &version_fops); + + mali_power_dir = debugfs_create_dir("power", mali_debugfs_dir); + if (mali_power_dir != NULL) { + debugfs_create_file("always_on", 0600, mali_power_dir, NULL, &power_always_on_fops); + debugfs_create_file("power_events", 0200, mali_power_dir, NULL, &power_power_events_fops); + } + + mali_gp_dir = debugfs_create_dir("gp", mali_debugfs_dir); + if (mali_gp_dir != NULL) { + u32 num_groups; + long i; + + num_groups = mali_group_get_glob_num_groups(); + for (i = 0; i < num_groups; i++) { + struct mali_group *group = mali_group_get_glob_group(i); + + struct mali_gp_core *gp_core = mali_group_get_gp_core(group); + if (NULL != gp_core) { + struct dentry *mali_gp_gpx_dir; + mali_gp_gpx_dir = debugfs_create_dir("gp0", mali_gp_dir); + if (NULL != mali_gp_gpx_dir) { + debugfs_create_file("base_addr", 0400, mali_gp_gpx_dir, &gp_core->hw_core, &hw_core_base_addr_fops); + debugfs_create_file("enabled", 0600, mali_gp_gpx_dir, group, &group_enabled_fops); + } + break; /* no need to look for any other GP cores */ + } + + } + } + + mali_pp_dir = debugfs_create_dir("pp", mali_debugfs_dir); + if (mali_pp_dir != NULL) { + u32 num_groups; + long i; + + debugfs_create_file("num_cores_total", 0400, mali_pp_dir, NULL, &pp_num_cores_total_fops); + debugfs_create_file("num_cores_enabled", 0600, mali_pp_dir, NULL, &pp_num_cores_enabled_fops); + debugfs_create_file("core_scaling_enabled", 0600, mali_pp_dir, NULL, &pp_core_scaling_enabled_fops); + + num_groups = mali_group_get_glob_num_groups(); + for (i = 0; i < num_groups; i++) { + struct mali_group *group = mali_group_get_glob_group(i); + + struct mali_pp_core *pp_core = mali_group_get_pp_core(group); + if (NULL != pp_core) { + char buf[16]; + struct dentry *mali_pp_ppx_dir; + _mali_osk_snprintf(buf, sizeof(buf), "pp%u", mali_pp_core_get_id(pp_core)); + mali_pp_ppx_dir = debugfs_create_dir(buf, mali_pp_dir); + if (NULL != mali_pp_ppx_dir) { + debugfs_create_file("base_addr", 0400, mali_pp_ppx_dir, &pp_core->hw_core, &hw_core_base_addr_fops); + if (!mali_group_is_virtual(group)) { + debugfs_create_file("enabled", 0600, mali_pp_ppx_dir, group, &group_enabled_fops); + } + } + } + } + } + + mali_l2_dir = debugfs_create_dir("l2", mali_debugfs_dir); + if (mali_l2_dir != NULL) { + struct dentry *mali_l2_all_dir; + u32 l2_id; + struct mali_l2_cache_core *l2_cache; + + mali_l2_all_dir = debugfs_create_dir("all", mali_l2_dir); + if (mali_l2_all_dir != NULL) { + debugfs_create_file("counter_src0", 0200, mali_l2_all_dir, NULL, &l2_all_counter_src0_fops); + debugfs_create_file("counter_src1", 0200, mali_l2_all_dir, NULL, &l2_all_counter_src1_fops); + } + + l2_id = 0; + l2_cache = mali_l2_cache_core_get_glob_l2_core(l2_id); + while (NULL != l2_cache) { + char buf[16]; + struct dentry *mali_l2_l2x_dir; + _mali_osk_snprintf(buf, sizeof(buf), "l2%u", l2_id); + mali_l2_l2x_dir = debugfs_create_dir(buf, mali_l2_dir); + if (NULL != mali_l2_l2x_dir) { + debugfs_create_file("counter_src0", 0600, mali_l2_l2x_dir, l2_cache, &l2_l2x_counter_src0_fops); + debugfs_create_file("counter_src1", 0600, mali_l2_l2x_dir, l2_cache, &l2_l2x_counter_src1_fops); + debugfs_create_file("counter_val0", 0600, mali_l2_l2x_dir, l2_cache, &l2_l2x_counter_val0_fops); + debugfs_create_file("counter_val1", 0600, mali_l2_l2x_dir, l2_cache, &l2_l2x_counter_val1_fops); + debugfs_create_file("base_addr", 0400, mali_l2_l2x_dir, &l2_cache->hw_core, &hw_core_base_addr_fops); + } + + /* try next L2 */ + l2_id++; + l2_cache = mali_l2_cache_core_get_glob_l2_core(l2_id); + } + } + + debugfs_create_file("gpu_memory", 0444, mali_debugfs_dir, NULL, &memory_usage_fops); + + debugfs_create_file("utilization_gp_pp", 0400, mali_debugfs_dir, NULL, &utilization_gp_pp_fops); + debugfs_create_file("utilization_gp", 0400, mali_debugfs_dir, NULL, &utilization_gp_fops); + debugfs_create_file("utilization_pp", 0400, mali_debugfs_dir, NULL, &utilization_pp_fops); + + mali_profiling_dir = debugfs_create_dir("profiling", mali_debugfs_dir); + if (mali_profiling_dir != NULL) { + u32 max_sub_jobs; + long i; + struct dentry *mali_profiling_gp_dir; + struct dentry *mali_profiling_pp_dir; +#if defined(CONFIG_MALI400_INTERNAL_PROFILING) + struct dentry *mali_profiling_proc_dir; +#endif + /* + * Create directory where we can set GP HW counters. + */ + mali_profiling_gp_dir = debugfs_create_dir("gp", mali_profiling_dir); + if (mali_profiling_gp_dir != NULL) { + debugfs_create_file("counter_src0", 0600, mali_profiling_gp_dir, (void *)PRIVATE_DATA_COUNTER_MAKE_GP(0), &profiling_counter_src_fops); + debugfs_create_file("counter_src1", 0600, mali_profiling_gp_dir, (void *)PRIVATE_DATA_COUNTER_MAKE_GP(1), &profiling_counter_src_fops); + } + + /* + * Create directory where we can set PP HW counters. + * Possible override with specific HW counters for a particular sub job + * (Disable core scaling before using the override!) + */ + mali_profiling_pp_dir = debugfs_create_dir("pp", mali_profiling_dir); + if (mali_profiling_pp_dir != NULL) { + debugfs_create_file("counter_src0", 0600, mali_profiling_pp_dir, (void *)PRIVATE_DATA_COUNTER_MAKE_PP(0), &profiling_counter_src_fops); + debugfs_create_file("counter_src1", 0600, mali_profiling_pp_dir, (void *)PRIVATE_DATA_COUNTER_MAKE_PP(1), &profiling_counter_src_fops); + } + + max_sub_jobs = mali_executor_get_num_cores_total(); + for (i = 0; i < max_sub_jobs; i++) { + char buf[16]; + struct dentry *mali_profiling_pp_x_dir; + _mali_osk_snprintf(buf, sizeof(buf), "%u", i); + mali_profiling_pp_x_dir = debugfs_create_dir(buf, mali_profiling_pp_dir); + if (NULL != mali_profiling_pp_x_dir) { + debugfs_create_file("counter_src0", + 0600, mali_profiling_pp_x_dir, + (void *)PRIVATE_DATA_COUNTER_MAKE_PP_SUB_JOB(0, i), + &profiling_counter_src_fops); + debugfs_create_file("counter_src1", + 0600, mali_profiling_pp_x_dir, + (void *)PRIVATE_DATA_COUNTER_MAKE_PP_SUB_JOB(1, i), + &profiling_counter_src_fops); + } + } + +#if defined(CONFIG_MALI400_INTERNAL_PROFILING) + mali_profiling_proc_dir = debugfs_create_dir("proc", mali_profiling_dir); + if (mali_profiling_proc_dir != NULL) { + struct dentry *mali_profiling_proc_default_dir = debugfs_create_dir("default", mali_profiling_proc_dir); + if (mali_profiling_proc_default_dir != NULL) { + debugfs_create_file("enable", 0600, mali_profiling_proc_default_dir, (void *)_MALI_UK_USER_SETTING_SW_EVENTS_ENABLE, &user_settings_fops); + } + } + debugfs_create_file("record", 0600, mali_profiling_dir, NULL, &profiling_record_fops); + debugfs_create_file("events", 0400, mali_profiling_dir, NULL, &profiling_events_fops); + debugfs_create_file("events_human_readable", 0400, mali_profiling_dir, NULL, &profiling_events_human_readable_fops); +#endif + } + +#if MALI_STATE_TRACKING + debugfs_create_file("state_dump", 0400, mali_debugfs_dir, NULL, &mali_seq_internal_state_fops); +#endif + +#if defined(DEBUG) + debugfs_create_file("timeline_dump", 0400, mali_debugfs_dir, NULL, &timeline_dump_fops); +#endif + if (mali_sysfs_user_settings_register()) { + /* Failed to create the debugfs entries for the user settings DB. */ + MALI_DEBUG_PRINT(2, ("Failed to create user setting debugfs files. Ignoring...\n")); + } + } + } + + /* Success! */ + return 0; +} + +int mali_sysfs_unregister(void) +{ + if (NULL != mali_debugfs_dir) { + debugfs_remove_recursive(mali_debugfs_dir); + } + return 0; +} + +#else /* MALI_LICENSE_IS_GPL */ + +/* Dummy implementations for non-GPL */ + +int mali_sysfs_register(struct mali_dev *device, dev_t dev, const char *mali_dev_name) +{ + return 0; +} + +int mali_sysfs_unregister(void) +{ + return 0; +} + +#endif /* MALI_LICENSE_IS_GPL */ diff --git a/drivers/gpu/arm/utgard/linux/mali_kernel_sysfs.h b/drivers/gpu/arm/utgard/linux/mali_kernel_sysfs.h new file mode 100644 index 00000000000000..a36a0cea997278 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_kernel_sysfs.h @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2011-2013, 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_KERNEL_SYSFS_H__ +#define __MALI_KERNEL_SYSFS_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#define MALI_PROC_DIR "driver/mali" + +int mali_sysfs_register(const char *mali_dev_name); +int mali_sysfs_unregister(void); + +#ifdef __cplusplus +} +#endif + +#endif /* __MALI_KERNEL_LINUX_H__ */ diff --git a/drivers/gpu/arm/utgard/linux/mali_linux_trace.h b/drivers/gpu/arm/utgard/linux/mali_linux_trace.h new file mode 100644 index 00000000000000..c6cd2bfb721790 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_linux_trace.h @@ -0,0 +1,162 @@ +/* + * Copyright (C) 2012-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#if !defined (MALI_LINUX_TRACE_H) || defined (TRACE_HEADER_MULTI_READ) +#define MALI_LINUX_TRACE_H + +#include + +#include +#include + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM mali +#define TRACE_SYSTEM_STRING __stringfy(TRACE_SYSTEM) + +#define TRACE_INCLUDE_PATH . +#define TRACE_INCLUDE_FILE mali_linux_trace + +/** + * Define the tracepoint used to communicate the status of a GPU. Called + * when a GPU turns on or turns off. + * + * @param event_id The type of the event. This parameter is a bitfield + * encoding the type of the event. + * + * @param d0 First data parameter. + * @param d1 Second data parameter. + * @param d2 Third data parameter. + * @param d3 Fourth data parameter. + * @param d4 Fifth data parameter. + */ +TRACE_EVENT(mali_timeline_event, + + TP_PROTO(unsigned int event_id, unsigned int d0, unsigned int d1, + unsigned int d2, unsigned int d3, unsigned int d4), + + TP_ARGS(event_id, d0, d1, d2, d3, d4), + + TP_STRUCT__entry( + __field(unsigned int, event_id) + __field(unsigned int, d0) + __field(unsigned int, d1) + __field(unsigned int, d2) + __field(unsigned int, d3) + __field(unsigned int, d4) + ), + + TP_fast_assign( + __entry->event_id = event_id; + __entry->d0 = d0; + __entry->d1 = d1; + __entry->d2 = d2; + __entry->d3 = d3; + __entry->d4 = d4; + ), + + TP_printk("event=%d", __entry->event_id) + ); + +/** + * Define a tracepoint used to regsiter the value of a hardware counter. + * Hardware counters belonging to the vertex or fragment processor are + * reported via this tracepoint each frame, whilst L2 cache hardware + * counters are reported continuously. + * + * @param counter_id The counter ID. + * @param value The value of the counter. + */ +TRACE_EVENT(mali_hw_counter, + + TP_PROTO(unsigned int counter_id, unsigned int value), + + TP_ARGS(counter_id, value), + + TP_STRUCT__entry( + __field(unsigned int, counter_id) + __field(unsigned int, value) + ), + + TP_fast_assign( + __entry->counter_id = counter_id; + ), + + TP_printk("event %d = %d", __entry->counter_id, __entry->value) + ); + +/** + * Define a tracepoint used to send a bundle of software counters. + * + * @param counters The bundle of counters. + */ +TRACE_EVENT(mali_sw_counters, + + TP_PROTO(pid_t pid, pid_t tid, void *surface_id, unsigned int *counters), + + TP_ARGS(pid, tid, surface_id, counters), + + TP_STRUCT__entry( + __field(pid_t, pid) + __field(pid_t, tid) + __field(void *, surface_id) + __field(unsigned int *, counters) + ), + + TP_fast_assign( + __entry->pid = pid; + __entry->tid = tid; + __entry->surface_id = surface_id; + __entry->counters = counters; + ), + + TP_printk("counters were %s", __entry->counters == NULL ? "NULL" : "not NULL") + ); + +/** + * Define a tracepoint used to gather core activity for systrace + * @param pid The process id for which the core activity originates from + * @param active If the core is active (1) or not (0) + * @param core_type The type of core active, either GP (1) or PP (0) + * @param core_id The core id that is active for the core_type + * @param frame_builder_id The frame builder id associated with this core activity + * @param flush_id The flush id associated with this core activity + */ +TRACE_EVENT(mali_core_active, + + TP_PROTO(pid_t pid, unsigned int active, unsigned int core_type, unsigned int core_id, unsigned int frame_builder_id, unsigned int flush_id), + + TP_ARGS(pid, active, core_type, core_id, frame_builder_id, flush_id), + + TP_STRUCT__entry( + __field(pid_t, pid) + __field(unsigned int, active) + __field(unsigned int, core_type) + __field(unsigned int, core_id) + __field(unsigned int, frame_builder_id) + __field(unsigned int, flush_id) + ), + + TP_fast_assign( + __entry->pid = pid; + __entry->active = active; + __entry->core_type = core_type; + __entry->core_id = core_id; + __entry->frame_builder_id = frame_builder_id; + __entry->flush_id = flush_id; + ), + + TP_printk("%s|%d|%s%i:%x|%d", __entry->active ? "S" : "F", __entry->pid, __entry->core_type ? "GP" : "PP", __entry->core_id, __entry->flush_id, __entry->frame_builder_id) + ); + +#endif /* MALI_LINUX_TRACE_H */ + +/* This part must exist outside the header guard. */ +#include + diff --git a/drivers/gpu/arm/utgard/linux/mali_memory.c b/drivers/gpu/arm/utgard/linux/mali_memory.c new file mode 100644 index 00000000000000..c45f6ee258878a --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_memory.c @@ -0,0 +1,510 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mali_osk.h" +#include "mali_executor.h" + +#include "mali_memory.h" +#include "mali_memory_os_alloc.h" +#include "mali_memory_block_alloc.h" +#include "mali_memory_util.h" +#include "mali_memory_virtual.h" +#include "mali_memory_manager.h" +#include "mali_memory_cow.h" +#include "mali_memory_swap_alloc.h" +#include "mali_memory_defer_bind.h" + +extern unsigned int mali_dedicated_mem_size; +extern unsigned int mali_shared_mem_size; + +#define MALI_VM_NUM_FAULT_PREFETCH (0x8) + +static void mali_mem_vma_open(struct vm_area_struct *vma) +{ + mali_mem_allocation *alloc = (mali_mem_allocation *)vma->vm_private_data; + MALI_DEBUG_PRINT(4, ("Open called on vma %p\n", vma)); + + /* If need to share the allocation, add ref_count here */ + mali_allocation_ref(alloc); + return; +} +static void mali_mem_vma_close(struct vm_area_struct *vma) +{ + /* If need to share the allocation, unref ref_count here */ + mali_mem_allocation *alloc = (mali_mem_allocation *)vma->vm_private_data; + + mali_allocation_unref(&alloc); + vma->vm_private_data = NULL; +} + +static int mali_mem_vma_fault(struct vm_area_struct *vma, struct vm_fault *vmf) +{ + mali_mem_allocation *alloc = (mali_mem_allocation *)vma->vm_private_data; + mali_mem_backend *mem_bkend = NULL; + int ret; + int prefetch_num = MALI_VM_NUM_FAULT_PREFETCH; + + unsigned long address = (unsigned long)vmf->virtual_address; + MALI_DEBUG_ASSERT(alloc->backend_handle); + MALI_DEBUG_ASSERT((unsigned long)alloc->cpu_mapping.addr <= address); + + /* Get backend memory & Map on CPU */ + mutex_lock(&mali_idr_mutex); + if (!(mem_bkend = idr_find(&mali_backend_idr, alloc->backend_handle))) { + MALI_DEBUG_PRINT(1, ("Can't find memory backend in mmap!\n")); + mutex_unlock(&mali_idr_mutex); + return VM_FAULT_SIGBUS; + } + mutex_unlock(&mali_idr_mutex); + MALI_DEBUG_ASSERT(mem_bkend->type == alloc->type); + + if ((mem_bkend->type == MALI_MEM_COW && (MALI_MEM_BACKEND_FLAG_SWAP_COWED != + (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED))) && + (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_COW_CPU_NO_WRITE)) { + /*check if use page fault to do COW*/ + MALI_DEBUG_PRINT(4, ("mali_vma_fault: do cow allocate on demand!, address=0x%x\n", address)); + mutex_lock(&mem_bkend->mutex); + ret = mali_mem_cow_allocate_on_demand(mem_bkend, + (address - vma->vm_start) / PAGE_SIZE); + mutex_unlock(&mem_bkend->mutex); + + if (ret != _MALI_OSK_ERR_OK) { + return VM_FAULT_OOM; + } + prefetch_num = 1; + + /* handle COW modified range cpu mapping + we zap the mapping in cow_modify_range, it will trigger page fault + when CPU access it, so here we map it to CPU*/ + mutex_lock(&mem_bkend->mutex); + ret = mali_mem_cow_cpu_map_pages_locked(mem_bkend, vma, address, prefetch_num); + mutex_unlock(&mem_bkend->mutex); + + if (unlikely(ret != _MALI_OSK_ERR_OK)) { + return VM_FAULT_SIGBUS; + } + } else if ((mem_bkend->type == MALI_MEM_SWAP) || + (mem_bkend->type == MALI_MEM_COW && (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED))) { + u32 offset_in_bkend = (address - vma->vm_start) / PAGE_SIZE; + int ret = _MALI_OSK_ERR_OK; + + mutex_lock(&mem_bkend->mutex); + if (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_COW_CPU_NO_WRITE) { + ret = mali_mem_swap_cow_page_on_demand(mem_bkend, offset_in_bkend, &vmf->page); + } else { + ret = mali_mem_swap_allocate_page_on_demand(mem_bkend, offset_in_bkend, &vmf->page); + } + mutex_unlock(&mem_bkend->mutex); + + if (ret != _MALI_OSK_ERR_OK) { + MALI_DEBUG_PRINT(2, ("Mali swap memory page fault process failed, address=0x%x\n", address)); + return VM_FAULT_OOM; + } else { + return VM_FAULT_LOCKED; + } + } else { + MALI_DEBUG_ASSERT(0); + /*NOT support yet*/ + } + return VM_FAULT_NOPAGE; +} + +static struct vm_operations_struct mali_kernel_vm_ops = { + .open = mali_mem_vma_open, + .close = mali_mem_vma_close, + .fault = mali_mem_vma_fault, +}; + + +/** @ map mali allocation to CPU address +* +* Supported backend types: +* --MALI_MEM_OS +* -- need to add COW? + *Not supported backend types: +* -_MALI_MEMORY_BIND_BACKEND_UMP +* -_MALI_MEMORY_BIND_BACKEND_DMA_BUF +* -_MALI_MEMORY_BIND_BACKEND_EXTERNAL_MEMORY +* +*/ +int mali_mmap(struct file *filp, struct vm_area_struct *vma) +{ + struct mali_session_data *session; + mali_mem_allocation *mali_alloc = NULL; + u32 mali_addr = vma->vm_pgoff << PAGE_SHIFT; + struct mali_vma_node *mali_vma_node = NULL; + mali_mem_backend *mem_bkend = NULL; + int ret = -EFAULT; + + session = (struct mali_session_data *)filp->private_data; + if (NULL == session) { + MALI_PRINT_ERROR(("mmap called without any session data available\n")); + return -EFAULT; + } + + MALI_DEBUG_PRINT(4, ("MMap() handler: start=0x%08X, phys=0x%08X, size=0x%08X vma->flags 0x%08x\n", + (unsigned int)vma->vm_start, (unsigned int)(vma->vm_pgoff << PAGE_SHIFT), + (unsigned int)(vma->vm_end - vma->vm_start), vma->vm_flags)); + + /* Operations used on any memory system */ + /* do not need to anything in vm open/close now */ + + /* find mali allocation structure by vaddress*/ + mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0); + if (likely(mali_vma_node)) { + mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); + MALI_DEBUG_ASSERT(mali_addr == mali_vma_node->vm_node.start); + if (unlikely(mali_addr != mali_vma_node->vm_node.start)) { + /* only allow to use start address for mmap */ + MALI_DEBUG_PRINT(1, ("mali_addr != mali_vma_node->vm_node.start\n")); + return -EFAULT; + } + } else { + MALI_DEBUG_ASSERT(NULL == mali_vma_node); + return -EFAULT; + } + + mali_alloc->cpu_mapping.addr = (void __user *)vma->vm_start; + + if (mali_alloc->flags & _MALI_MEMORY_ALLOCATE_DEFER_BIND) { + MALI_DEBUG_PRINT(1, ("ERROR : trying to access varying memory by CPU!\n")); + return -EFAULT; + } + + /* Get backend memory & Map on CPU */ + mutex_lock(&mali_idr_mutex); + if (!(mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle))) { + MALI_DEBUG_PRINT(1, ("Can't find memory backend in mmap!\n")); + mutex_unlock(&mali_idr_mutex); + return -EFAULT; + } + mutex_unlock(&mali_idr_mutex); + + if (!(MALI_MEM_SWAP == mali_alloc->type || + (MALI_MEM_COW == mali_alloc->type && (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED)))) { + /* Set some bits which indicate that, the memory is IO memory, meaning + * that no paging is to be performed and the memory should not be + * included in crash dumps. And that the memory is reserved, meaning + * that it's present and can never be paged out (see also previous + * entry) + */ + vma->vm_flags |= VM_IO; + vma->vm_flags |= VM_DONTCOPY; + vma->vm_flags |= VM_PFNMAP; +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 7, 0) + vma->vm_flags |= VM_RESERVED; +#else + vma->vm_flags |= VM_DONTDUMP; + vma->vm_flags |= VM_DONTEXPAND; +#endif + } else if (MALI_MEM_SWAP == mali_alloc->type) { + vma->vm_pgoff = mem_bkend->start_idx; + } + + vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); + vma->vm_ops = &mali_kernel_vm_ops; + + mali_alloc->cpu_mapping.addr = (void __user *)vma->vm_start; + + /* If it's a copy-on-write mapping, map to read only */ + if (!(vma->vm_flags & VM_WRITE)) { + MALI_DEBUG_PRINT(4, ("mmap allocation with read only !\n")); + /* add VM_WRITE for do_page_fault will check this when a write fault */ + vma->vm_flags |= VM_WRITE | VM_READ; + vma->vm_page_prot = PAGE_READONLY; + vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); + mem_bkend->flags |= MALI_MEM_BACKEND_FLAG_COW_CPU_NO_WRITE; + goto out; + } + + if (mem_bkend->type == MALI_MEM_OS) { + ret = mali_mem_os_cpu_map(mem_bkend, vma); + } else if (mem_bkend->type == MALI_MEM_COW && + (MALI_MEM_BACKEND_FLAG_SWAP_COWED != (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED))) { + ret = mali_mem_cow_cpu_map(mem_bkend, vma); + } else if (mem_bkend->type == MALI_MEM_BLOCK) { + ret = mali_mem_block_cpu_map(mem_bkend, vma); + } else if ((mem_bkend->type == MALI_MEM_SWAP) || (mem_bkend->type == MALI_MEM_COW && + (MALI_MEM_BACKEND_FLAG_SWAP_COWED == (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED)))) { + /*For swappable memory, CPU page table will be created by page fault handler. */ + ret = 0; + } else { + /* Not support yet*/ + MALI_DEBUG_ASSERT(0); + } + + if (ret != 0) { + MALI_DEBUG_PRINT(1, ("ret != 0\n")); + return -EFAULT; + } +out: + MALI_DEBUG_ASSERT(MALI_MEM_ALLOCATION_VALID_MAGIC == mali_alloc->magic); + + vma->vm_private_data = (void *)mali_alloc; + mali_alloc->cpu_mapping.vma = vma; + + mali_allocation_ref(mali_alloc); + + return 0; +} + +_mali_osk_errcode_t mali_mem_mali_map_prepare(mali_mem_allocation *descriptor) +{ + u32 size = descriptor->psize; + struct mali_session_data *session = descriptor->session; + + MALI_DEBUG_ASSERT(MALI_MEM_ALLOCATION_VALID_MAGIC == descriptor->magic); + + /* Map dma-buf into this session's page tables */ + + if (descriptor->flags & MALI_MEM_FLAG_MALI_GUARD_PAGE) { + size += MALI_MMU_PAGE_SIZE; + } + + return mali_mmu_pagedir_map(session->page_directory, descriptor->mali_vma_node.vm_node.start, size); +} + +_mali_osk_errcode_t mali_mem_mali_map_resize(mali_mem_allocation *descriptor, u32 new_size) +{ + u32 old_size = descriptor->psize; + struct mali_session_data *session = descriptor->session; + + MALI_DEBUG_ASSERT(MALI_MEM_ALLOCATION_VALID_MAGIC == descriptor->magic); + + if (descriptor->flags & MALI_MEM_FLAG_MALI_GUARD_PAGE) { + new_size += MALI_MMU_PAGE_SIZE; + } + + if (new_size > old_size) { + MALI_DEBUG_ASSERT(new_size <= descriptor->mali_vma_node.vm_node.size); + return mali_mmu_pagedir_map(session->page_directory, descriptor->mali_vma_node.vm_node.start + old_size, new_size - old_size); + } + return _MALI_OSK_ERR_OK; +} + +void mali_mem_mali_map_free(struct mali_session_data *session, u32 size, mali_address_t vaddr, u32 flags) +{ + if (flags & MALI_MEM_FLAG_MALI_GUARD_PAGE) { + size += MALI_MMU_PAGE_SIZE; + } + + /* Umap and flush L2 */ + mali_mmu_pagedir_unmap(session->page_directory, vaddr, size); + mali_executor_zap_all_active(session); +} + +u32 _mali_ukk_report_memory_usage(void) +{ + u32 sum = 0; + + if (MALI_TRUE == mali_memory_have_dedicated_memory()) { + sum += mali_mem_block_allocator_stat(); + } + + sum += mali_mem_os_stat(); + + return sum; +} + +u32 _mali_ukk_report_total_memory_size(void) +{ + return mali_dedicated_mem_size + mali_shared_mem_size; +} + + +/** + * Per-session memory descriptor mapping table sizes + */ +#define MALI_MEM_DESCRIPTORS_INIT 64 +#define MALI_MEM_DESCRIPTORS_MAX 65536 + +_mali_osk_errcode_t mali_memory_session_begin(struct mali_session_data *session_data) +{ + MALI_DEBUG_PRINT(5, ("Memory session begin\n")); + + session_data->memory_lock = _mali_osk_mutex_init(_MALI_OSK_LOCKFLAG_ORDERED, + _MALI_OSK_LOCK_ORDER_MEM_SESSION); + + if (NULL == session_data->memory_lock) { + MALI_ERROR(_MALI_OSK_ERR_FAULT); + } + + mali_memory_manager_init(&session_data->allocation_mgr); + + MALI_DEBUG_PRINT(5, ("MMU session begin: success\n")); + MALI_SUCCESS; +} + +void mali_memory_session_end(struct mali_session_data *session) +{ + MALI_DEBUG_PRINT(3, ("MMU session end\n")); + + if (NULL == session) { + MALI_DEBUG_PRINT(1, ("No session data found during session end\n")); + return; + } + /* free allocation */ + mali_free_session_allocations(session); + /* do some check in unint*/ + mali_memory_manager_uninit(&session->allocation_mgr); + + /* Free the lock */ + _mali_osk_mutex_term(session->memory_lock); + + return; +} + +_mali_osk_errcode_t mali_memory_initialize(void) +{ + _mali_osk_errcode_t err; + + idr_init(&mali_backend_idr); + mutex_init(&mali_idr_mutex); + + err = mali_mem_swap_init(); + if (err != _MALI_OSK_ERR_OK) { + return err; + } + err = mali_mem_os_init(); + if (_MALI_OSK_ERR_OK == err) { + err = mali_mem_defer_bind_manager_init(); + } + + return err; +} + +void mali_memory_terminate(void) +{ + mali_mem_swap_term(); + mali_mem_defer_bind_manager_destory(); + mali_mem_os_term(); + if (mali_memory_have_dedicated_memory()) { + mali_mem_block_allocator_destroy(); + } +} + + +struct mali_page_node *_mali_page_node_allocate(mali_page_node_type type) +{ + mali_page_node *page_node = NULL; + + page_node = kzalloc(sizeof(mali_page_node), GFP_KERNEL); + MALI_DEBUG_ASSERT(NULL != page_node); + + if (page_node) { + page_node->type = type; + INIT_LIST_HEAD(&page_node->list); + } + + return page_node; +} + +void _mali_page_node_ref(struct mali_page_node *node) +{ + if (node->type == MALI_PAGE_NODE_OS) { + /* add ref to this page */ + get_page(node->page); + } else if (node->type == MALI_PAGE_NODE_BLOCK) { + mali_mem_block_add_ref(node); + } else if (node->type == MALI_PAGE_NODE_SWAP) { + atomic_inc(&node->swap_it->ref_count); + } else + MALI_DEBUG_ASSERT(0); +} + +void _mali_page_node_unref(struct mali_page_node *node) +{ + if (node->type == MALI_PAGE_NODE_OS) { + /* unref to this page */ + put_page(node->page); + } else if (node->type == MALI_PAGE_NODE_BLOCK) { + mali_mem_block_dec_ref(node); + } else + MALI_DEBUG_ASSERT(0); +} + + +void _mali_page_node_add_page(struct mali_page_node *node, struct page *page) +{ + MALI_DEBUG_ASSERT(MALI_PAGE_NODE_OS == node->type); + node->page = page; +} + + +void _mali_page_node_add_swap_item(struct mali_page_node *node, struct mali_swap_item *item) +{ + MALI_DEBUG_ASSERT(MALI_PAGE_NODE_SWAP == node->type); + node->swap_it = item; +} + +void _mali_page_node_add_block_item(struct mali_page_node *node, mali_block_item *item) +{ + MALI_DEBUG_ASSERT(MALI_PAGE_NODE_BLOCK == node->type); + node->blk_it = item; +} + + +int _mali_page_node_get_ref_count(struct mali_page_node *node) +{ + if (node->type == MALI_PAGE_NODE_OS) { + /* get ref count of this page */ + return page_count(node->page); + } else if (node->type == MALI_PAGE_NODE_BLOCK) { + return mali_mem_block_get_ref_count(node); + } else if (node->type == MALI_PAGE_NODE_SWAP) { + return atomic_read(&node->swap_it->ref_count); + } else { + MALI_DEBUG_ASSERT(0); + } + return -1; +} + + +dma_addr_t _mali_page_node_get_dma_addr(struct mali_page_node *node) +{ + if (node->type == MALI_PAGE_NODE_OS) { + return page_private(node->page); + } else if (node->type == MALI_PAGE_NODE_BLOCK) { + return _mali_blk_item_get_phy_addr(node->blk_it); + } else if (node->type == MALI_PAGE_NODE_SWAP) { + return node->swap_it->dma_addr; + } else { + MALI_DEBUG_ASSERT(0); + } + return 0; +} + + +unsigned long _mali_page_node_get_pfn(struct mali_page_node *node) +{ + if (node->type == MALI_PAGE_NODE_OS) { + return page_to_pfn(node->page); + } else if (node->type == MALI_PAGE_NODE_BLOCK) { + /* get phy addr for BLOCK page*/ + return _mali_blk_item_get_pfn(node->blk_it); + } else if (node->type == MALI_PAGE_NODE_SWAP) { + return page_to_pfn(node->swap_it->page); + } else { + MALI_DEBUG_ASSERT(0); + } + return 0; +} + + diff --git a/drivers/gpu/arm/utgard/linux/mali_memory.h b/drivers/gpu/arm/utgard/linux/mali_memory.h new file mode 100644 index 00000000000000..3140c6c98d2c76 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_memory.h @@ -0,0 +1,143 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_MEMORY_H__ +#define __MALI_MEMORY_H__ + +#include "mali_osk.h" +#include "mali_session.h" + +#include +#include + +#include "mali_memory_types.h" +#include "mali_memory_os_alloc.h" + +_mali_osk_errcode_t mali_memory_initialize(void); +void mali_memory_terminate(void); + +/** @brief Allocate a page table page + * + * Allocate a page for use as a page directory or page table. The page is + * mapped into kernel space. + * + * @return _MALI_OSK_ERR_OK on success, otherwise an error code + * @param table_page GPU pointer to the allocated page + * @param mapping CPU pointer to the mapping of the allocated page + */ +MALI_STATIC_INLINE _mali_osk_errcode_t +mali_mmu_get_table_page(mali_dma_addr *table_page, mali_io_address *mapping) +{ + return mali_mem_os_get_table_page(table_page, mapping); +} + +/** @brief Release a page table page + * + * Release a page table page allocated through \a mali_mmu_get_table_page + * + * @param pa the GPU address of the page to release + */ +MALI_STATIC_INLINE void +mali_mmu_release_table_page(mali_dma_addr phys, void *virt) +{ + mali_mem_os_release_table_page(phys, virt); +} + +/** @brief mmap function + * + * mmap syscalls on the Mali device node will end up here. + * + * This function allocates Mali memory and maps it on CPU and Mali. + */ +int mali_mmap(struct file *filp, struct vm_area_struct *vma); + +/** @brief Start a new memory session + * + * Called when a process opens the Mali device node. + * + * @param session Pointer to session to initialize + */ +_mali_osk_errcode_t mali_memory_session_begin(struct mali_session_data *session); + +/** @brief Close a memory session + * + * Called when a process closes the Mali device node. + * + * Memory allocated by the session will be freed + * + * @param session Pointer to the session to terminate + */ +void mali_memory_session_end(struct mali_session_data *session); + +/** @brief Prepare Mali page tables for mapping + * + * This function will prepare the Mali page tables for mapping the memory + * described by \a descriptor. + * + * Page tables will be reference counted and allocated, if not yet present. + * + * @param descriptor Pointer to the memory descriptor to the mapping + */ +_mali_osk_errcode_t mali_mem_mali_map_prepare(mali_mem_allocation *descriptor); + +/** @brief Resize Mali page tables for mapping + * + * This function will Resize the Mali page tables for mapping the memory + * described by \a descriptor. + * + * Page tables will be reference counted and allocated, if not yet present. + * + * @param descriptor Pointer to the memory descriptor to the mapping + * @param new_size The new size of descriptor + */ +_mali_osk_errcode_t mali_mem_mali_map_resize(mali_mem_allocation *descriptor, u32 new_size); + +/** @brief Free Mali page tables for mapping + * + * This function will unmap pages from Mali memory and free the page tables + * that are now unused. + * + * The updated pages in the Mali L2 cache will be invalidated, and the MMU TLBs will be zapped if necessary. + * + * @param descriptor Pointer to the memory descriptor to unmap + */ +void mali_mem_mali_map_free(struct mali_session_data *session, u32 size, mali_address_t vaddr, u32 flags); + +/** @brief Parse resource and prepare the OS memory allocator + * + * @param size Maximum size to allocate for Mali GPU. + * @return _MALI_OSK_ERR_OK on success, otherwise failure. + */ +_mali_osk_errcode_t mali_memory_core_resource_os_memory(u32 size); + +/** @brief Parse resource and prepare the dedicated memory allocator + * + * @param start Physical start address of dedicated Mali GPU memory. + * @param size Size of dedicated Mali GPU memory. + * @return _MALI_OSK_ERR_OK on success, otherwise failure. + */ +_mali_osk_errcode_t mali_memory_core_resource_dedicated_memory(u32 start, u32 size); + + +struct mali_page_node *_mali_page_node_allocate(mali_page_node_type type); + +void _mali_page_node_ref(struct mali_page_node *node); +void _mali_page_node_unref(struct mali_page_node *node); +void _mali_page_node_add_page(struct mali_page_node *node, struct page *page); + +void _mali_page_node_add_block_item(struct mali_page_node *node, mali_block_item *item); + +void _mali_page_node_add_swap_item(struct mali_page_node *node, struct mali_swap_item *item); + +int _mali_page_node_get_ref_count(struct mali_page_node *node); +dma_addr_t _mali_page_node_get_dma_addr(struct mali_page_node *node); +unsigned long _mali_page_node_get_pfn(struct mali_page_node *node); + +#endif /* __MALI_MEMORY_H__ */ diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_block_alloc.c b/drivers/gpu/arm/utgard/linux/mali_memory_block_alloc.c new file mode 100644 index 00000000000000..453ddda3080f5f --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_memory_block_alloc.c @@ -0,0 +1,362 @@ +/* + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_kernel_common.h" +#include "mali_memory.h" +#include "mali_memory_block_alloc.h" +#include "mali_osk.h" +#include + + +static mali_block_allocator *mali_mem_block_gobal_allocator = NULL; + +unsigned long _mali_blk_item_get_phy_addr(mali_block_item *item) +{ + return (item->phy_addr & ~(MALI_BLOCK_REF_MASK)); +} + + +unsigned long _mali_blk_item_get_pfn(mali_block_item *item) +{ + return (item->phy_addr / MALI_BLOCK_SIZE); +} + + +u32 mali_mem_block_get_ref_count(mali_page_node *node) +{ + MALI_DEBUG_ASSERT(node->type == MALI_PAGE_NODE_BLOCK); + return (node->blk_it->phy_addr & MALI_BLOCK_REF_MASK); +} + + +/* Increase the refence count +* It not atomic, so it need to get sp_lock before call this function +*/ + +u32 mali_mem_block_add_ref(mali_page_node *node) +{ + MALI_DEBUG_ASSERT(node->type == MALI_PAGE_NODE_BLOCK); + MALI_DEBUG_ASSERT(mali_mem_block_get_ref_count(node) < MALI_BLOCK_MAX_REF_COUNT); + return (node->blk_it->phy_addr++ & MALI_BLOCK_REF_MASK); +} + +/* Decase the refence count +* It not atomic, so it need to get sp_lock before call this function +*/ +u32 mali_mem_block_dec_ref(mali_page_node *node) +{ + MALI_DEBUG_ASSERT(node->type == MALI_PAGE_NODE_BLOCK); + MALI_DEBUG_ASSERT(mali_mem_block_get_ref_count(node) > 0); + return (node->blk_it->phy_addr-- & MALI_BLOCK_REF_MASK); +} + + +static mali_block_allocator *mali_mem_block_allocator_create(u32 base_address, u32 size) +{ + mali_block_allocator *info; + u32 usable_size; + u32 num_blocks; + mali_page_node *m_node; + mali_block_item *mali_blk_items = NULL; + int i = 0; + + usable_size = size & ~(MALI_BLOCK_SIZE - 1); + MALI_DEBUG_PRINT(3, ("Mali block allocator create for region starting at 0x%08X length 0x%08X\n", base_address, size)); + MALI_DEBUG_PRINT(4, ("%d usable bytes\n", usable_size)); + num_blocks = usable_size / MALI_BLOCK_SIZE; + MALI_DEBUG_PRINT(4, ("which becomes %d blocks\n", num_blocks)); + + if (usable_size == 0) { + MALI_DEBUG_PRINT(1, ("Memory block of size %d is unusable\n", size)); + return NULL; + } + + info = _mali_osk_calloc(1, sizeof(mali_block_allocator)); + if (NULL != info) { + INIT_LIST_HEAD(&info->free); + spin_lock_init(&info->sp_lock); + info->total_num = num_blocks; + mali_blk_items = _mali_osk_calloc(1, sizeof(mali_block_item) * num_blocks); + + if (mali_blk_items) { + info->items = mali_blk_items; + /* add blocks(4k size) to free list*/ + for (i = 0 ; i < num_blocks ; i++) { + /* add block information*/ + mali_blk_items[i].phy_addr = base_address + (i * MALI_BLOCK_SIZE); + /* add to free list */ + m_node = _mali_page_node_allocate(MALI_PAGE_NODE_BLOCK); + if (m_node == NULL) + goto fail; + _mali_page_node_add_block_item(m_node, &(mali_blk_items[i])); + list_add_tail(&m_node->list, &info->free); + atomic_add(1, &info->free_num); + } + return info; + } + } +fail: + mali_mem_block_allocator_destroy(); + return NULL; +} + +void mali_mem_block_allocator_destroy(void) +{ + struct mali_page_node *m_page, *m_tmp; + mali_block_allocator *info = mali_mem_block_gobal_allocator; + MALI_DEBUG_ASSERT_POINTER(info); + MALI_DEBUG_PRINT(4, ("Memory block destroy !\n")); + + if (NULL == info) + return; + + list_for_each_entry_safe(m_page, m_tmp , &info->free, list) { + MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_BLOCK); + list_del(&m_page->list); + kfree(m_page); + } + + _mali_osk_free(info->items); + _mali_osk_free(info); +} + +u32 mali_mem_block_release(mali_mem_backend *mem_bkend) +{ + mali_mem_allocation *alloc = mem_bkend->mali_allocation; + u32 free_pages_nr = 0; + MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_BLOCK); + + /* Unmap the memory from the mali virtual address space. */ + mali_mem_block_mali_unmap(alloc); + mutex_lock(&mem_bkend->mutex); + free_pages_nr = mali_mem_block_free(&mem_bkend->block_mem); + mutex_unlock(&mem_bkend->mutex); + return free_pages_nr; +} + + +int mali_mem_block_alloc(mali_mem_block_mem *block_mem, u32 size) +{ + struct mali_page_node *m_page, *m_tmp; + size_t page_count = PAGE_ALIGN(size) / _MALI_OSK_MALI_PAGE_SIZE; + mali_block_allocator *info = mali_mem_block_gobal_allocator; + MALI_DEBUG_ASSERT_POINTER(info); + + MALI_DEBUG_PRINT(4, ("BLOCK Mem: Allocate size = 0x%x\n", size)); + /*do some init */ + INIT_LIST_HEAD(&block_mem->pfns); + + spin_lock(&info->sp_lock); + /*check if have enough space*/ + if (atomic_read(&info->free_num) > page_count) { + list_for_each_entry_safe(m_page, m_tmp , &info->free, list) { + if (page_count > 0) { + MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_BLOCK); + MALI_DEBUG_ASSERT(mali_mem_block_get_ref_count(m_page) == 0); + list_move(&m_page->list, &block_mem->pfns); + block_mem->count++; + atomic_dec(&info->free_num); + _mali_page_node_ref(m_page); + } else { + break; + } + page_count--; + } + } else { + /* can't allocate from BLOCK memory*/ + spin_unlock(&info->sp_lock); + return -1; + } + + spin_unlock(&info->sp_lock); + return 0; +} + +u32 mali_mem_block_free(mali_mem_block_mem *block_mem) +{ + u32 free_pages_nr = 0; + + free_pages_nr = mali_mem_block_free_list(&block_mem->pfns); + MALI_DEBUG_PRINT(4, ("BLOCK Mem free : allocated size = 0x%x, free size = 0x%x\n", block_mem->count * _MALI_OSK_MALI_PAGE_SIZE, + free_pages_nr * _MALI_OSK_MALI_PAGE_SIZE)); + block_mem->count = 0; + MALI_DEBUG_ASSERT(list_empty(&block_mem->pfns)); + + return free_pages_nr; +} + + +u32 mali_mem_block_free_list(struct list_head *list) +{ + struct mali_page_node *m_page, *m_tmp; + mali_block_allocator *info = mali_mem_block_gobal_allocator; + u32 free_pages_nr = 0; + + if (info) { + spin_lock(&info->sp_lock); + list_for_each_entry_safe(m_page, m_tmp , list, list) { + if (1 == _mali_page_node_get_ref_count(m_page)) { + free_pages_nr++; + } + mali_mem_block_free_node(m_page); + } + spin_unlock(&info->sp_lock); + } + return free_pages_nr; +} + +/* free the node,*/ +void mali_mem_block_free_node(struct mali_page_node *node) +{ + mali_block_allocator *info = mali_mem_block_gobal_allocator; + + /* only handle BLOCK node */ + if (node->type == MALI_PAGE_NODE_BLOCK && info) { + /*Need to make this atomic?*/ + if (1 == _mali_page_node_get_ref_count(node)) { + /*Move to free list*/ + _mali_page_node_unref(node); + list_move_tail(&node->list, &info->free); + atomic_add(1, &info->free_num); + } else { + _mali_page_node_unref(node); + list_del(&node->list); + kfree(node); + } + } +} + +/* unref the node, but not free it */ +_mali_osk_errcode_t mali_mem_block_unref_node(struct mali_page_node *node) +{ + mali_block_allocator *info = mali_mem_block_gobal_allocator; + mali_page_node *new_node; + + /* only handle BLOCK node */ + if (node->type == MALI_PAGE_NODE_BLOCK && info) { + /*Need to make this atomic?*/ + if (1 == _mali_page_node_get_ref_count(node)) { + /* allocate a new node, Add to free list, keep the old node*/ + _mali_page_node_unref(node); + new_node = _mali_page_node_allocate(MALI_PAGE_NODE_BLOCK); + if (new_node) { + memcpy(new_node, node, sizeof(mali_page_node)); + list_add(&new_node->list, &info->free); + atomic_add(1, &info->free_num); + } else + return _MALI_OSK_ERR_FAULT; + + } else { + _mali_page_node_unref(node); + } + } + return _MALI_OSK_ERR_OK; +} + + +int mali_mem_block_mali_map(mali_mem_block_mem *block_mem, struct mali_session_data *session, u32 vaddr, u32 props) +{ + struct mali_page_directory *pagedir = session->page_directory; + struct mali_page_node *m_page; + dma_addr_t phys; + u32 virt = vaddr; + u32 prop = props; + + list_for_each_entry(m_page, &block_mem->pfns, list) { + MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_BLOCK); + phys = _mali_page_node_get_dma_addr(m_page); +#if defined(CONFIG_ARCH_DMA_ADDR_T_64BIT) + /* Verify that the "physical" address is 32-bit and + * usable for Mali, when on a system with bus addresses + * wider than 32-bit. */ + MALI_DEBUG_ASSERT(0 == (phys >> 32)); +#endif + mali_mmu_pagedir_update(pagedir, virt, (mali_dma_addr)phys, MALI_MMU_PAGE_SIZE, prop); + virt += MALI_MMU_PAGE_SIZE; + } + + return 0; +} + +void mali_mem_block_mali_unmap(mali_mem_allocation *alloc) +{ + struct mali_session_data *session; + MALI_DEBUG_ASSERT_POINTER(alloc); + session = alloc->session; + MALI_DEBUG_ASSERT_POINTER(session); + + mali_session_memory_lock(session); + mali_mem_mali_map_free(session, alloc->psize, alloc->mali_vma_node.vm_node.start, + alloc->flags); + mali_session_memory_unlock(session); +} + + +int mali_mem_block_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma) +{ + int ret; + mali_mem_block_mem *block_mem = &mem_bkend->block_mem; + unsigned long addr = vma->vm_start; + struct mali_page_node *m_page; + MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_BLOCK); + + list_for_each_entry(m_page, &block_mem->pfns, list) { + MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_BLOCK); + ret = vm_insert_pfn(vma, addr, _mali_page_node_get_pfn(m_page)); + + if (unlikely(0 != ret)) { + return -EFAULT; + } + addr += _MALI_OSK_MALI_PAGE_SIZE; + + } + + return 0; +} + + +_mali_osk_errcode_t mali_memory_core_resource_dedicated_memory(u32 start, u32 size) +{ + mali_block_allocator *allocator; + + /* Do the low level linux operation first */ + + /* Request ownership of the memory */ + if (_MALI_OSK_ERR_OK != _mali_osk_mem_reqregion(start, size, "Dedicated Mali GPU memory")) { + MALI_DEBUG_PRINT(1, ("Failed to request memory region for frame buffer (0x%08X - 0x%08X)\n", start, start + size - 1)); + return _MALI_OSK_ERR_FAULT; + } + + /* Create generic block allocator object to handle it */ + allocator = mali_mem_block_allocator_create(start, size); + + if (NULL == allocator) { + MALI_DEBUG_PRINT(1, ("Memory bank registration failed\n")); + _mali_osk_mem_unreqregion(start, size); + MALI_ERROR(_MALI_OSK_ERR_FAULT); + } + + mali_mem_block_gobal_allocator = (mali_block_allocator *)allocator; + + return _MALI_OSK_ERR_OK; +} + +mali_bool mali_memory_have_dedicated_memory(void) +{ + return mali_mem_block_gobal_allocator ? MALI_TRUE : MALI_FALSE; +} + +u32 mali_mem_block_allocator_stat(void) +{ + mali_block_allocator *allocator = mali_mem_block_gobal_allocator; + MALI_DEBUG_ASSERT_POINTER(allocator); + + return (allocator->total_num - atomic_read(&allocator->free_num)) * _MALI_OSK_MALI_PAGE_SIZE; +} diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_block_alloc.h b/drivers/gpu/arm/utgard/linux/mali_memory_block_alloc.h new file mode 100644 index 00000000000000..129434de67df2f --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_memory_block_alloc.h @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2010, 2013, 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_BLOCK_ALLOCATOR_H__ +#define __MALI_BLOCK_ALLOCATOR_H__ + +#include "mali_session.h" +#include "mali_memory.h" +#include + +#include "mali_memory_types.h" + +#define MALI_BLOCK_SIZE (PAGE_SIZE) /* 4 kB, manage BLOCK memory as page size */ +#define MALI_BLOCK_REF_MASK (0xFFF) +#define MALI_BLOCK_MAX_REF_COUNT (0xFFF) + + + +typedef struct mali_block_allocator { + /* + * In free list, each node's ref_count is 0, + * ref_count added when allocated or referenced in COW + */ + mali_block_item *items; /* information for each block item*/ + struct list_head free; /*free list of mali_memory_node*/ + spinlock_t sp_lock; /*lock for reference count & free list opertion*/ + u32 total_num; /* Number of total pages*/ + atomic_t free_num; /*number of free pages*/ +} mali_block_allocator; + +unsigned long _mali_blk_item_get_phy_addr(mali_block_item *item); +unsigned long _mali_blk_item_get_pfn(mali_block_item *item); +u32 mali_mem_block_get_ref_count(mali_page_node *node); +u32 mali_mem_block_add_ref(mali_page_node *node); +u32 mali_mem_block_dec_ref(mali_page_node *node); +u32 mali_mem_block_release(mali_mem_backend *mem_bkend); +int mali_mem_block_alloc(mali_mem_block_mem *block_mem, u32 size); +int mali_mem_block_mali_map(mali_mem_block_mem *block_mem, struct mali_session_data *session, u32 vaddr, u32 props); +void mali_mem_block_mali_unmap(mali_mem_allocation *alloc); + +int mali_mem_block_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma); +_mali_osk_errcode_t mali_memory_core_resource_dedicated_memory(u32 start, u32 size); +mali_bool mali_memory_have_dedicated_memory(void); +u32 mali_mem_block_free(mali_mem_block_mem *block_mem); +u32 mali_mem_block_free_list(struct list_head *list); +void mali_mem_block_free_node(struct mali_page_node *node); +void mali_mem_block_allocator_destroy(void); +_mali_osk_errcode_t mali_mem_block_unref_node(struct mali_page_node *node); +u32 mali_mem_block_allocator_stat(void); + +#endif /* __MALI_BLOCK_ALLOCATOR_H__ */ diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_cow.c b/drivers/gpu/arm/utgard/linux/mali_memory_cow.c new file mode 100644 index 00000000000000..bcd0f871377191 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_memory_cow.c @@ -0,0 +1,754 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef CONFIG_ARM +#include +#endif +#include + +#include "mali_memory.h" +#include "mali_kernel_common.h" +#include "mali_uk_types.h" +#include "mali_osk.h" +#include "mali_kernel_linux.h" +#include "mali_memory_cow.h" +#include "mali_memory_block_alloc.h" +#include "mali_memory_swap_alloc.h" + +/** +* allocate pages for COW backend and flush cache +*/ +static struct page *mali_mem_cow_alloc_page(void) + +{ + mali_mem_os_mem os_mem; + struct mali_page_node *node; + struct page *new_page; + + int ret = 0; + /* allocate pages from os mem */ + ret = mali_mem_os_alloc_pages(&os_mem, _MALI_OSK_MALI_PAGE_SIZE); + + if (ret) { + return NULL; + } + + MALI_DEBUG_ASSERT(1 == os_mem.count); + + node = _MALI_OSK_CONTAINER_OF(os_mem.pages.next, struct mali_page_node, list); + new_page = node->page; + node->page = NULL; + list_del(&node->list); + kfree(node); + + return new_page; +} + + +static struct list_head *_mali_memory_cow_get_node_list(mali_mem_backend *target_bk, + u32 target_offset, + u32 target_size) +{ + MALI_DEBUG_ASSERT(MALI_MEM_OS == target_bk->type || MALI_MEM_COW == target_bk->type || + MALI_MEM_BLOCK == target_bk->type || MALI_MEM_SWAP == target_bk->type); + + if (MALI_MEM_OS == target_bk->type) { + MALI_DEBUG_ASSERT(&target_bk->os_mem); + MALI_DEBUG_ASSERT(((target_size + target_offset) / _MALI_OSK_MALI_PAGE_SIZE) <= target_bk->os_mem.count); + return &target_bk->os_mem.pages; + } else if (MALI_MEM_COW == target_bk->type) { + MALI_DEBUG_ASSERT(&target_bk->cow_mem); + MALI_DEBUG_ASSERT(((target_size + target_offset) / _MALI_OSK_MALI_PAGE_SIZE) <= target_bk->cow_mem.count); + return &target_bk->cow_mem.pages; + } else if (MALI_MEM_BLOCK == target_bk->type) { + MALI_DEBUG_ASSERT(&target_bk->block_mem); + MALI_DEBUG_ASSERT(((target_size + target_offset) / _MALI_OSK_MALI_PAGE_SIZE) <= target_bk->block_mem.count); + return &target_bk->block_mem.pfns; + } else if (MALI_MEM_SWAP == target_bk->type) { + MALI_DEBUG_ASSERT(&target_bk->swap_mem); + MALI_DEBUG_ASSERT(((target_size + target_offset) / _MALI_OSK_MALI_PAGE_SIZE) <= target_bk->swap_mem.count); + return &target_bk->swap_mem.pages; + } + + return NULL; +} + +/** +* Do COW for os memory - support do COW for memory from bank memory +* The range_start/size can be zero, which means it will call cow_modify_range +* latter. +* This function allocate new pages for COW backend from os mem for a modified range +* It will keep the page which not in the modified range and Add ref to it +* +* @target_bk - target allocation's backend(the allocation need to do COW) +* @target_offset - the offset in target allocation to do COW(for support COW a memory allocated from memory_bank, 4K align) +* @target_size - size of target allocation to do COW (for support memory bank) +* @backend -COW backend +* @range_start - offset of modified range (4K align) +* @range_size - size of modified range +*/ +_mali_osk_errcode_t mali_memory_cow_os_memory(mali_mem_backend *target_bk, + u32 target_offset, + u32 target_size, + mali_mem_backend *backend, + u32 range_start, + u32 range_size) +{ + mali_mem_cow *cow = &backend->cow_mem; + struct mali_page_node *m_page, *m_tmp, *page_node; + int target_page = 0; + struct page *new_page; + struct list_head *pages = NULL; + + pages = _mali_memory_cow_get_node_list(target_bk, target_offset, target_size); + + if (NULL == pages) { + MALI_DEBUG_ASSERT(0); + return _MALI_OSK_ERR_FAULT; + } + + MALI_DEBUG_ASSERT(0 == cow->count); + + INIT_LIST_HEAD(&cow->pages); + mutex_lock(&target_bk->mutex); + list_for_each_entry_safe(m_page, m_tmp, pages, list) { + /* add page from (target_offset,target_offset+size) to cow backend */ + if ((target_page >= target_offset / _MALI_OSK_MALI_PAGE_SIZE) && + (target_page < ((target_size + target_offset) / _MALI_OSK_MALI_PAGE_SIZE))) { + + /* allocate a new page node, alway use OS memory for COW */ + page_node = _mali_page_node_allocate(MALI_PAGE_NODE_OS); + + if (NULL == page_node) { + mutex_unlock(&target_bk->mutex); + goto error; + } + + INIT_LIST_HEAD(&page_node->list); + + /* check if in the modified range*/ + if ((cow->count >= range_start / _MALI_OSK_MALI_PAGE_SIZE) && + (cow->count < (range_start + range_size) / _MALI_OSK_MALI_PAGE_SIZE)) { + /* need to allocate a new page */ + /* To simplify the case, All COW memory is allocated from os memory ?*/ + new_page = mali_mem_cow_alloc_page(); + + if (NULL == new_page) { + kfree(page_node); + mutex_unlock(&target_bk->mutex); + goto error; + } + + _mali_page_node_add_page(page_node, new_page); + } else { + /*Add Block memory case*/ + if (m_page->type != MALI_PAGE_NODE_BLOCK) { + _mali_page_node_add_page(page_node, m_page->page); + } else { + page_node->type = MALI_PAGE_NODE_BLOCK; + _mali_page_node_add_block_item(page_node, m_page->blk_it); + } + + /* add ref to this page */ + _mali_page_node_ref(m_page); + } + + /* add it to COW backend page list */ + list_add_tail(&page_node->list, &cow->pages); + cow->count++; + } + target_page++; + } + mutex_unlock(&target_bk->mutex); + return _MALI_OSK_ERR_OK; +error: + mali_mem_cow_release(backend, MALI_FALSE); + return _MALI_OSK_ERR_FAULT; +} + +_mali_osk_errcode_t mali_memory_cow_swap_memory(mali_mem_backend *target_bk, + u32 target_offset, + u32 target_size, + mali_mem_backend *backend, + u32 range_start, + u32 range_size) +{ + mali_mem_cow *cow = &backend->cow_mem; + struct mali_page_node *m_page, *m_tmp, *page_node; + int target_page = 0; + struct mali_swap_item *swap_item; + struct list_head *pages = NULL; + + pages = _mali_memory_cow_get_node_list(target_bk, target_offset, target_size); + if (NULL == pages) { + MALI_DEBUG_ASSERT(0); + return _MALI_OSK_ERR_FAULT; + } + + MALI_DEBUG_ASSERT(0 == cow->count); + + INIT_LIST_HEAD(&cow->pages); + mutex_lock(&target_bk->mutex); + + backend->flags |= MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN; + + list_for_each_entry_safe(m_page, m_tmp, pages, list) { + /* add page from (target_offset,target_offset+size) to cow backend */ + if ((target_page >= target_offset / _MALI_OSK_MALI_PAGE_SIZE) && + (target_page < ((target_size + target_offset) / _MALI_OSK_MALI_PAGE_SIZE))) { + + /* allocate a new page node, use swap memory for COW memory swap cowed flag. */ + page_node = _mali_page_node_allocate(MALI_PAGE_NODE_SWAP); + + if (NULL == page_node) { + mutex_unlock(&target_bk->mutex); + goto error; + } + + /* check if in the modified range*/ + if ((cow->count >= range_start / _MALI_OSK_MALI_PAGE_SIZE) && + (cow->count < (range_start + range_size) / _MALI_OSK_MALI_PAGE_SIZE)) { + /* need to allocate a new page */ + /* To simplify the case, All COW memory is allocated from os memory ?*/ + swap_item = mali_mem_swap_alloc_swap_item(); + + if (NULL == swap_item) { + kfree(page_node); + mutex_unlock(&target_bk->mutex); + goto error; + } + + swap_item->idx = mali_mem_swap_idx_alloc(); + + if (_MALI_OSK_BITMAP_INVALIDATE_INDEX == swap_item->idx) { + MALI_DEBUG_PRINT(1, ("Failed to allocate swap index in swap CoW.\n")); + kfree(page_node); + kfree(swap_item); + mutex_unlock(&target_bk->mutex); + goto error; + } + + _mali_page_node_add_swap_item(page_node, swap_item); + } else { + _mali_page_node_add_swap_item(page_node, m_page->swap_it); + + /* add ref to this page */ + _mali_page_node_ref(m_page); + } + + list_add_tail(&page_node->list, &cow->pages); + cow->count++; + } + target_page++; + } + mutex_unlock(&target_bk->mutex); + + return _MALI_OSK_ERR_OK; +error: + mali_mem_swap_release(backend, MALI_FALSE); + return _MALI_OSK_ERR_FAULT; + +} + + +_mali_osk_errcode_t _mali_mem_put_page_node(mali_page_node *node) +{ + if (node->type == MALI_PAGE_NODE_OS) { + return mali_mem_os_put_page(node->page); + } else if (node->type == MALI_PAGE_NODE_BLOCK) { + return mali_mem_block_unref_node(node); + } else if (node->type == MALI_PAGE_NODE_SWAP) { + return _mali_mem_swap_put_page_node(node); + } else + MALI_DEBUG_ASSERT(0); + return _MALI_OSK_ERR_FAULT; +} + + +/** +* Modify a range of a exist COW backend +* @backend -COW backend +* @range_start - offset of modified range (4K align) +* @range_size - size of modified range(in byte) +*/ +_mali_osk_errcode_t mali_memory_cow_modify_range(mali_mem_backend *backend, + u32 range_start, + u32 range_size) +{ + mali_mem_allocation *alloc = NULL; + mali_mem_cow *cow = &backend->cow_mem; + struct mali_page_node *m_page, *m_tmp; + LIST_HEAD(pages); + struct page *new_page; + u32 count = 0; + s32 change_pages_nr = 0; + _mali_osk_errcode_t ret = _MALI_OSK_ERR_OK; + + if (range_start % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); + if (range_size % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); + + alloc = backend->mali_allocation; + MALI_DEBUG_ASSERT_POINTER(alloc); + + MALI_DEBUG_ASSERT(MALI_MEM_COW == backend->type); + MALI_DEBUG_ASSERT(((range_start + range_size) / _MALI_OSK_MALI_PAGE_SIZE) <= cow->count); + + mutex_lock(&backend->mutex); + + /* free pages*/ + list_for_each_entry_safe(m_page, m_tmp, &cow->pages, list) { + + /* check if in the modified range*/ + if ((count >= range_start / _MALI_OSK_MALI_PAGE_SIZE) && + (count < (range_start + range_size) / _MALI_OSK_MALI_PAGE_SIZE)) { + if (MALI_PAGE_NODE_SWAP != m_page->type) { + new_page = mali_mem_cow_alloc_page(); + + if (NULL == new_page) { + goto error; + } + if (1 != _mali_page_node_get_ref_count(m_page)) + change_pages_nr++; + /* unref old page*/ + if (_mali_mem_put_page_node(m_page)) { + __free_page(new_page); + goto error; + } + /* add new page*/ + /* always use OS for COW*/ + m_page->type = MALI_PAGE_NODE_OS; + _mali_page_node_add_page(m_page, new_page); + } else { + struct mali_swap_item *swap_item; + + swap_item = mali_mem_swap_alloc_swap_item(); + + if (NULL == swap_item) { + goto error; + } + + swap_item->idx = mali_mem_swap_idx_alloc(); + + if (_MALI_OSK_BITMAP_INVALIDATE_INDEX == swap_item->idx) { + MALI_DEBUG_PRINT(1, ("Failed to allocate swap index in swap CoW modify range.\n")); + kfree(swap_item); + goto error; + } + + if (1 != _mali_page_node_get_ref_count(m_page)) { + change_pages_nr++; + } + + if (_mali_mem_put_page_node(m_page)) { + mali_mem_swap_free_swap_item(swap_item); + goto error; + } + + _mali_page_node_add_swap_item(m_page, swap_item); + } + } + count++; + } + cow->change_pages_nr = change_pages_nr; + + MALI_DEBUG_ASSERT(MALI_MEM_COW == alloc->type); + + /* ZAP cpu mapping(modified range), and do cpu mapping here if need */ + if (NULL != alloc->cpu_mapping.vma) { + MALI_DEBUG_ASSERT(0 != alloc->backend_handle); + MALI_DEBUG_ASSERT(NULL != alloc->cpu_mapping.vma); + MALI_DEBUG_ASSERT(alloc->cpu_mapping.vma->vm_end - alloc->cpu_mapping.vma->vm_start >= range_size); + + if (MALI_MEM_BACKEND_FLAG_SWAP_COWED != (backend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED)) { + zap_vma_ptes(alloc->cpu_mapping.vma, alloc->cpu_mapping.vma->vm_start + range_start, range_size); + + ret = mali_mem_cow_cpu_map_pages_locked(backend, alloc->cpu_mapping.vma, alloc->cpu_mapping.vma->vm_start + range_start, range_size / _MALI_OSK_MALI_PAGE_SIZE); + + if (unlikely(ret != _MALI_OSK_ERR_OK)) { + MALI_DEBUG_PRINT(2, ("mali_memory_cow_modify_range: cpu mapping failed !\n")); + ret = _MALI_OSK_ERR_FAULT; + } + } else { + /* used to trigger page fault for swappable cowed memory. */ + alloc->cpu_mapping.vma->vm_flags |= VM_PFNMAP; + alloc->cpu_mapping.vma->vm_flags |= VM_MIXEDMAP; + + zap_vma_ptes(alloc->cpu_mapping.vma, alloc->cpu_mapping.vma->vm_start + range_start, range_size); + /* delete this flag to let swappble is ummapped regard to stauct page not page frame. */ + alloc->cpu_mapping.vma->vm_flags &= ~VM_PFNMAP; + alloc->cpu_mapping.vma->vm_flags &= ~VM_MIXEDMAP; + } + } + +error: + mutex_unlock(&backend->mutex); + return ret; + +} + + +/** +* Allocate pages for COW backend +* @alloc -allocation for COW allocation +* @target_bk - target allocation's backend(the allocation need to do COW) +* @target_offset - the offset in target allocation to do COW(for support COW a memory allocated from memory_bank, 4K align) +* @target_size - size of target allocation to do COW (for support memory bank)(in byte) +* @backend -COW backend +* @range_start - offset of modified range (4K align) +* @range_size - size of modified range(in byte) +*/ +_mali_osk_errcode_t mali_memory_do_cow(mali_mem_backend *target_bk, + u32 target_offset, + u32 target_size, + mali_mem_backend *backend, + u32 range_start, + u32 range_size) +{ + struct mali_session_data *session = backend->mali_allocation->session; + + MALI_CHECK_NON_NULL(session, _MALI_OSK_ERR_INVALID_ARGS); + + /* size & offset must be a multiple of the system page size */ + if (target_size % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); + if (range_size % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); + if (target_offset % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); + if (range_start % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); + + /* check backend type */ + MALI_DEBUG_ASSERT(MALI_MEM_COW == backend->type); + + switch (target_bk->type) { + case MALI_MEM_OS: + case MALI_MEM_BLOCK: + return mali_memory_cow_os_memory(target_bk, target_offset, target_size, backend, range_start, range_size); + break; + case MALI_MEM_COW: + if (backend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED) { + return mali_memory_cow_swap_memory(target_bk, target_offset, target_size, backend, range_start, range_size); + } else { + return mali_memory_cow_os_memory(target_bk, target_offset, target_size, backend, range_start, range_size); + } + break; + case MALI_MEM_SWAP: + return mali_memory_cow_swap_memory(target_bk, target_offset, target_size, backend, range_start, range_size); + break; + case MALI_MEM_EXTERNAL: + /*NOT support yet*/ + MALI_DEBUG_ASSERT(0); + break; + case MALI_MEM_DMA_BUF: + /*NOT support yet*/ + MALI_DEBUG_ASSERT(0); + break; + case MALI_MEM_UMP: + /*NOT support yet*/ + MALI_DEBUG_ASSERT(0); + break; + default: + /*Not support yet*/ + MALI_DEBUG_ASSERT(0); + break; + } + return _MALI_OSK_ERR_OK; +} + + +/** +* Map COW backend memory to mali +* Support OS/BLOCK for mali_page_node +*/ +int mali_mem_cow_mali_map(mali_mem_backend *mem_bkend, u32 range_start, u32 range_size) +{ + mali_mem_allocation *cow_alloc; + struct mali_page_node *m_page; + struct mali_session_data *session; + struct mali_page_directory *pagedir; + u32 virt, start; + + cow_alloc = mem_bkend->mali_allocation; + virt = cow_alloc->mali_vma_node.vm_node.start; + start = virt; + + MALI_DEBUG_ASSERT_POINTER(mem_bkend); + MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_bkend->type); + MALI_DEBUG_ASSERT_POINTER(cow_alloc); + + session = cow_alloc->session; + pagedir = session->page_directory; + MALI_CHECK_NON_NULL(session, _MALI_OSK_ERR_INVALID_ARGS); + list_for_each_entry(m_page, &mem_bkend->cow_mem.pages, list) { + if ((virt - start >= range_start) && (virt - start < range_start + range_size)) { + dma_addr_t phys = _mali_page_node_get_dma_addr(m_page); +#if defined(CONFIG_ARCH_DMA_ADDR_T_64BIT) + MALI_DEBUG_ASSERT(0 == (phys >> 32)); +#endif + mali_mmu_pagedir_update(pagedir, virt, (mali_dma_addr)phys, + MALI_MMU_PAGE_SIZE, MALI_MMU_FLAGS_DEFAULT); + } + virt += MALI_MMU_PAGE_SIZE; + } + return 0; +} + +/** +* Map COW backend to cpu +* support OS/BLOCK memory +*/ +int mali_mem_cow_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma) +{ + mali_mem_cow *cow = &mem_bkend->cow_mem; + struct mali_page_node *m_page; + int ret; + unsigned long addr = vma->vm_start; + MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_COW); + + list_for_each_entry(m_page, &cow->pages, list) { + /* We should use vm_insert_page, but it does a dcache + * flush which makes it way slower than remap_pfn_range or vm_insert_pfn. + ret = vm_insert_page(vma, addr, page); + */ + ret = vm_insert_pfn(vma, addr, _mali_page_node_get_pfn(m_page)); + + if (unlikely(0 != ret)) { + return ret; + } + addr += _MALI_OSK_MALI_PAGE_SIZE; + } + + return 0; +} + +/** +* Map some pages(COW backend) to CPU vma@vaddr +*@ mem_bkend - COW backend +*@ vma +*@ vaddr -start CPU vaddr mapped to +*@ num - max number of pages to map to CPU vaddr +*/ +_mali_osk_errcode_t mali_mem_cow_cpu_map_pages_locked(mali_mem_backend *mem_bkend, + struct vm_area_struct *vma, + unsigned long vaddr, + int num) +{ + mali_mem_cow *cow = &mem_bkend->cow_mem; + struct mali_page_node *m_page; + int ret; + int offset; + int count ; + unsigned long vstart = vma->vm_start; + count = 0; + MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_COW); + MALI_DEBUG_ASSERT(0 == vaddr % _MALI_OSK_MALI_PAGE_SIZE); + MALI_DEBUG_ASSERT(0 == vstart % _MALI_OSK_MALI_PAGE_SIZE); + offset = (vaddr - vstart) / _MALI_OSK_MALI_PAGE_SIZE; + + list_for_each_entry(m_page, &cow->pages, list) { + if ((count >= offset) && (count < offset + num)) { + ret = vm_insert_pfn(vma, vaddr, _mali_page_node_get_pfn(m_page)); + + if (unlikely(0 != ret)) { + if (count == offset) { + return _MALI_OSK_ERR_FAULT; + } else { + /* ret is EBUSY when page isn't in modify range, but now it's OK*/ + return _MALI_OSK_ERR_OK; + } + } + vaddr += _MALI_OSK_MALI_PAGE_SIZE; + } + count++; + } + return _MALI_OSK_ERR_OK; +} + +/** +* Release COW backend memory +* free it directly(put_page--unref page), not put into pool +*/ +u32 mali_mem_cow_release(mali_mem_backend *mem_bkend, mali_bool is_mali_mapped) +{ + mali_mem_allocation *alloc; + u32 free_pages_nr = 0; + MALI_DEBUG_ASSERT_POINTER(mem_bkend); + MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_bkend->type); + alloc = mem_bkend->mali_allocation; + MALI_DEBUG_ASSERT_POINTER(alloc); + + if (MALI_MEM_BACKEND_FLAG_SWAP_COWED != (MALI_MEM_BACKEND_FLAG_SWAP_COWED & mem_bkend->flags)) { + /* Unmap the memory from the mali virtual address space. */ + if (MALI_TRUE == is_mali_mapped) + mali_mem_os_mali_unmap(alloc); + /* free cow backend list*/ + free_pages_nr = mali_mem_os_free(&mem_bkend->cow_mem.pages, mem_bkend->cow_mem.count, MALI_TRUE); + free_pages_nr += mali_mem_block_free_list(&mem_bkend->cow_mem.pages); + + MALI_DEBUG_ASSERT(list_empty(&mem_bkend->cow_mem.pages)); + } else { + free_pages_nr = mali_mem_swap_release(mem_bkend, is_mali_mapped); + } + + + MALI_DEBUG_PRINT(4, ("COW Mem free : allocated size = 0x%x, free size = 0x%x\n", mem_bkend->cow_mem.count * _MALI_OSK_MALI_PAGE_SIZE, + free_pages_nr * _MALI_OSK_MALI_PAGE_SIZE)); + + mem_bkend->cow_mem.count = 0; + return free_pages_nr; +} + + +/* Dst node could os node or swap node. */ +void _mali_mem_cow_copy_page(mali_page_node *src_node, mali_page_node *dst_node) +{ + void *dst, *src; + struct page *dst_page; + dma_addr_t dma_addr; + + MALI_DEBUG_ASSERT(src_node != NULL); + MALI_DEBUG_ASSERT(dst_node != NULL); + MALI_DEBUG_ASSERT(dst_node->type == MALI_PAGE_NODE_OS + || dst_node->type == MALI_PAGE_NODE_SWAP); + + if (dst_node->type == MALI_PAGE_NODE_OS) { + dst_page = dst_node->page; + } else { + dst_page = dst_node->swap_it->page; + } + + dma_unmap_page(&mali_platform_device->dev, _mali_page_node_get_dma_addr(dst_node), + _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL); + + /* map it , and copy the content*/ + dst = kmap_atomic(dst_page); + + if (src_node->type == MALI_PAGE_NODE_OS || + src_node->type == MALI_PAGE_NODE_SWAP) { + struct page *src_page; + + if (src_node->type == MALI_PAGE_NODE_OS) { + src_page = src_node->page; + } else { + src_page = src_node->swap_it->page; + } + + /* Clear and invaliate cache */ + /* In ARM architecture, speculative read may pull stale data into L1 cache + * for kernel linear mapping page table. DMA_BIDIRECTIONAL could + * invalidate the L1 cache so that following read get the latest data + */ + dma_unmap_page(&mali_platform_device->dev, _mali_page_node_get_dma_addr(src_node), + _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL); + + src = kmap_atomic(src_page); + memcpy(dst, src , _MALI_OSK_MALI_PAGE_SIZE); + kunmap_atomic(src); + dma_addr = dma_map_page(&mali_platform_device->dev, src_page, + 0, _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL); + + if (src_node->type == MALI_PAGE_NODE_SWAP) { + src_node->swap_it->dma_addr = dma_addr; + } + } else if (src_node->type == MALI_PAGE_NODE_BLOCK) { + /* + * use ioremap to map src for BLOCK memory + */ + src = ioremap_nocache(_mali_page_node_get_dma_addr(src_node), _MALI_OSK_MALI_PAGE_SIZE); + memcpy(dst, src , _MALI_OSK_MALI_PAGE_SIZE); + iounmap(src); + } + kunmap_atomic(dst); + dma_addr = dma_map_page(&mali_platform_device->dev, dst_page, + 0, _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE); + + if (dst_node->type == MALI_PAGE_NODE_SWAP) { + dst_node->swap_it->dma_addr = dma_addr; + } +} + + +/* +* allocate page on demand when CPU access it, +* THis used in page fault handler +*/ +_mali_osk_errcode_t mali_mem_cow_allocate_on_demand(mali_mem_backend *mem_bkend, u32 offset_page) +{ + struct page *new_page = NULL; + struct mali_page_node *new_node = NULL; + int i = 0; + struct mali_page_node *m_page, *found_node = NULL; + struct mali_session_data *session = NULL; + mali_mem_cow *cow = &mem_bkend->cow_mem; + MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_bkend->type); + MALI_DEBUG_ASSERT(offset_page < mem_bkend->size / _MALI_OSK_MALI_PAGE_SIZE); + MALI_DEBUG_PRINT(4, ("mali_mem_cow_allocate_on_demand !, offset_page =0x%x\n", offset_page)); + + /* allocate new page here */ + new_page = mali_mem_cow_alloc_page(); + if (!new_page) + return _MALI_OSK_ERR_NOMEM; + + new_node = _mali_page_node_allocate(MALI_PAGE_NODE_OS); + if (!new_node) { + __free_page(new_page); + return _MALI_OSK_ERR_NOMEM; + } + + /* find the page in backend*/ + list_for_each_entry(m_page, &cow->pages, list) { + if (i == offset_page) { + found_node = m_page; + break; + } + i++; + } + MALI_DEBUG_ASSERT(found_node); + if (NULL == found_node) { + __free_page(new_page); + kfree(new_node); + return _MALI_OSK_ERR_ITEM_NOT_FOUND; + } + + _mali_page_node_add_page(new_node, new_page); + + /* Copy the src page's content to new page */ + _mali_mem_cow_copy_page(found_node, new_node); + + MALI_DEBUG_ASSERT_POINTER(mem_bkend->mali_allocation); + session = mem_bkend->mali_allocation->session; + MALI_DEBUG_ASSERT_POINTER(session); + if (1 != _mali_page_node_get_ref_count(found_node)) { + atomic_add(1, &session->mali_mem_allocated_pages); + if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) { + session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE; + } + mem_bkend->cow_mem.change_pages_nr++; + } + if (_mali_mem_put_page_node(found_node)) { + __free_page(new_page); + kfree(new_node); + return _MALI_OSK_ERR_NOMEM; + } + + list_replace(&found_node->list, &new_node->list); + + kfree(found_node); + + /* map to GPU side*/ + _mali_osk_mutex_wait(session->memory_lock); + mali_mem_cow_mali_map(mem_bkend, offset_page * _MALI_OSK_MALI_PAGE_SIZE, _MALI_OSK_MALI_PAGE_SIZE); + _mali_osk_mutex_signal(session->memory_lock); + return _MALI_OSK_ERR_OK; +} diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_cow.h b/drivers/gpu/arm/utgard/linux/mali_memory_cow.h new file mode 100644 index 00000000000000..c16ec7633c0af2 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_memory_cow.h @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_MEMORY_COW_H__ +#define __MALI_MEMORY_COW_H__ + +#include "mali_osk.h" +#include "mali_session.h" +#include "mali_memory_types.h" + +int mali_mem_cow_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma); +_mali_osk_errcode_t mali_mem_cow_cpu_map_pages_locked(mali_mem_backend *mem_bkend, + struct vm_area_struct *vma, + unsigned long vaddr, + int num); + +_mali_osk_errcode_t mali_memory_do_cow(mali_mem_backend *target_bk, + u32 target_offset, + u32 target_size, + mali_mem_backend *backend, + u32 range_start, + u32 range_size); + +_mali_osk_errcode_t mali_memory_cow_modify_range(mali_mem_backend *backend, + u32 range_start, + u32 range_size); + +_mali_osk_errcode_t mali_memory_cow_os_memory(mali_mem_backend *target_bk, + u32 target_offset, + u32 target_size, + mali_mem_backend *backend, + u32 range_start, + u32 range_size); + +void _mali_mem_cow_copy_page(mali_page_node *src_node, mali_page_node *dst_node); + +int mali_mem_cow_mali_map(mali_mem_backend *mem_bkend, u32 range_start, u32 range_size); +u32 mali_mem_cow_release(mali_mem_backend *mem_bkend, mali_bool is_mali_mapped); +_mali_osk_errcode_t mali_mem_cow_allocate_on_demand(mali_mem_backend *mem_bkend, u32 offset_page); +#endif + diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_defer_bind.c b/drivers/gpu/arm/utgard/linux/mali_memory_defer_bind.c new file mode 100644 index 00000000000000..e51902f1bf65ed --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_memory_defer_bind.c @@ -0,0 +1,246 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef CONFIG_ARM +#include +#endif +#include + +#include "mali_memory.h" +#include "mali_kernel_common.h" +#include "mali_uk_types.h" +#include "mali_osk.h" +#include "mali_kernel_linux.h" +#include "mali_memory_defer_bind.h" +#include "mali_executor.h" +#include "mali_osk.h" +#include "mali_scheduler.h" +#include "mali_gp_job.h" + +mali_defer_bind_manager *mali_dmem_man = NULL; + +static u32 mali_dmem_get_gp_varying_size(struct mali_gp_job *gp_job) +{ + return gp_job->uargs.varying_memsize / _MALI_OSK_MALI_PAGE_SIZE; +} + +_mali_osk_errcode_t mali_mem_defer_bind_manager_init(void) +{ + mali_dmem_man = _mali_osk_calloc(1, sizeof(struct mali_defer_bind_manager)); + if (!mali_dmem_man) + return _MALI_OSK_ERR_NOMEM; + + atomic_set(&mali_dmem_man->num_used_pages, 0); + atomic_set(&mali_dmem_man->num_dmem, 0); + + return _MALI_OSK_ERR_OK; +} + + +void mali_mem_defer_bind_manager_destory(void) +{ + if (mali_dmem_man) { + MALI_DEBUG_ASSERT(0 == atomic_read(&mali_dmem_man->num_dmem)); + kfree(mali_dmem_man); + } + mali_dmem_man = NULL; +} + + +/*allocate pages from OS memory*/ +_mali_osk_errcode_t mali_mem_defer_alloc_mem(u32 require, struct mali_session_data *session, mali_defer_mem_block *dblock) +{ + int retval = 0; + u32 num_pages = require; + mali_mem_os_mem os_mem; + + retval = mali_mem_os_alloc_pages(&os_mem, num_pages * _MALI_OSK_MALI_PAGE_SIZE); + + /* add to free pages list */ + if (0 == retval) { + MALI_DEBUG_PRINT(4, ("mali_mem_defer_alloc_mem ,,*** pages allocate = 0x%x \n", num_pages)); + list_splice(&os_mem.pages, &dblock->free_pages); + atomic_add(os_mem.count, &dblock->num_free_pages); + atomic_add(os_mem.count, &session->mali_mem_allocated_pages); + if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) { + session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE; + } + return _MALI_OSK_ERR_OK; + } else + return _MALI_OSK_ERR_FAULT; +} + +_mali_osk_errcode_t mali_mem_prepare_mem_for_job(struct mali_gp_job *next_gp_job, mali_defer_mem_block *dblock) +{ + u32 require_page; + + if (!next_gp_job) + return _MALI_OSK_ERR_FAULT; + + require_page = mali_dmem_get_gp_varying_size(next_gp_job); + + MALI_DEBUG_PRINT(4, ("mali_mem_defer_prepare_mem_work, require alloc page 0x%x\n", + require_page)); + /* allocate more pages from OS */ + if (_MALI_OSK_ERR_OK != mali_mem_defer_alloc_mem(require_page, next_gp_job->session, dblock)) { + MALI_DEBUG_PRINT(1, ("ERROR##mali_mem_defer_prepare_mem_work, allocate page failed!!")); + return _MALI_OSK_ERR_NOMEM; + } + + next_gp_job->bind_flag = MALI_DEFER_BIND_MEMORY_PREPARED; + + return _MALI_OSK_ERR_OK; +} + + +/* do preparetion for allocation before defer bind */ +_mali_osk_errcode_t mali_mem_defer_bind_allocation_prepare(mali_mem_allocation *alloc, struct list_head *list) +{ + mali_mem_backend *mem_bkend = NULL; + struct mali_backend_bind_list *bk_list = _mali_osk_calloc(1, sizeof(struct mali_backend_bind_list)); + if (NULL == bk_list) + return _MALI_OSK_ERR_FAULT; + + INIT_LIST_HEAD(&bk_list->node); + /* Get backend memory */ + mutex_lock(&mali_idr_mutex); + if (!(mem_bkend = idr_find(&mali_backend_idr, alloc->backend_handle))) { + MALI_DEBUG_PRINT(1, ("Can't find memory backend in defer bind!\n")); + mutex_unlock(&mali_idr_mutex); + kfree(bk_list); + return _MALI_OSK_ERR_FAULT; + } + mutex_unlock(&mali_idr_mutex); + MALI_DEBUG_PRINT(4, ("bind_allocation_prepare:: allocation =%x vaddr=0x%x!\n", alloc, alloc->mali_vma_node.vm_node.start)); + + INIT_LIST_HEAD(&mem_bkend->os_mem.pages); + + bk_list->bkend = mem_bkend; + bk_list->vaddr = alloc->mali_vma_node.vm_node.start; + bk_list->session = alloc->session; + bk_list->page_num = mem_bkend->size / _MALI_OSK_MALI_PAGE_SIZE; + MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_OS); + + /* add to job to do list */ + list_add(&bk_list->node, list); + + return _MALI_OSK_ERR_OK; +} + + + +/* bind phyiscal memory to allocation +This function will be called in IRQ handler*/ +static _mali_osk_errcode_t mali_mem_defer_bind_allocation(struct mali_backend_bind_list *bk_node, + struct list_head *pages) +{ + struct mali_session_data *session = bk_node->session; + mali_mem_backend *mem_bkend = bk_node->bkend; + MALI_DEBUG_PRINT(4, ("mali_mem_defer_bind_allocation, bind bkend = %x page num=0x%x vaddr=%x session=%x\n", mem_bkend, bk_node->page_num, bk_node->vaddr, session)); + + MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_OS); + list_splice(pages, &mem_bkend->os_mem.pages); + mem_bkend->os_mem.count = bk_node->page_num; + + if (mem_bkend->type == MALI_MEM_OS) { + mali_mem_os_mali_map(&mem_bkend->os_mem, session, bk_node->vaddr, 0, + mem_bkend->os_mem.count, MALI_MMU_FLAGS_DEFAULT); + } + smp_wmb(); + bk_node->flag = MALI_DEFER_BIND_MEMORY_BINDED; + mem_bkend->flags &= ~MALI_MEM_BACKEND_FLAG_NOT_BINDED; + mem_bkend->flags |= MALI_MEM_BACKEND_FLAG_BINDED; + return _MALI_OSK_ERR_OK; +} + + +static struct list_head *mali_mem_defer_get_free_page_list(u32 count, struct list_head *pages, mali_defer_mem_block *dblock) +{ + int i = 0; + struct mali_page_node *m_page, *m_tmp; + + if (atomic_read(&dblock->num_free_pages) < count) { + return NULL; + } else { + list_for_each_entry_safe(m_page, m_tmp, &dblock->free_pages, list) { + if (i < count) { + list_move_tail(&m_page->list, pages); + } else { + break; + } + i++; + } + MALI_DEBUG_ASSERT(i == count); + atomic_sub(count, &dblock->num_free_pages); + return pages; + } +} + + +/* called in job start IOCTL to bind physical memory for each allocations +@ bk_list backend list to do defer bind +@ pages page list to do this bind +@ count number of pages +*/ +_mali_osk_errcode_t mali_mem_defer_bind(u32 count, struct mali_gp_job *gp, + struct mali_defer_mem_block *dmem_block) +{ + struct mali_defer_mem *dmem = NULL; + struct mali_backend_bind_list *bkn, *bkn_tmp; + LIST_HEAD(pages); + + MALI_DEBUG_PRINT(4, ("#BIND: GP job=%x## \n", gp)); + dmem = (mali_defer_mem *)_mali_osk_calloc(1, sizeof(struct mali_defer_mem)); + if (dmem) { + INIT_LIST_HEAD(&dmem->node); + gp->dmem = dmem; + } else { + return _MALI_OSK_ERR_NOMEM; + } + + atomic_add(1, &mali_dmem_man->num_dmem); + /* for each bk_list backend, do bind */ + list_for_each_entry_safe(bkn, bkn_tmp , &gp->vary_todo, node) { + INIT_LIST_HEAD(&pages); + if (likely(mali_mem_defer_get_free_page_list(bkn->page_num, &pages, dmem_block))) { + list_del(&bkn->node); + mali_mem_defer_bind_allocation(bkn, &pages); + _mali_osk_free(bkn); + } else { + /* not enough memory will not happen */ + MALI_DEBUG_PRINT(1, ("#BIND: NOT enough memory when binded !!## \n")); + MALI_DEBUG_ASSERT(0); + } + } + + if (!list_empty(&gp->vary_todo)) { + MALI_DEBUG_ASSERT(0); + } + + dmem->flag = MALI_DEFER_BIND_MEMORY_BINDED; + + return _MALI_OSK_ERR_OK; +} + +void mali_mem_defer_dmem_free(struct mali_gp_job *gp) +{ + if (gp->dmem) { + atomic_dec(&mali_dmem_man->num_dmem); + _mali_osk_free(gp->dmem); + } +} + diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_defer_bind.h b/drivers/gpu/arm/utgard/linux/mali_memory_defer_bind.h new file mode 100644 index 00000000000000..ef67540434f51a --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_memory_defer_bind.h @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#ifndef __MALI_MEMORY_DEFER_BIND_H_ +#define __MALI_MEMORY_DEFER_BIND_H_ + + +#include "mali_osk.h" +#include "mali_session.h" + +#include +#include +#include +#include +#include + + +#include "mali_memory_types.h" +#include "mali_memory_os_alloc.h" +#include "mali_uk_types.h" + +struct mali_gp_job; + +typedef struct mali_defer_mem { + struct list_head node; /*dlist node in bind manager */ + u32 flag; +} mali_defer_mem; + + +typedef struct mali_defer_mem_block { + struct list_head free_pages; /* page pool */ + atomic_t num_free_pages; +} mali_defer_mem_block; + +/* varying memory list need to bind */ +typedef struct mali_backend_bind_list { + struct list_head node; + struct mali_mem_backend *bkend; + u32 vaddr; + u32 page_num; + struct mali_session_data *session; + u32 flag; +} mali_backend_bind_lists; + + +typedef struct mali_defer_bind_manager { + atomic_t num_used_pages; + atomic_t num_dmem; +} mali_defer_bind_manager; + +_mali_osk_errcode_t mali_mem_defer_bind_manager_init(void); +void mali_mem_defer_bind_manager_destory(void); +_mali_osk_errcode_t mali_mem_defer_bind(u32 count, struct mali_gp_job *gp, + struct mali_defer_mem_block *dmem_block); +_mali_osk_errcode_t mali_mem_defer_bind_allocation_prepare(mali_mem_allocation *alloc, struct list_head *list); +_mali_osk_errcode_t mali_mem_prepare_mem_for_job(struct mali_gp_job *next_gp_job, mali_defer_mem_block *dblock); +void mali_mem_defer_dmem_free(struct mali_gp_job *gp); + +#endif diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_dma_buf.c b/drivers/gpu/arm/utgard/linux/mali_memory_dma_buf.c new file mode 100644 index 00000000000000..2fa5028d8342a0 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_memory_dma_buf.c @@ -0,0 +1,369 @@ +/* + * Copyright (C) 2012-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include /* file system operations */ +#include /* user space access */ +#include +#include +#include +#include +#include +#include +#include + +#include "mali_ukk.h" +#include "mali_osk.h" +#include "mali_kernel_common.h" +#include "mali_session.h" +#include "mali_kernel_linux.h" + +#include "mali_memory.h" +#include "mali_memory_dma_buf.h" +#include "mali_memory_virtual.h" +#include "mali_pp_job.h" + +/* + * Map DMA buf attachment \a mem into \a session at virtual address \a virt. + */ +static int mali_dma_buf_map(mali_mem_backend *mem_backend) +{ + mali_mem_allocation *alloc; + struct mali_dma_buf_attachment *mem; + struct mali_session_data *session; + struct mali_page_directory *pagedir; + _mali_osk_errcode_t err; + struct scatterlist *sg; + u32 virt, flags; + int i; + + MALI_DEBUG_ASSERT_POINTER(mem_backend); + + alloc = mem_backend->mali_allocation; + MALI_DEBUG_ASSERT_POINTER(alloc); + + mem = mem_backend->dma_buf.attachment; + MALI_DEBUG_ASSERT_POINTER(mem); + + session = alloc->session; + MALI_DEBUG_ASSERT_POINTER(session); + MALI_DEBUG_ASSERT(mem->session == session); + + virt = alloc->mali_vma_node.vm_node.start; + flags = alloc->flags; + + mali_session_memory_lock(session); + mem->map_ref++; + + MALI_DEBUG_PRINT(5, ("Mali DMA-buf: map attachment %p, new map_ref = %d\n", mem, mem->map_ref)); + + if (1 == mem->map_ref) { + + /* First reference taken, so we need to map the dma buf */ + MALI_DEBUG_ASSERT(!mem->is_mapped); + + mem->sgt = dma_buf_map_attachment(mem->attachment, DMA_BIDIRECTIONAL); + if (IS_ERR_OR_NULL(mem->sgt)) { + MALI_DEBUG_PRINT_ERROR(("Failed to map dma-buf attachment\n")); + mem->map_ref--; + mali_session_memory_unlock(session); + return -EFAULT; + } + + err = mali_mem_mali_map_prepare(alloc); + if (_MALI_OSK_ERR_OK != err) { + MALI_DEBUG_PRINT(1, ("Mapping of DMA memory failed\n")); + mem->map_ref--; + mali_session_memory_unlock(session); + return -ENOMEM; + } + + pagedir = mali_session_get_page_directory(session); + MALI_DEBUG_ASSERT_POINTER(pagedir); + + for_each_sg(mem->sgt->sgl, sg, mem->sgt->nents, i) { + u32 size = sg_dma_len(sg); + dma_addr_t phys = sg_dma_address(sg); + + /* sg must be page aligned. */ + MALI_DEBUG_ASSERT(0 == size % MALI_MMU_PAGE_SIZE); + MALI_DEBUG_ASSERT(0 == (phys & ~(uintptr_t)0xFFFFFFFF)); + + mali_mmu_pagedir_update(pagedir, virt, phys, size, MALI_MMU_FLAGS_DEFAULT); + + virt += size; + } + + if (flags & MALI_MEM_FLAG_MALI_GUARD_PAGE) { + u32 guard_phys; + MALI_DEBUG_PRINT(7, ("Mapping in extra guard page\n")); + + guard_phys = sg_dma_address(mem->sgt->sgl); + mali_mmu_pagedir_update(pagedir, virt, guard_phys, MALI_MMU_PAGE_SIZE, MALI_MMU_FLAGS_DEFAULT); + } + + mem->is_mapped = MALI_TRUE; + mali_session_memory_unlock(session); + /* Wake up any thread waiting for buffer to become mapped */ + wake_up_all(&mem->wait_queue); + } else { + MALI_DEBUG_ASSERT(mem->is_mapped); + mali_session_memory_unlock(session); + } + + return 0; +} + +static void mali_dma_buf_unmap(mali_mem_allocation *alloc, struct mali_dma_buf_attachment *mem) +{ + MALI_DEBUG_ASSERT_POINTER(alloc); + MALI_DEBUG_ASSERT_POINTER(mem); + MALI_DEBUG_ASSERT_POINTER(mem->attachment); + MALI_DEBUG_ASSERT_POINTER(mem->buf); + MALI_DEBUG_ASSERT_POINTER(alloc->session); + + mali_session_memory_lock(alloc->session); + mem->map_ref--; + + MALI_DEBUG_PRINT(5, ("Mali DMA-buf: unmap attachment %p, new map_ref = %d\n", mem, mem->map_ref)); + + if (0 == mem->map_ref) { + dma_buf_unmap_attachment(mem->attachment, mem->sgt, DMA_BIDIRECTIONAL); + if (MALI_TRUE == mem->is_mapped) { + mali_mem_mali_map_free(alloc->session, alloc->psize, alloc->mali_vma_node.vm_node.start, + alloc->flags); + } + mem->is_mapped = MALI_FALSE; + } + mali_session_memory_unlock(alloc->session); + /* Wake up any thread waiting for buffer to become unmapped */ + wake_up_all(&mem->wait_queue); +} + +#if !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) +int mali_dma_buf_map_job(struct mali_pp_job *job) +{ + struct mali_dma_buf_attachment *mem; + _mali_osk_errcode_t err; + int i; + int ret = 0; + u32 num_memory_cookies; + struct mali_session_data *session; + struct mali_vma_node *mali_vma_node = NULL; + mali_mem_allocation *mali_alloc = NULL; + mali_mem_backend *mem_bkend = NULL; + + MALI_DEBUG_ASSERT_POINTER(job); + + num_memory_cookies = mali_pp_job_num_memory_cookies(job); + + session = mali_pp_job_get_session(job); + + MALI_DEBUG_ASSERT_POINTER(session); + + for (i = 0; i < num_memory_cookies; i++) { + u32 mali_addr = mali_pp_job_get_memory_cookie(job, i); + mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0); + MALI_DEBUG_ASSERT(NULL != mali_vma_node); + mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); + MALI_DEBUG_ASSERT(NULL != mali_alloc); + if (MALI_MEM_DMA_BUF != mali_alloc->type) { + continue; + } + + /* Get backend memory & Map on CPU */ + mutex_lock(&mali_idr_mutex); + mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle); + mutex_unlock(&mali_idr_mutex); + MALI_DEBUG_ASSERT(NULL != mem_bkend); + + mem = mem_bkend->dma_buf.attachment; + + MALI_DEBUG_ASSERT_POINTER(mem); + MALI_DEBUG_ASSERT(mem->session == mali_pp_job_get_session(job)); + + err = mali_dma_buf_map(mem_bkend); + if (0 != err) { + MALI_DEBUG_PRINT_ERROR(("Mali DMA-buf: Failed to map dma-buf for mali address %x\n", mali_addr)); + ret = -EFAULT; + continue; + } + } + return ret; +} + +void mali_dma_buf_unmap_job(struct mali_pp_job *job) +{ + struct mali_dma_buf_attachment *mem; + int i; + u32 num_memory_cookies; + struct mali_session_data *session; + struct mali_vma_node *mali_vma_node = NULL; + mali_mem_allocation *mali_alloc = NULL; + mali_mem_backend *mem_bkend = NULL; + + MALI_DEBUG_ASSERT_POINTER(job); + + num_memory_cookies = mali_pp_job_num_memory_cookies(job); + + session = mali_pp_job_get_session(job); + + MALI_DEBUG_ASSERT_POINTER(session); + + for (i = 0; i < num_memory_cookies; i++) { + u32 mali_addr = mali_pp_job_get_memory_cookie(job, i); + mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0); + MALI_DEBUG_ASSERT(NULL != mali_vma_node); + mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); + MALI_DEBUG_ASSERT(NULL != mali_alloc); + if (MALI_MEM_DMA_BUF != mali_alloc->type) { + continue; + } + + /* Get backend memory & Map on CPU */ + mutex_lock(&mali_idr_mutex); + mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle); + mutex_unlock(&mali_idr_mutex); + MALI_DEBUG_ASSERT(NULL != mem_bkend); + + mem = mem_bkend->dma_buf.attachment; + + MALI_DEBUG_ASSERT_POINTER(mem); + MALI_DEBUG_ASSERT(mem->session == mali_pp_job_get_session(job)); + mali_dma_buf_unmap(mem_bkend->mali_allocation, mem); + } +} +#endif /* !CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH */ + +int mali_dma_buf_get_size(struct mali_session_data *session, _mali_uk_dma_buf_get_size_s __user *user_arg) +{ + _mali_uk_dma_buf_get_size_s args; + int fd; + struct dma_buf *buf; + + /* get call arguments from user space. copy_from_user returns how many bytes which where NOT copied */ + if (0 != copy_from_user(&args, (void __user *)user_arg, sizeof(_mali_uk_dma_buf_get_size_s))) { + return -EFAULT; + } + + /* Do DMA-BUF stuff */ + fd = args.mem_fd; + + buf = dma_buf_get(fd); + if (IS_ERR_OR_NULL(buf)) { + MALI_DEBUG_PRINT_ERROR(("Failed to get dma-buf from fd: %d\n", fd)); + return PTR_RET(buf); + } + + if (0 != put_user(buf->size, &user_arg->size)) { + dma_buf_put(buf); + return -EFAULT; + } + + dma_buf_put(buf); + + return 0; +} + +_mali_osk_errcode_t mali_mem_bind_dma_buf(mali_mem_allocation *alloc, + mali_mem_backend *mem_backend, + int fd, u32 flags) +{ + struct dma_buf *buf; + struct mali_dma_buf_attachment *dma_mem; + struct mali_session_data *session = alloc->session; + + MALI_DEBUG_ASSERT_POINTER(session); + MALI_DEBUG_ASSERT_POINTER(mem_backend); + MALI_DEBUG_ASSERT_POINTER(alloc); + + /* get dma buffer */ + buf = dma_buf_get(fd); + if (IS_ERR_OR_NULL(buf)) { + return _MALI_OSK_ERR_FAULT; + } + + /* Currently, mapping of the full buffer are supported. */ + if (alloc->psize != buf->size) { + goto failed_alloc_mem; + } + + dma_mem = _mali_osk_calloc(1, sizeof(struct mali_dma_buf_attachment)); + if (NULL == dma_mem) { + goto failed_alloc_mem; + } + + dma_mem->buf = buf; + dma_mem->session = session; + dma_mem->map_ref = 0; + init_waitqueue_head(&dma_mem->wait_queue); + + dma_mem->attachment = dma_buf_attach(dma_mem->buf, &mali_platform_device->dev); + if (NULL == dma_mem->attachment) { + goto failed_dma_attach; + } + + mem_backend->dma_buf.attachment = dma_mem; + + alloc->flags |= MALI_MEM_FLAG_DONT_CPU_MAP; + if (flags & _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE) { + alloc->flags |= MALI_MEM_FLAG_MALI_GUARD_PAGE; + } + + +#if defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) + /* Map memory into session's Mali virtual address space. */ + if (0 != mali_dma_buf_map(mem_backend)) { + goto Failed_dma_map; + } +#endif + + return _MALI_OSK_ERR_OK; + +#if defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) +Failed_dma_map: + mali_dma_buf_unmap(alloc, dma_mem); +#endif + /* Wait for buffer to become unmapped */ + wait_event(dma_mem->wait_queue, !dma_mem->is_mapped); + MALI_DEBUG_ASSERT(!dma_mem->is_mapped); + dma_buf_detach(dma_mem->buf, dma_mem->attachment); +failed_dma_attach: + _mali_osk_free(dma_mem); +failed_alloc_mem: + dma_buf_put(buf); + return _MALI_OSK_ERR_FAULT; +} + +void mali_mem_unbind_dma_buf(mali_mem_backend *mem_backend) +{ + struct mali_dma_buf_attachment *mem; + MALI_DEBUG_ASSERT_POINTER(mem_backend); + MALI_DEBUG_ASSERT(MALI_MEM_DMA_BUF == mem_backend->type); + + mem = mem_backend->dma_buf.attachment; + MALI_DEBUG_ASSERT_POINTER(mem); + MALI_DEBUG_ASSERT_POINTER(mem->attachment); + MALI_DEBUG_ASSERT_POINTER(mem->buf); + MALI_DEBUG_PRINT(3, ("Mali DMA-buf: release attachment %p\n", mem)); + +#if defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) + MALI_DEBUG_ASSERT_POINTER(mem_backend->mali_allocation); + /* We mapped implicitly on attach, so we need to unmap on release */ + mali_dma_buf_unmap(mem_backend->mali_allocation, mem); +#endif + /* Wait for buffer to become unmapped */ + wait_event(mem->wait_queue, !mem->is_mapped); + MALI_DEBUG_ASSERT(!mem->is_mapped); + + dma_buf_detach(mem->buf, mem->attachment); + dma_buf_put(mem->buf); + + _mali_osk_free(mem); +} diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_dma_buf.h b/drivers/gpu/arm/utgard/linux/mali_memory_dma_buf.h new file mode 100644 index 00000000000000..859d3849e6b34c --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_memory_dma_buf.h @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_MEMORY_DMA_BUF_H__ +#define __MALI_MEMORY_DMA_BUF_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "mali_uk_types.h" +#include "mali_osk.h" +#include "mali_memory.h" + +struct mali_pp_job; + +struct mali_dma_buf_attachment; +struct mali_dma_buf_attachment { + struct dma_buf *buf; + struct dma_buf_attachment *attachment; + struct sg_table *sgt; + struct mali_session_data *session; + int map_ref; + struct mutex map_lock; + mali_bool is_mapped; + wait_queue_head_t wait_queue; +}; + +int mali_dma_buf_get_size(struct mali_session_data *session, _mali_uk_dma_buf_get_size_s __user *arg); + +void mali_mem_unbind_dma_buf(mali_mem_backend *mem_backend); + +_mali_osk_errcode_t mali_mem_bind_dma_buf(mali_mem_allocation *alloc, + mali_mem_backend *mem_backend, + int fd, u32 flags); + +#if !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) +int mali_dma_buf_map_job(struct mali_pp_job *job); +void mali_dma_buf_unmap_job(struct mali_pp_job *job); +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* __MALI_MEMORY_DMA_BUF_H__ */ diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_external.c b/drivers/gpu/arm/utgard/linux/mali_memory_external.c new file mode 100644 index 00000000000000..3733f2e0c80add --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_memory_external.c @@ -0,0 +1,91 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_kernel_common.h" +#include "mali_osk.h" +#include "mali_ukk.h" +#include "mali_memory.h" +#include "mali_mem_validation.h" +#include "mali_uk_types.h" + +void mali_mem_unbind_ext_buf(mali_mem_backend *mem_backend) +{ + mali_mem_allocation *alloc; + struct mali_session_data *session; + MALI_DEBUG_ASSERT_POINTER(mem_backend); + alloc = mem_backend->mali_allocation; + MALI_DEBUG_ASSERT_POINTER(alloc); + MALI_DEBUG_ASSERT(MALI_MEM_EXTERNAL == mem_backend->type); + + session = alloc->session; + MALI_DEBUG_ASSERT_POINTER(session); + mali_session_memory_lock(session); + mali_mem_mali_map_free(session, alloc->psize, alloc->mali_vma_node.vm_node.start, + alloc->flags); + mali_session_memory_unlock(session); +} + +_mali_osk_errcode_t mali_mem_bind_ext_buf(mali_mem_allocation *alloc, + mali_mem_backend *mem_backend, + u32 phys_addr, + u32 flag) +{ + struct mali_session_data *session; + _mali_osk_errcode_t err; + u32 virt, phys, size; + MALI_DEBUG_ASSERT_POINTER(mem_backend); + MALI_DEBUG_ASSERT_POINTER(alloc); + size = alloc->psize; + session = (struct mali_session_data *)(uintptr_t)alloc->session; + MALI_CHECK_NON_NULL(session, _MALI_OSK_ERR_INVALID_ARGS); + + /* check arguments */ + /* NULL might be a valid Mali address */ + if (!size) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); + + /* size must be a multiple of the system page size */ + if (size % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); + +#if 0 + /* Validate the mali physical range */ + if (_MALI_OSK_ERR_OK != mali_mem_validation_check(phys_addr, size)) { + return _MALI_OSK_ERR_FAULT; + } +#endif + + if (flag & _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE) { + alloc->flags |= MALI_MEM_FLAG_MALI_GUARD_PAGE; + } + + mali_session_memory_lock(session); + + virt = alloc->mali_vma_node.vm_node.start; + phys = phys_addr; + + err = mali_mem_mali_map_prepare(alloc); + if (_MALI_OSK_ERR_OK != err) { + mali_session_memory_unlock(session); + return _MALI_OSK_ERR_NOMEM; + } + + mali_mmu_pagedir_update(session->page_directory, virt, phys, size, MALI_MMU_FLAGS_DEFAULT); + + if (alloc->flags & MALI_MEM_FLAG_MALI_GUARD_PAGE) { + mali_mmu_pagedir_update(session->page_directory, virt + size, phys, _MALI_OSK_MALI_PAGE_SIZE, MALI_MMU_FLAGS_DEFAULT); + } + MALI_DEBUG_PRINT(3, + ("Requested to map physical memory 0x%x-0x%x into virtual memory 0x%x\n", + phys_addr, (phys_addr + size - 1), + virt)); + mali_session_memory_unlock(session); + + MALI_SUCCESS; +} + diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_external.h b/drivers/gpu/arm/utgard/linux/mali_memory_external.h new file mode 100644 index 00000000000000..645580b51fc989 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_memory_external.h @@ -0,0 +1,29 @@ + +/* + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_MEMORY_EXTERNAL_H__ +#define __MALI_MEMORY_EXTERNAL_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +_mali_osk_errcode_t mali_mem_bind_ext_buf(mali_mem_allocation *alloc, + mali_mem_backend *mem_backend, + u32 phys_addr, + u32 flag); +void mali_mem_unbind_ext_buf(mali_mem_backend *mem_backend); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_manager.c b/drivers/gpu/arm/utgard/linux/mali_memory_manager.c new file mode 100644 index 00000000000000..55e2c092d597c4 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_memory_manager.c @@ -0,0 +1,965 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#if defined(CONFIG_DMA_SHARED_BUFFER) +#include +#endif +#include + +#include "mali_osk.h" +#include "mali_osk_mali.h" +#include "mali_kernel_linux.h" +#include "mali_scheduler.h" +#include "mali_memory.h" +#include "mali_memory_os_alloc.h" +#if defined(CONFIG_DMA_SHARED_BUFFER) +#include "mali_memory_dma_buf.h" +#endif +#if defined(CONFIG_MALI400_UMP) +#include "mali_memory_ump.h" +#endif +#include "mali_memory_manager.h" +#include "mali_memory_virtual.h" +#include "mali_memory_util.h" +#include "mali_memory_external.h" +#include "mali_memory_cow.h" +#include "mali_memory_block_alloc.h" +#include "mali_ukk.h" +#include "mali_memory_swap_alloc.h" + +/* +* New memory system interface +*/ + +/*inti idr for backend memory */ +struct idr mali_backend_idr; +struct mutex mali_idr_mutex; + +/* init allocation manager */ +int mali_memory_manager_init(struct mali_allocation_manager *mgr) +{ + /* init Locks */ + rwlock_init(&mgr->vm_lock); + mutex_init(&mgr->list_mutex); + + /* init link */ + INIT_LIST_HEAD(&mgr->head); + + /* init RB tree */ + mgr->allocation_mgr_rb = RB_ROOT; + mgr->mali_allocation_num = 0; + return 0; +} + +/* Deinit allocation manager +* Do some check for debug +*/ +void mali_memory_manager_uninit(struct mali_allocation_manager *mgr) +{ + /* check RB tree is empty */ + MALI_DEBUG_ASSERT(((void *)(mgr->allocation_mgr_rb.rb_node) == (void *)rb_last(&mgr->allocation_mgr_rb))); + /* check allocation List */ + MALI_DEBUG_ASSERT(list_empty(&mgr->head)); +} + +/* Prepare memory descriptor */ +static mali_mem_allocation *mali_mem_allocation_struct_create(struct mali_session_data *session) +{ + mali_mem_allocation *mali_allocation; + + /* Allocate memory */ + mali_allocation = (mali_mem_allocation *)kzalloc(sizeof(mali_mem_allocation), GFP_KERNEL); + if (NULL == mali_allocation) { + MALI_DEBUG_PRINT(1, ("mali_mem_allocation_struct_create: descriptor was NULL\n")); + return NULL; + } + + MALI_DEBUG_CODE(mali_allocation->magic = MALI_MEM_ALLOCATION_VALID_MAGIC); + + /* do init */ + mali_allocation->flags = 0; + mali_allocation->session = session; + + INIT_LIST_HEAD(&mali_allocation->list); + _mali_osk_atomic_init(&mali_allocation->mem_alloc_refcount, 1); + + /** + *add to session list + */ + mutex_lock(&session->allocation_mgr.list_mutex); + list_add_tail(&mali_allocation->list, &session->allocation_mgr.head); + session->allocation_mgr.mali_allocation_num++; + mutex_unlock(&session->allocation_mgr.list_mutex); + + return mali_allocation; +} + +void mali_mem_allocation_struct_destory(mali_mem_allocation *alloc) +{ + MALI_DEBUG_ASSERT_POINTER(alloc); + MALI_DEBUG_ASSERT_POINTER(alloc->session); + mutex_lock(&alloc->session->allocation_mgr.list_mutex); + list_del(&alloc->list); + alloc->session->allocation_mgr.mali_allocation_num--; + mutex_unlock(&alloc->session->allocation_mgr.list_mutex); + + kfree(alloc); +} + +int mali_mem_backend_struct_create(mali_mem_backend **backend, u32 psize) +{ + mali_mem_backend *mem_backend = NULL; + s32 ret = -ENOSPC; + s32 index = -1; + *backend = (mali_mem_backend *)kzalloc(sizeof(mali_mem_backend), GFP_KERNEL); + if (NULL == *backend) { + MALI_DEBUG_PRINT(1, ("mali_mem_backend_struct_create: backend descriptor was NULL\n")); + return -1; + } + mem_backend = *backend; + mem_backend->size = psize; + mutex_init(&mem_backend->mutex); + INIT_LIST_HEAD(&mem_backend->list); + mem_backend->using_count = 0; + + + /* link backend with id */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 9, 0) +again: + if (!idr_pre_get(&mali_backend_idr, GFP_KERNEL)) { + kfree(mem_backend); + return -ENOMEM; + } + mutex_lock(&mali_idr_mutex); + ret = idr_get_new_above(&mali_backend_idr, mem_backend, 1, &index); + mutex_unlock(&mali_idr_mutex); + + if (-ENOSPC == ret) { + kfree(mem_backend); + return -ENOSPC; + } + if (-EAGAIN == ret) + goto again; +#else + mutex_lock(&mali_idr_mutex); + ret = idr_alloc(&mali_backend_idr, mem_backend, 1, MALI_S32_MAX, GFP_KERNEL); + mutex_unlock(&mali_idr_mutex); + index = ret; + if (ret < 0) { + MALI_DEBUG_PRINT(1, ("mali_mem_backend_struct_create: Can't allocate idr for backend! \n")); + kfree(mem_backend); + return -ENOSPC; + } +#endif + return index; +} + + +static void mali_mem_backend_struct_destory(mali_mem_backend **backend, s32 backend_handle) +{ + mali_mem_backend *mem_backend = *backend; + + mutex_lock(&mali_idr_mutex); + idr_remove(&mali_backend_idr, backend_handle); + mutex_unlock(&mali_idr_mutex); + kfree(mem_backend); + *backend = NULL; +} + +mali_mem_backend *mali_mem_backend_struct_search(struct mali_session_data *session, u32 mali_address) +{ + struct mali_vma_node *mali_vma_node = NULL; + mali_mem_backend *mem_bkend = NULL; + mali_mem_allocation *mali_alloc = NULL; + MALI_DEBUG_ASSERT_POINTER(session); + mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_address, 0); + if (NULL == mali_vma_node) { + MALI_DEBUG_PRINT(1, ("mali_mem_backend_struct_search:vma node was NULL\n")); + return NULL; + } + mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); + /* Get backend memory & Map on CPU */ + mutex_lock(&mali_idr_mutex); + mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle); + mutex_unlock(&mali_idr_mutex); + MALI_DEBUG_ASSERT(NULL != mem_bkend); + return mem_bkend; +} + +static _mali_osk_errcode_t mali_mem_resize(struct mali_session_data *session, mali_mem_backend *mem_backend, u32 physical_size) +{ + _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; + int retval = 0; + mali_mem_allocation *mali_allocation = NULL; + mali_mem_os_mem tmp_os_mem; + s32 change_page_count; + + MALI_DEBUG_ASSERT_POINTER(session); + MALI_DEBUG_ASSERT_POINTER(mem_backend); + MALI_DEBUG_PRINT(4, (" mali_mem_resize_memory called! \n")); + MALI_DEBUG_ASSERT(0 == physical_size % MALI_MMU_PAGE_SIZE); + + mali_allocation = mem_backend->mali_allocation; + MALI_DEBUG_ASSERT_POINTER(mali_allocation); + + MALI_DEBUG_ASSERT(MALI_MEM_FLAG_CAN_RESIZE & mali_allocation->flags); + MALI_DEBUG_ASSERT(MALI_MEM_OS == mali_allocation->type); + + mutex_lock(&mem_backend->mutex); + + /* Do resize*/ + if (physical_size > mem_backend->size) { + u32 add_size = physical_size - mem_backend->size; + + MALI_DEBUG_ASSERT(0 == add_size % MALI_MMU_PAGE_SIZE); + + /* Allocate new pages from os mem */ + retval = mali_mem_os_alloc_pages(&tmp_os_mem, add_size); + + if (retval) { + if (-ENOMEM == retval) { + ret = _MALI_OSK_ERR_NOMEM; + } else { + ret = _MALI_OSK_ERR_FAULT; + } + MALI_DEBUG_PRINT(2, ("_mali_ukk_mem_resize: memory allocation failed !\n")); + goto failed_alloc_memory; + } + + MALI_DEBUG_ASSERT(tmp_os_mem.count == add_size / MALI_MMU_PAGE_SIZE); + + /* Resize the memory of the backend */ + ret = mali_mem_os_resize_pages(&tmp_os_mem, &mem_backend->os_mem, 0, tmp_os_mem.count); + + if (ret) { + MALI_DEBUG_PRINT(2, ("_mali_ukk_mem_resize: memory resizing failed !\n")); + goto failed_resize_pages; + } + + /*Resize cpu mapping */ + if (NULL != mali_allocation->cpu_mapping.vma) { + ret = mali_mem_os_resize_cpu_map_locked(mem_backend, mali_allocation->cpu_mapping.vma, mali_allocation->cpu_mapping.vma->vm_start + mem_backend->size, add_size); + if (unlikely(ret != _MALI_OSK_ERR_OK)) { + MALI_DEBUG_PRINT(2, ("_mali_ukk_mem_resize: cpu mapping failed !\n")); + goto failed_cpu_map; + } + } + + /* Resize mali mapping */ + _mali_osk_mutex_wait(session->memory_lock); + ret = mali_mem_mali_map_resize(mali_allocation, physical_size); + + if (ret) { + MALI_DEBUG_PRINT(1, ("_mali_ukk_mem_resize: mali map resize fail !\n")); + goto failed_gpu_map; + } + + ret = mali_mem_os_mali_map(&mem_backend->os_mem, session, mali_allocation->mali_vma_node.vm_node.start, + mali_allocation->psize / MALI_MMU_PAGE_SIZE, add_size / MALI_MMU_PAGE_SIZE, mali_allocation->mali_mapping.properties); + if (ret) { + MALI_DEBUG_PRINT(2, ("_mali_ukk_mem_resize: mali mapping failed !\n")); + goto failed_gpu_map; + } + + _mali_osk_mutex_signal(session->memory_lock); + } else { + u32 dec_size, page_count; + u32 vaddr = 0; + INIT_LIST_HEAD(&tmp_os_mem.pages); + tmp_os_mem.count = 0; + + dec_size = mem_backend->size - physical_size; + MALI_DEBUG_ASSERT(0 == dec_size % MALI_MMU_PAGE_SIZE); + + page_count = dec_size / MALI_MMU_PAGE_SIZE; + vaddr = mali_allocation->mali_vma_node.vm_node.start + physical_size; + + /* Resize the memory of the backend */ + ret = mali_mem_os_resize_pages(&mem_backend->os_mem, &tmp_os_mem, physical_size / MALI_MMU_PAGE_SIZE, page_count); + + if (ret) { + MALI_DEBUG_PRINT(4, ("_mali_ukk_mem_resize: mali map resize failed!\n")); + goto failed_resize_pages; + } + + /* Resize mali map */ + _mali_osk_mutex_wait(session->memory_lock); + mali_mem_mali_map_free(session, dec_size, vaddr, mali_allocation->flags); + _mali_osk_mutex_signal(session->memory_lock); + + /* Zap cpu mapping */ + if (0 != mali_allocation->cpu_mapping.addr) { + MALI_DEBUG_ASSERT(NULL != mali_allocation->cpu_mapping.vma); + zap_vma_ptes(mali_allocation->cpu_mapping.vma, mali_allocation->cpu_mapping.vma->vm_start + physical_size, dec_size); + } + + /* Free those extra pages */ + mali_mem_os_free(&tmp_os_mem.pages, tmp_os_mem.count, MALI_FALSE); + } + + /* Resize memory allocation and memory backend */ + change_page_count = (s32)(physical_size - mem_backend->size) / MALI_MMU_PAGE_SIZE; + mali_allocation->psize = physical_size; + mem_backend->size = physical_size; + mutex_unlock(&mem_backend->mutex); + + if (change_page_count > 0) { + atomic_add(change_page_count, &session->mali_mem_allocated_pages); + if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) { + session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE; + } + + } else { + atomic_sub((s32)(-change_page_count), &session->mali_mem_allocated_pages); + } + + return _MALI_OSK_ERR_OK; + +failed_gpu_map: + _mali_osk_mutex_signal(session->memory_lock); +failed_cpu_map: + if (physical_size > mem_backend->size) { + mali_mem_os_resize_pages(&mem_backend->os_mem, &tmp_os_mem, mem_backend->size / MALI_MMU_PAGE_SIZE, + (physical_size - mem_backend->size) / MALI_MMU_PAGE_SIZE); + } else { + mali_mem_os_resize_pages(&tmp_os_mem, &mem_backend->os_mem, 0, tmp_os_mem.count); + } +failed_resize_pages: + if (0 != tmp_os_mem.count) + mali_mem_os_free(&tmp_os_mem.pages, tmp_os_mem.count, MALI_FALSE); +failed_alloc_memory: + + mutex_unlock(&mem_backend->mutex); + return ret; +} + + +/* Set GPU MMU properties */ +static void _mali_memory_gpu_map_property_set(u32 *properties, u32 flags) +{ + if (_MALI_MEMORY_GPU_READ_ALLOCATE & flags) { + *properties = MALI_MMU_FLAGS_FORCE_GP_READ_ALLOCATE; + } else { + *properties = MALI_MMU_FLAGS_DEFAULT; + } +} + +_mali_osk_errcode_t mali_mem_add_mem_size(struct mali_session_data *session, u32 mali_addr, u32 add_size) +{ + mali_mem_backend *mem_backend = NULL; + _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; + mali_mem_allocation *mali_allocation = NULL; + u32 new_physical_size; + MALI_DEBUG_ASSERT_POINTER(session); + MALI_DEBUG_ASSERT(0 == add_size % MALI_MMU_PAGE_SIZE); + + /* Get the memory backend that need to be resize. */ + mem_backend = mali_mem_backend_struct_search(session, mali_addr); + + if (NULL == mem_backend) { + MALI_DEBUG_PRINT(2, ("_mali_ukk_mem_resize: memory backend = NULL!\n")); + return ret; + } + + mali_allocation = mem_backend->mali_allocation; + + MALI_DEBUG_ASSERT_POINTER(mali_allocation); + + new_physical_size = add_size + mem_backend->size; + + if (new_physical_size > (mali_allocation->mali_vma_node.vm_node.size)) + return ret; + + MALI_DEBUG_ASSERT(new_physical_size != mem_backend->size); + + ret = mali_mem_resize(session, mem_backend, new_physical_size); + + return ret; +} + +/** +* function@_mali_ukk_mem_allocate - allocate mali memory +*/ +_mali_osk_errcode_t _mali_ukk_mem_allocate(_mali_uk_alloc_mem_s *args) +{ + struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; + mali_mem_backend *mem_backend = NULL; + _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; + int retval = 0; + mali_mem_allocation *mali_allocation = NULL; + struct mali_vma_node *mali_vma_node = NULL; + + MALI_DEBUG_PRINT(4, (" _mali_ukk_mem_allocate, vaddr=0x%x, size =0x%x! \n", args->gpu_vaddr, args->psize)); + + /* Check if the address is allocated + */ + mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, args->gpu_vaddr, 0); + + if (unlikely(mali_vma_node)) { + MALI_DEBUG_ASSERT(0); + return _MALI_OSK_ERR_FAULT; + } + /** + *create mali memory allocation + */ + + mali_allocation = mali_mem_allocation_struct_create(session); + + if (mali_allocation == NULL) { + MALI_DEBUG_PRINT(1, ("_mali_ukk_mem_allocate: Failed to create allocation struct! \n")); + return _MALI_OSK_ERR_NOMEM; + } + mali_allocation->psize = args->psize; + mali_allocation->vsize = args->vsize; + + /* MALI_MEM_OS if need to support mem resize, + * or MALI_MEM_BLOCK if have dedicated memory, + * or MALI_MEM_OS, + * or MALI_MEM_SWAP. + */ + if (args->flags & _MALI_MEMORY_ALLOCATE_SWAPPABLE) { + mali_allocation->type = MALI_MEM_SWAP; + } else if (args->flags & _MALI_MEMORY_ALLOCATE_RESIZEABLE) { + mali_allocation->type = MALI_MEM_OS; + mali_allocation->flags |= MALI_MEM_FLAG_CAN_RESIZE; + } else if (MALI_TRUE == mali_memory_have_dedicated_memory()) { + mali_allocation->type = MALI_MEM_BLOCK; + } else { + mali_allocation->type = MALI_MEM_OS; + } + + /** + *add allocation node to RB tree for index + */ + mali_allocation->mali_vma_node.vm_node.start = args->gpu_vaddr; + mali_allocation->mali_vma_node.vm_node.size = args->vsize; + + mali_vma_offset_add(&session->allocation_mgr, &mali_allocation->mali_vma_node); + + mali_allocation->backend_handle = mali_mem_backend_struct_create(&mem_backend, args->psize); + if (mali_allocation->backend_handle < 0) { + ret = _MALI_OSK_ERR_NOMEM; + MALI_DEBUG_PRINT(1, ("mali_allocation->backend_handle < 0! \n")); + goto failed_alloc_backend; + } + + + mem_backend->mali_allocation = mali_allocation; + mem_backend->type = mali_allocation->type; + + mali_allocation->mali_mapping.addr = args->gpu_vaddr; + + /* set gpu mmu propery */ + _mali_memory_gpu_map_property_set(&mali_allocation->mali_mapping.properties, args->flags); + /* do prepare for MALI mapping */ + if (!(args->flags & _MALI_MEMORY_ALLOCATE_NO_BIND_GPU) && mali_allocation->psize > 0) { + _mali_osk_mutex_wait(session->memory_lock); + + ret = mali_mem_mali_map_prepare(mali_allocation); + if (0 != ret) { + _mali_osk_mutex_signal(session->memory_lock); + goto failed_prepare_map; + } + _mali_osk_mutex_signal(session->memory_lock); + } + + if (mali_allocation->psize == 0) { + mem_backend->os_mem.count = 0; + INIT_LIST_HEAD(&mem_backend->os_mem.pages); + goto done; + } + + if (args->flags & _MALI_MEMORY_ALLOCATE_DEFER_BIND) { + mali_allocation->flags |= _MALI_MEMORY_ALLOCATE_DEFER_BIND; + mem_backend->flags |= MALI_MEM_BACKEND_FLAG_NOT_BINDED; + /* init for defer bind backend*/ + mem_backend->os_mem.count = 0; + INIT_LIST_HEAD(&mem_backend->os_mem.pages); + + goto done; + } + /** + *allocate physical memory + */ + if (likely(mali_allocation->psize > 0)) { + + if (mem_backend->type == MALI_MEM_OS) { + retval = mali_mem_os_alloc_pages(&mem_backend->os_mem, mem_backend->size); + } else if (mem_backend->type == MALI_MEM_BLOCK) { + /* try to allocated from BLOCK memory first, then try OS memory if failed.*/ + if (mali_mem_block_alloc(&mem_backend->block_mem, mem_backend->size)) { + retval = mali_mem_os_alloc_pages(&mem_backend->os_mem, mem_backend->size); + mem_backend->type = MALI_MEM_OS; + mali_allocation->type = MALI_MEM_OS; + } + } else if (MALI_MEM_SWAP == mem_backend->type) { + retval = mali_mem_swap_alloc_pages(&mem_backend->swap_mem, mali_allocation->mali_vma_node.vm_node.size, &mem_backend->start_idx); + } else { + /* ONLY support mem_os type */ + MALI_DEBUG_ASSERT(0); + } + + if (retval) { + ret = _MALI_OSK_ERR_NOMEM; + MALI_DEBUG_PRINT(1, (" can't allocate enough pages! \n")); + goto failed_alloc_pages; + } + } + + /** + *map to GPU side + */ + if (!(args->flags & _MALI_MEMORY_ALLOCATE_NO_BIND_GPU) && mali_allocation->psize > 0) { + _mali_osk_mutex_wait(session->memory_lock); + /* Map on Mali */ + + if (mem_backend->type == MALI_MEM_OS) { + ret = mali_mem_os_mali_map(&mem_backend->os_mem, session, args->gpu_vaddr, 0, + mem_backend->size / MALI_MMU_PAGE_SIZE, mali_allocation->mali_mapping.properties); + + } else if (mem_backend->type == MALI_MEM_BLOCK) { + mali_mem_block_mali_map(&mem_backend->block_mem, session, args->gpu_vaddr, + mali_allocation->mali_mapping.properties); + } else if (mem_backend->type == MALI_MEM_SWAP) { + ret = mali_mem_swap_mali_map(&mem_backend->swap_mem, session, args->gpu_vaddr, + mali_allocation->mali_mapping.properties); + } else { /* unsupport type */ + MALI_DEBUG_ASSERT(0); + } + + _mali_osk_mutex_signal(session->memory_lock); + } +done: + if (MALI_MEM_OS == mem_backend->type) { + atomic_add(mem_backend->os_mem.count, &session->mali_mem_allocated_pages); + } else if (MALI_MEM_BLOCK == mem_backend->type) { + atomic_add(mem_backend->block_mem.count, &session->mali_mem_allocated_pages); + } else { + MALI_DEBUG_ASSERT(MALI_MEM_SWAP == mem_backend->type); + atomic_add(mem_backend->swap_mem.count, &session->mali_mem_allocated_pages); + atomic_add(mem_backend->swap_mem.count, &session->mali_mem_array[mem_backend->type]); + } + + if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) { + session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE; + } + return _MALI_OSK_ERR_OK; + +failed_alloc_pages: + mali_mem_mali_map_free(session, mali_allocation->psize, mali_allocation->mali_vma_node.vm_node.start, mali_allocation->flags); +failed_prepare_map: + mali_mem_backend_struct_destory(&mem_backend, mali_allocation->backend_handle); +failed_alloc_backend: + + mali_vma_offset_remove(&session->allocation_mgr, &mali_allocation->mali_vma_node); + mali_mem_allocation_struct_destory(mali_allocation); + + return ret; +} + + +_mali_osk_errcode_t _mali_ukk_mem_free(_mali_uk_free_mem_s *args) +{ + struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; + u32 vaddr = args->gpu_vaddr; + mali_mem_allocation *mali_alloc = NULL; + struct mali_vma_node *mali_vma_node = NULL; + + /* find mali allocation structure by vaddress*/ + mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, vaddr, 0); + if (NULL == mali_vma_node) { + MALI_DEBUG_PRINT(1, ("_mali_ukk_mem_free: invalid addr: 0x%x\n", vaddr)); + return _MALI_OSK_ERR_INVALID_ARGS; + } + MALI_DEBUG_ASSERT(NULL != mali_vma_node); + mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); + + if (mali_alloc) + /* check ref_count */ + args->free_pages_nr = mali_allocation_unref(&mali_alloc); + + return _MALI_OSK_ERR_OK; +} + + +/** +* Function _mali_ukk_mem_bind -- bind a external memory to a new GPU address +* It will allocate a new mem allocation and bind external memory to it. +* Supported backend type are: +* _MALI_MEMORY_BIND_BACKEND_UMP +* _MALI_MEMORY_BIND_BACKEND_DMA_BUF +* _MALI_MEMORY_BIND_BACKEND_EXTERNAL_MEMORY +* CPU access is not supported yet +*/ +_mali_osk_errcode_t _mali_ukk_mem_bind(_mali_uk_bind_mem_s *args) +{ + struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; + mali_mem_backend *mem_backend = NULL; + _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; + mali_mem_allocation *mali_allocation = NULL; + MALI_DEBUG_PRINT(5, (" _mali_ukk_mem_bind, vaddr=0x%x, size =0x%x! \n", args->vaddr, args->size)); + + /** + * allocate mali allocation. + */ + mali_allocation = mali_mem_allocation_struct_create(session); + + if (mali_allocation == NULL) { + return _MALI_OSK_ERR_NOMEM; + } + mali_allocation->psize = args->size; + mali_allocation->vsize = args->size; + mali_allocation->mali_mapping.addr = args->vaddr; + + /* add allocation node to RB tree for index */ + mali_allocation->mali_vma_node.vm_node.start = args->vaddr; + mali_allocation->mali_vma_node.vm_node.size = args->size; + mali_vma_offset_add(&session->allocation_mgr, &mali_allocation->mali_vma_node); + + /* allocate backend*/ + if (mali_allocation->psize > 0) { + mali_allocation->backend_handle = mali_mem_backend_struct_create(&mem_backend, mali_allocation->psize); + if (mali_allocation->backend_handle < 0) { + goto Failed_alloc_backend; + } + + } else { + goto Failed_alloc_backend; + } + + mem_backend->size = mali_allocation->psize; + mem_backend->mali_allocation = mali_allocation; + + switch (args->flags & _MALI_MEMORY_BIND_BACKEND_MASK) { + case _MALI_MEMORY_BIND_BACKEND_UMP: +#if defined(CONFIG_MALI400_UMP) + mali_allocation->type = MALI_MEM_UMP; + mem_backend->type = MALI_MEM_UMP; + ret = mali_mem_bind_ump_buf(mali_allocation, mem_backend, + args->mem_union.bind_ump.secure_id, args->mem_union.bind_ump.flags); + if (_MALI_OSK_ERR_OK != ret) { + MALI_DEBUG_PRINT(1, ("Bind ump buf failed\n")); + goto Failed_bind_backend; + } +#else + MALI_DEBUG_PRINT(1, ("UMP not supported\n")); + goto Failed_bind_backend; +#endif + break; + case _MALI_MEMORY_BIND_BACKEND_DMA_BUF: +#if defined(CONFIG_DMA_SHARED_BUFFER) + mali_allocation->type = MALI_MEM_DMA_BUF; + mem_backend->type = MALI_MEM_DMA_BUF; + ret = mali_mem_bind_dma_buf(mali_allocation, mem_backend, + args->mem_union.bind_dma_buf.mem_fd, args->mem_union.bind_dma_buf.flags); + if (_MALI_OSK_ERR_OK != ret) { + MALI_DEBUG_PRINT(1, ("Bind dma buf failed\n")); + goto Failed_bind_backend; + } +#else + MALI_DEBUG_PRINT(1, ("DMA not supported\n")); + goto Failed_bind_backend; +#endif + break; + case _MALI_MEMORY_BIND_BACKEND_MALI_MEMORY: + /* not allowed */ + MALI_DEBUG_ASSERT(0); + break; + + case _MALI_MEMORY_BIND_BACKEND_EXTERNAL_MEMORY: + mali_allocation->type = MALI_MEM_EXTERNAL; + mem_backend->type = MALI_MEM_EXTERNAL; + ret = mali_mem_bind_ext_buf(mali_allocation, mem_backend, args->mem_union.bind_ext_memory.phys_addr, + args->mem_union.bind_ext_memory.flags); + if (_MALI_OSK_ERR_OK != ret) { + MALI_DEBUG_PRINT(1, ("Bind external buf failed\n")); + goto Failed_bind_backend; + } + break; + + case _MALI_MEMORY_BIND_BACKEND_EXT_COW: + /* not allowed */ + MALI_DEBUG_ASSERT(0); + break; + + default: + MALI_DEBUG_ASSERT(0); + break; + } + MALI_DEBUG_ASSERT(0 == mem_backend->size % MALI_MMU_PAGE_SIZE); + atomic_add(mem_backend->size / MALI_MMU_PAGE_SIZE, &session->mali_mem_array[mem_backend->type]); + return _MALI_OSK_ERR_OK; + +Failed_bind_backend: + mali_mem_backend_struct_destory(&mem_backend, mali_allocation->backend_handle); + +Failed_alloc_backend: + mali_vma_offset_remove(&session->allocation_mgr, &mali_allocation->mali_vma_node); + mali_mem_allocation_struct_destory(mali_allocation); + + MALI_DEBUG_PRINT(1, (" _mali_ukk_mem_bind, return ERROR! \n")); + return ret; +} + + +/* +* Function _mali_ukk_mem_unbind -- unbind a external memory to a new GPU address +* This function unbind the backend memory and free the allocation +* no ref_count for this type of memory +*/ +_mali_osk_errcode_t _mali_ukk_mem_unbind(_mali_uk_unbind_mem_s *args) +{ + /**/ + struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; + mali_mem_allocation *mali_allocation = NULL; + struct mali_vma_node *mali_vma_node = NULL; + u32 mali_addr = args->vaddr; + MALI_DEBUG_PRINT(5, (" _mali_ukk_mem_unbind, vaddr=0x%x! \n", args->vaddr)); + + /* find the allocation by vaddr */ + mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0); + if (likely(mali_vma_node)) { + MALI_DEBUG_ASSERT(mali_addr == mali_vma_node->vm_node.start); + mali_allocation = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); + } else { + MALI_DEBUG_ASSERT(NULL != mali_vma_node); + return _MALI_OSK_ERR_INVALID_ARGS; + } + + if (NULL != mali_allocation) + /* check ref_count */ + mali_allocation_unref(&mali_allocation); + return _MALI_OSK_ERR_OK; +} + +/* +* Function _mali_ukk_mem_cow -- COW for an allocation +* This function allocate new pages for a range (range, range+size) of allocation +* And Map it(keep use the not in range pages from target allocation ) to an GPU vaddr +*/ +_mali_osk_errcode_t _mali_ukk_mem_cow(_mali_uk_cow_mem_s *args) +{ + _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; + mali_mem_backend *target_backend = NULL; + mali_mem_backend *mem_backend = NULL; + struct mali_vma_node *mali_vma_node = NULL; + mali_mem_allocation *mali_allocation = NULL; + + struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; + /* Get the target backend for cow */ + target_backend = mali_mem_backend_struct_search(session, args->target_handle); + + if (NULL == target_backend || 0 == target_backend->size) { + MALI_DEBUG_ASSERT_POINTER(target_backend); + MALI_DEBUG_ASSERT(0 != target_backend->size); + return ret; + } + + /*Cow not support resized mem */ + MALI_DEBUG_ASSERT(MALI_MEM_FLAG_CAN_RESIZE != (MALI_MEM_FLAG_CAN_RESIZE & target_backend->mali_allocation->flags)); + + /* Check if the new mali address is allocated */ + mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, args->vaddr, 0); + + if (unlikely(mali_vma_node)) { + MALI_DEBUG_ASSERT(0); + return ret; + } + + /* create new alloction for COW*/ + mali_allocation = mali_mem_allocation_struct_create(session); + if (mali_allocation == NULL) { + MALI_DEBUG_PRINT(1, ("_mali_ukk_mem_cow: Failed to create allocation struct!\n")); + return _MALI_OSK_ERR_NOMEM; + } + mali_allocation->psize = args->target_size; + mali_allocation->vsize = args->target_size; + mali_allocation->type = MALI_MEM_COW; + + /*add allocation node to RB tree for index*/ + mali_allocation->mali_vma_node.vm_node.start = args->vaddr; + mali_allocation->mali_vma_node.vm_node.size = mali_allocation->vsize; + mali_vma_offset_add(&session->allocation_mgr, &mali_allocation->mali_vma_node); + + /* create new backend for COW memory */ + mali_allocation->backend_handle = mali_mem_backend_struct_create(&mem_backend, mali_allocation->psize); + if (mali_allocation->backend_handle < 0) { + ret = _MALI_OSK_ERR_NOMEM; + MALI_DEBUG_PRINT(1, ("mali_allocation->backend_handle < 0! \n")); + goto failed_alloc_backend; + } + mem_backend->mali_allocation = mali_allocation; + mem_backend->type = mali_allocation->type; + + if (target_backend->type == MALI_MEM_SWAP || + (MALI_MEM_COW == target_backend->type && (MALI_MEM_BACKEND_FLAG_SWAP_COWED & target_backend->flags))) { + mem_backend->flags |= MALI_MEM_BACKEND_FLAG_SWAP_COWED; + /** + * CoWed swap backends couldn't be mapped as non-linear vma, because if one + * vma is set with flag VM_NONLINEAR, the vma->vm_private_data will be used by kernel, + * while in mali driver, we use this variable to store the pointer of mali_allocation, so there + * is a conflict. + * To resolve this problem, we have to do some fake things, we reserved about 64MB + * space from index 0, there isn't really page's index will be set from 0 to (64MB>>PAGE_SHIFT_NUM), + * and all of CoWed swap memory backends' start_idx will be assigned with 0, and these + * backends will be mapped as linear and will add to priority tree of global swap file, while + * these vmas will never be found by using normal page->index, these pages in those vma + * also couldn't be swapped out. + */ + mem_backend->start_idx = 0; + } + + /* Add the target backend's cow count, also allocate new pages for COW backend from os mem + *for a modified range and keep the page which not in the modified range and Add ref to it + */ + MALI_DEBUG_PRINT(3, ("Cow mapping: target_addr: 0x%x; cow_addr: 0x%x, size: %u\n", target_backend->mali_allocation->mali_vma_node.vm_node.start, + mali_allocation->mali_vma_node.vm_node.start, mali_allocation->mali_vma_node.vm_node.size)); + + ret = mali_memory_do_cow(target_backend, args->target_offset, args->target_size, mem_backend, args->range_start, args->range_size); + if (_MALI_OSK_ERR_OK != ret) { + MALI_DEBUG_PRINT(1, ("_mali_ukk_mem_cow: Failed to cow!\n")); + goto failed_do_cow; + } + + /** + *map to GPU side + */ + mali_allocation->mali_mapping.addr = args->vaddr; + /* set gpu mmu propery */ + _mali_memory_gpu_map_property_set(&mali_allocation->mali_mapping.properties, args->flags); + + _mali_osk_mutex_wait(session->memory_lock); + /* Map on Mali */ + ret = mali_mem_mali_map_prepare(mali_allocation); + if (0 != ret) { + MALI_DEBUG_PRINT(1, (" prepare map fail! \n")); + goto failed_gpu_map; + } + + if (!(mem_backend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED)) { + mali_mem_cow_mali_map(mem_backend, 0, mem_backend->size); + } + + _mali_osk_mutex_signal(session->memory_lock); + + mutex_lock(&target_backend->mutex); + target_backend->flags |= MALI_MEM_BACKEND_FLAG_COWED; + mutex_unlock(&target_backend->mutex); + + atomic_add(args->range_size / MALI_MMU_PAGE_SIZE, &session->mali_mem_allocated_pages); + if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) { + session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE; + } + return _MALI_OSK_ERR_OK; + +failed_gpu_map: + _mali_osk_mutex_signal(session->memory_lock); + mali_mem_cow_release(mem_backend, MALI_FALSE); + mem_backend->cow_mem.count = 0; +failed_do_cow: + mali_mem_backend_struct_destory(&mem_backend, mali_allocation->backend_handle); +failed_alloc_backend: + mali_vma_offset_remove(&session->allocation_mgr, &mali_allocation->mali_vma_node); + mali_mem_allocation_struct_destory(mali_allocation); + + return ret; +} + +_mali_osk_errcode_t _mali_ukk_mem_cow_modify_range(_mali_uk_cow_modify_range_s *args) +{ + _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; + mali_mem_backend *mem_backend = NULL; + struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; + + MALI_DEBUG_PRINT(4, (" _mali_ukk_mem_cow_modify_range called! \n")); + /* Get the backend that need to be modified. */ + mem_backend = mali_mem_backend_struct_search(session, args->vaddr); + + if (NULL == mem_backend || 0 == mem_backend->size) { + MALI_DEBUG_ASSERT_POINTER(mem_backend); + MALI_DEBUG_ASSERT(0 != mem_backend->size); + return ret; + } + + MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_backend->type); + + ret = mali_memory_cow_modify_range(mem_backend, args->range_start, args->size); + args->change_pages_nr = mem_backend->cow_mem.change_pages_nr; + if (_MALI_OSK_ERR_OK != ret) + return ret; + _mali_osk_mutex_wait(session->memory_lock); + if (!(mem_backend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED)) { + mali_mem_cow_mali_map(mem_backend, args->range_start, args->size); + } + _mali_osk_mutex_signal(session->memory_lock); + + atomic_add(args->change_pages_nr, &session->mali_mem_allocated_pages); + if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) { + session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE; + } + + return _MALI_OSK_ERR_OK; +} + + +_mali_osk_errcode_t _mali_ukk_mem_resize(_mali_uk_mem_resize_s *args) +{ + mali_mem_backend *mem_backend = NULL; + _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; + + struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; + + MALI_DEBUG_ASSERT_POINTER(session); + MALI_DEBUG_PRINT(4, (" mali_mem_resize_memory called! \n")); + MALI_DEBUG_ASSERT(0 == args->psize % MALI_MMU_PAGE_SIZE); + + /* Get the memory backend that need to be resize. */ + mem_backend = mali_mem_backend_struct_search(session, args->vaddr); + + if (NULL == mem_backend) { + MALI_DEBUG_PRINT(2, ("_mali_ukk_mem_resize: memory backend = NULL!\n")); + return ret; + } + + MALI_DEBUG_ASSERT(args->psize != mem_backend->size); + + ret = mali_mem_resize(session, mem_backend, args->psize); + + return ret; +} + +_mali_osk_errcode_t _mali_ukk_mem_usage_get(_mali_uk_profiling_memory_usage_get_s *args) +{ + args->memory_usage = _mali_ukk_report_memory_usage(); + if (0 != args->vaddr) { + mali_mem_backend *mem_backend = NULL; + struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; + /* Get the backend that need to be modified. */ + mem_backend = mali_mem_backend_struct_search(session, args->vaddr); + if (NULL == mem_backend) { + MALI_DEBUG_ASSERT_POINTER(mem_backend); + return _MALI_OSK_ERR_FAULT; + } + + if (MALI_MEM_COW == mem_backend->type) + args->change_pages_nr = mem_backend->cow_mem.change_pages_nr; + } + return _MALI_OSK_ERR_OK; +} diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_manager.h b/drivers/gpu/arm/utgard/linux/mali_memory_manager.h new file mode 100644 index 00000000000000..c454b935467688 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_memory_manager.h @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_MEMORY_MANAGER_H__ +#define __MALI_MEMORY_MANAGER_H__ + +#include "mali_osk.h" +#include +#include +#include +#include +#include +#include "mali_memory_types.h" +#include "mali_memory_os_alloc.h" +#include "mali_uk_types.h" + +struct mali_allocation_manager { + rwlock_t vm_lock; + struct rb_root allocation_mgr_rb; + struct list_head head; + struct mutex list_mutex; + u32 mali_allocation_num; +}; + +extern struct idr mali_backend_idr; +extern struct mutex mali_idr_mutex; + +int mali_memory_manager_init(struct mali_allocation_manager *mgr); +void mali_memory_manager_uninit(struct mali_allocation_manager *mgr); + +void mali_mem_allocation_struct_destory(mali_mem_allocation *alloc); +_mali_osk_errcode_t mali_mem_add_mem_size(struct mali_session_data *session, u32 mali_addr, u32 add_size); +mali_mem_backend *mali_mem_backend_struct_search(struct mali_session_data *session, u32 mali_address); +_mali_osk_errcode_t _mali_ukk_mem_allocate(_mali_uk_alloc_mem_s *args); +_mali_osk_errcode_t _mali_ukk_mem_free(_mali_uk_free_mem_s *args); +_mali_osk_errcode_t _mali_ukk_mem_bind(_mali_uk_bind_mem_s *args); +_mali_osk_errcode_t _mali_ukk_mem_unbind(_mali_uk_unbind_mem_s *args); +_mali_osk_errcode_t _mali_ukk_mem_cow(_mali_uk_cow_mem_s *args); +_mali_osk_errcode_t _mali_ukk_mem_cow_modify_range(_mali_uk_cow_modify_range_s *args); +_mali_osk_errcode_t _mali_ukk_mem_usage_get(_mali_uk_profiling_memory_usage_get_s *args); +_mali_osk_errcode_t _mali_ukk_mem_resize(_mali_uk_mem_resize_s *args); + +#endif + diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_os_alloc.c b/drivers/gpu/arm/utgard/linux/mali_memory_os_alloc.c new file mode 100644 index 00000000000000..f77f18e34f2379 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_memory_os_alloc.c @@ -0,0 +1,805 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mali_osk.h" +#include "mali_memory.h" +#include "mali_memory_os_alloc.h" +#include "mali_kernel_linux.h" + +/* Minimum size of allocator page pool */ +#define MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_MB * 256) +#define MALI_OS_MEMORY_POOL_TRIM_JIFFIES (10 * CONFIG_HZ) /* Default to 10s */ + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) +/* Write combine dma_attrs */ +static unsigned long dma_attrs_wc; +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 0, 0) +#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35) +static int mali_mem_os_shrink(int nr_to_scan, gfp_t gfp_mask); +#else +static int mali_mem_os_shrink(struct shrinker *shrinker, int nr_to_scan, gfp_t gfp_mask); +#endif +#else +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) +static int mali_mem_os_shrink(struct shrinker *shrinker, struct shrink_control *sc); +#else +static unsigned long mali_mem_os_shrink(struct shrinker *shrinker, struct shrink_control *sc); +static unsigned long mali_mem_os_shrink_count(struct shrinker *shrinker, struct shrink_control *sc); +#endif +#endif +static void mali_mem_os_trim_pool(struct work_struct *work); + +struct mali_mem_os_allocator mali_mem_os_allocator = { + .pool_lock = __SPIN_LOCK_UNLOCKED(pool_lock), + .pool_pages = LIST_HEAD_INIT(mali_mem_os_allocator.pool_pages), + .pool_count = 0, + + .allocated_pages = ATOMIC_INIT(0), + .allocation_limit = 0, + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) + .shrinker.shrink = mali_mem_os_shrink, +#else + .shrinker.count_objects = mali_mem_os_shrink_count, + .shrinker.scan_objects = mali_mem_os_shrink, +#endif + .shrinker.seeks = DEFAULT_SEEKS, +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0) + .timed_shrinker = __DELAYED_WORK_INITIALIZER(mali_mem_os_allocator.timed_shrinker, mali_mem_os_trim_pool, TIMER_DEFERRABLE), +#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 38) + .timed_shrinker = __DEFERRED_WORK_INITIALIZER(mali_mem_os_allocator.timed_shrinker, mali_mem_os_trim_pool), +#else + .timed_shrinker = __DELAYED_WORK_INITIALIZER(mali_mem_os_allocator.timed_shrinker, mali_mem_os_trim_pool), +#endif +}; + +u32 mali_mem_os_free(struct list_head *os_pages, u32 pages_count, mali_bool cow_flag) +{ + LIST_HEAD(pages); + struct mali_page_node *m_page, *m_tmp; + u32 free_pages_nr = 0; + + if (MALI_TRUE == cow_flag) { + list_for_each_entry_safe(m_page, m_tmp, os_pages, list) { + /*only handle OS node here */ + if (m_page->type == MALI_PAGE_NODE_OS) { + if (1 == _mali_page_node_get_ref_count(m_page)) { + list_move(&m_page->list, &pages); + atomic_sub(1, &mali_mem_os_allocator.allocated_pages); + free_pages_nr ++; + } else { + _mali_page_node_unref(m_page); + m_page->page = NULL; + list_del(&m_page->list); + kfree(m_page); + } + } + } + } else { + list_cut_position(&pages, os_pages, os_pages->prev); + atomic_sub(pages_count, &mali_mem_os_allocator.allocated_pages); + free_pages_nr = pages_count; + } + + /* Put pages on pool. */ + spin_lock(&mali_mem_os_allocator.pool_lock); + list_splice(&pages, &mali_mem_os_allocator.pool_pages); + mali_mem_os_allocator.pool_count += free_pages_nr; + spin_unlock(&mali_mem_os_allocator.pool_lock); + + if (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES < mali_mem_os_allocator.pool_count) { + MALI_DEBUG_PRINT(5, ("OS Mem: Starting pool trim timer %u\n", mali_mem_os_allocator.pool_count)); + queue_delayed_work(mali_mem_os_allocator.wq, &mali_mem_os_allocator.timed_shrinker, MALI_OS_MEMORY_POOL_TRIM_JIFFIES); + } + return free_pages_nr; +} + +/** +* put page without put it into page pool +*/ +_mali_osk_errcode_t mali_mem_os_put_page(struct page *page) +{ + MALI_DEBUG_ASSERT_POINTER(page); + if (1 == page_count(page)) { + atomic_sub(1, &mali_mem_os_allocator.allocated_pages); + dma_unmap_page(&mali_platform_device->dev, page_private(page), + _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE); + ClearPagePrivate(page); + } + put_page(page); + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t mali_mem_os_resize_pages(mali_mem_os_mem *mem_from, mali_mem_os_mem *mem_to, u32 start_page, u32 page_count) +{ + struct mali_page_node *m_page, *m_tmp; + u32 i = 0; + + MALI_DEBUG_ASSERT_POINTER(mem_from); + MALI_DEBUG_ASSERT_POINTER(mem_to); + + if (mem_from->count < start_page + page_count) { + return _MALI_OSK_ERR_INVALID_ARGS; + } + + list_for_each_entry_safe(m_page, m_tmp, &mem_from->pages, list) { + if (i >= start_page && i < start_page + page_count) { + list_move_tail(&m_page->list, &mem_to->pages); + mem_from->count--; + mem_to->count++; + } + i++; + } + + return _MALI_OSK_ERR_OK; +} + + +int mali_mem_os_alloc_pages(mali_mem_os_mem *os_mem, u32 size) +{ + struct page *new_page; + LIST_HEAD(pages_list); + size_t page_count = PAGE_ALIGN(size) / _MALI_OSK_MALI_PAGE_SIZE; + size_t remaining = page_count; + struct mali_page_node *m_page, *m_tmp; + u32 i; + + MALI_DEBUG_ASSERT_POINTER(os_mem); + + if (atomic_read(&mali_mem_os_allocator.allocated_pages) * _MALI_OSK_MALI_PAGE_SIZE + size > mali_mem_os_allocator.allocation_limit) { + MALI_DEBUG_PRINT(2, ("Mali Mem: Unable to allocate %u bytes. Currently allocated: %lu, max limit %lu\n", + size, + atomic_read(&mali_mem_os_allocator.allocated_pages) * _MALI_OSK_MALI_PAGE_SIZE, + mali_mem_os_allocator.allocation_limit)); + return -ENOMEM; + } + + INIT_LIST_HEAD(&os_mem->pages); + os_mem->count = page_count; + + /* Grab pages from pool. */ + { + size_t pool_pages; + spin_lock(&mali_mem_os_allocator.pool_lock); + pool_pages = min(remaining, mali_mem_os_allocator.pool_count); + for (i = pool_pages; i > 0; i--) { + BUG_ON(list_empty(&mali_mem_os_allocator.pool_pages)); + list_move(mali_mem_os_allocator.pool_pages.next, &pages_list); + } + mali_mem_os_allocator.pool_count -= pool_pages; + remaining -= pool_pages; + spin_unlock(&mali_mem_os_allocator.pool_lock); + } + + /* Process pages from pool. */ + i = 0; + list_for_each_entry_safe(m_page, m_tmp, &pages_list, list) { + BUG_ON(NULL == m_page); + + list_move_tail(&m_page->list, &os_mem->pages); + } + + /* Allocate new pages, if needed. */ + for (i = 0; i < remaining; i++) { + dma_addr_t dma_addr; + gfp_t flags = __GFP_ZERO | __GFP_NORETRY | __GFP_NOWARN | __GFP_COLD; + int err; + +#if defined(CONFIG_ARM) && !defined(CONFIG_ARM_LPAE) + flags |= GFP_HIGHUSER; +#else +#ifdef CONFIG_ZONE_DMA32 + flags |= GFP_DMA32; +#else +#ifdef CONFIG_ZONE_DMA + flags |= GFP_DMA; +#else + /* arm64 utgard only work on < 4G, but the kernel + * didn't provide method to allocte memory < 4G + */ + MALI_DEBUG_ASSERT(0); +#endif +#endif +#endif + + new_page = alloc_page(flags); + + if (unlikely(NULL == new_page)) { + /* Calculate the number of pages actually allocated, and free them. */ + os_mem->count = (page_count - remaining) + i; + atomic_add(os_mem->count, &mali_mem_os_allocator.allocated_pages); + mali_mem_os_free(&os_mem->pages, os_mem->count, MALI_FALSE); + return -ENOMEM; + } + + /* Ensure page is flushed from CPU caches. */ + dma_addr = dma_map_page(&mali_platform_device->dev, new_page, + 0, _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE); + + err = dma_mapping_error(&mali_platform_device->dev, dma_addr); + if (unlikely(err)) { + MALI_DEBUG_PRINT_ERROR(("OS Mem: Failed to DMA map page %p: %u", + new_page, err)); + __free_page(new_page); + os_mem->count = (page_count - remaining) + i; + atomic_add(os_mem->count, &mali_mem_os_allocator.allocated_pages); + mali_mem_os_free(&os_mem->pages, os_mem->count, MALI_FALSE); + return -EFAULT; + } + + /* Store page phys addr */ + SetPagePrivate(new_page); + set_page_private(new_page, dma_addr); + + m_page = _mali_page_node_allocate(MALI_PAGE_NODE_OS); + if (unlikely(NULL == m_page)) { + MALI_PRINT_ERROR(("OS Mem: Can't allocate mali_page node! \n")); + dma_unmap_page(&mali_platform_device->dev, page_private(new_page), + _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE); + ClearPagePrivate(new_page); + __free_page(new_page); + os_mem->count = (page_count - remaining) + i; + atomic_add(os_mem->count, &mali_mem_os_allocator.allocated_pages); + mali_mem_os_free(&os_mem->pages, os_mem->count, MALI_FALSE); + return -EFAULT; + } + m_page->page = new_page; + + list_add_tail(&m_page->list, &os_mem->pages); + } + + atomic_add(page_count, &mali_mem_os_allocator.allocated_pages); + + if (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES > mali_mem_os_allocator.pool_count) { + MALI_DEBUG_PRINT(4, ("OS Mem: Stopping pool trim timer, only %u pages on pool\n", mali_mem_os_allocator.pool_count)); + cancel_delayed_work(&mali_mem_os_allocator.timed_shrinker); + } + + return 0; +} + + +_mali_osk_errcode_t mali_mem_os_mali_map(mali_mem_os_mem *os_mem, struct mali_session_data *session, u32 vaddr, u32 start_page, u32 mapping_pgae_num, u32 props) +{ + struct mali_page_directory *pagedir = session->page_directory; + struct mali_page_node *m_page; + u32 virt; + u32 prop = props; + + MALI_DEBUG_ASSERT_POINTER(session); + MALI_DEBUG_ASSERT_POINTER(os_mem); + + MALI_DEBUG_ASSERT(start_page <= os_mem->count); + MALI_DEBUG_ASSERT((start_page + mapping_pgae_num) <= os_mem->count); + + if ((start_page + mapping_pgae_num) == os_mem->count) { + + virt = vaddr + MALI_MMU_PAGE_SIZE * (start_page + mapping_pgae_num); + + list_for_each_entry_reverse(m_page, &os_mem->pages, list) { + + virt -= MALI_MMU_PAGE_SIZE; + if (mapping_pgae_num > 0) { + dma_addr_t phys = page_private(m_page->page); +#if defined(CONFIG_ARCH_DMA_ADDR_T_64BIT) + /* Verify that the "physical" address is 32-bit and + * usable for Mali, when on a system with bus addresses + * wider than 32-bit. */ + MALI_DEBUG_ASSERT(0 == (phys >> 32)); +#endif + mali_mmu_pagedir_update(pagedir, virt, (mali_dma_addr)phys, MALI_MMU_PAGE_SIZE, prop); + } else { + break; + } + mapping_pgae_num--; + } + + } else { + u32 i = 0; + virt = vaddr; + list_for_each_entry(m_page, &os_mem->pages, list) { + + if (i >= start_page) { + dma_addr_t phys = page_private(m_page->page); + +#if defined(CONFIG_ARCH_DMA_ADDR_T_64BIT) + /* Verify that the "physical" address is 32-bit and + * usable for Mali, when on a system with bus addresses + * wider than 32-bit. */ + MALI_DEBUG_ASSERT(0 == (phys >> 32)); +#endif + mali_mmu_pagedir_update(pagedir, virt, (mali_dma_addr)phys, MALI_MMU_PAGE_SIZE, prop); + } + i++; + virt += MALI_MMU_PAGE_SIZE; + } + } + return _MALI_OSK_ERR_OK; +} + + +void mali_mem_os_mali_unmap(mali_mem_allocation *alloc) +{ + struct mali_session_data *session; + MALI_DEBUG_ASSERT_POINTER(alloc); + session = alloc->session; + MALI_DEBUG_ASSERT_POINTER(session); + + mali_session_memory_lock(session); + mali_mem_mali_map_free(session, alloc->psize, alloc->mali_vma_node.vm_node.start, + alloc->flags); + mali_session_memory_unlock(session); +} + +int mali_mem_os_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma) +{ + mali_mem_os_mem *os_mem = &mem_bkend->os_mem; + struct mali_page_node *m_page; + struct page *page; + int ret; + unsigned long addr = vma->vm_start; + MALI_DEBUG_ASSERT(MALI_MEM_OS == mem_bkend->type); + + list_for_each_entry(m_page, &os_mem->pages, list) { + /* We should use vm_insert_page, but it does a dcache + * flush which makes it way slower than remap_pfn_range or vm_insert_pfn. + ret = vm_insert_page(vma, addr, page); + */ + page = m_page->page; + ret = vm_insert_pfn(vma, addr, page_to_pfn(page)); + + if (unlikely(0 != ret)) { + return -EFAULT; + } + addr += _MALI_OSK_MALI_PAGE_SIZE; + } + + return 0; +} + +_mali_osk_errcode_t mali_mem_os_resize_cpu_map_locked(mali_mem_backend *mem_bkend, struct vm_area_struct *vma, unsigned long start_vaddr, u32 mappig_size) +{ + mali_mem_os_mem *os_mem = &mem_bkend->os_mem; + struct mali_page_node *m_page; + int ret; + int offset; + int mapping_page_num; + int count ; + + unsigned long vstart = vma->vm_start; + count = 0; + MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_OS); + MALI_DEBUG_ASSERT(0 == start_vaddr % _MALI_OSK_MALI_PAGE_SIZE); + MALI_DEBUG_ASSERT(0 == vstart % _MALI_OSK_MALI_PAGE_SIZE); + offset = (start_vaddr - vstart) / _MALI_OSK_MALI_PAGE_SIZE; + MALI_DEBUG_ASSERT(offset <= os_mem->count); + mapping_page_num = mappig_size / _MALI_OSK_MALI_PAGE_SIZE; + MALI_DEBUG_ASSERT((offset + mapping_page_num) <= os_mem->count); + + if ((offset + mapping_page_num) == os_mem->count) { + + unsigned long vm_end = start_vaddr + mappig_size; + + list_for_each_entry_reverse(m_page, &os_mem->pages, list) { + + vm_end -= _MALI_OSK_MALI_PAGE_SIZE; + if (mapping_page_num > 0) { + ret = vm_insert_pfn(vma, vm_end, page_to_pfn(m_page->page)); + + if (unlikely(0 != ret)) { + /*will return -EBUSY If the page has already been mapped into table, but it's OK*/ + if (-EBUSY == ret) { + break; + } else { + MALI_DEBUG_PRINT(1, ("OS Mem: mali_mem_os_resize_cpu_map_locked failed, ret = %d, offset is %d,page_count is %d\n", + ret, offset + mapping_page_num, os_mem->count)); + } + return _MALI_OSK_ERR_FAULT; + } + } else { + break; + } + mapping_page_num--; + + } + } else { + + list_for_each_entry(m_page, &os_mem->pages, list) { + if (count >= offset) { + + ret = vm_insert_pfn(vma, vstart, page_to_pfn(m_page->page)); + + if (unlikely(0 != ret)) { + /*will return -EBUSY If the page has already been mapped into table, but it's OK*/ + if (-EBUSY == ret) { + break; + } else { + MALI_DEBUG_PRINT(1, ("OS Mem: mali_mem_os_resize_cpu_map_locked failed, ret = %d, count is %d, offset is %d,page_count is %d\n", + ret, count, offset, os_mem->count)); + } + return _MALI_OSK_ERR_FAULT; + } + } + count++; + vstart += _MALI_OSK_MALI_PAGE_SIZE; + } + } + return _MALI_OSK_ERR_OK; +} + +u32 mali_mem_os_release(mali_mem_backend *mem_bkend) +{ + + mali_mem_allocation *alloc; + u32 free_pages_nr = 0; + MALI_DEBUG_ASSERT_POINTER(mem_bkend); + MALI_DEBUG_ASSERT(MALI_MEM_OS == mem_bkend->type); + + alloc = mem_bkend->mali_allocation; + MALI_DEBUG_ASSERT_POINTER(alloc); + + /* Unmap the memory from the mali virtual address space. */ + mali_mem_os_mali_unmap(alloc); + mutex_lock(&mem_bkend->mutex); + /* Free pages */ + if (MALI_MEM_BACKEND_FLAG_COWED & mem_bkend->flags) { + free_pages_nr = mali_mem_os_free(&mem_bkend->os_mem.pages, mem_bkend->os_mem.count, MALI_TRUE); + } else { + free_pages_nr = mali_mem_os_free(&mem_bkend->os_mem.pages, mem_bkend->os_mem.count, MALI_FALSE); + } + mutex_unlock(&mem_bkend->mutex); + + MALI_DEBUG_PRINT(4, ("OS Mem free : allocated size = 0x%x, free size = 0x%x\n", mem_bkend->os_mem.count * _MALI_OSK_MALI_PAGE_SIZE, + free_pages_nr * _MALI_OSK_MALI_PAGE_SIZE)); + + mem_bkend->os_mem.count = 0; + return free_pages_nr; +} + + +#define MALI_MEM_OS_PAGE_TABLE_PAGE_POOL_SIZE 128 +static struct { + struct { + mali_dma_addr phys; + mali_io_address mapping; + } page[MALI_MEM_OS_PAGE_TABLE_PAGE_POOL_SIZE]; + size_t count; + spinlock_t lock; +} mali_mem_page_table_page_pool = { + .count = 0, + .lock = __SPIN_LOCK_UNLOCKED(pool_lock), +}; + +_mali_osk_errcode_t mali_mem_os_get_table_page(mali_dma_addr *phys, mali_io_address *mapping) +{ + _mali_osk_errcode_t ret = _MALI_OSK_ERR_NOMEM; + dma_addr_t tmp_phys; + + spin_lock(&mali_mem_page_table_page_pool.lock); + if (0 < mali_mem_page_table_page_pool.count) { + u32 i = --mali_mem_page_table_page_pool.count; + *phys = mali_mem_page_table_page_pool.page[i].phys; + *mapping = mali_mem_page_table_page_pool.page[i].mapping; + + ret = _MALI_OSK_ERR_OK; + } + spin_unlock(&mali_mem_page_table_page_pool.lock); + + if (_MALI_OSK_ERR_OK != ret) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) + *mapping = dma_alloc_attrs(&mali_platform_device->dev, + _MALI_OSK_MALI_PAGE_SIZE, &tmp_phys, + GFP_KERNEL, dma_attrs_wc); +#else + *mapping = dma_alloc_writecombine(&mali_platform_device->dev, + _MALI_OSK_MALI_PAGE_SIZE, &tmp_phys, GFP_KERNEL); +#endif + if (NULL != *mapping) { + ret = _MALI_OSK_ERR_OK; + +#if defined(CONFIG_ARCH_DMA_ADDR_T_64BIT) + /* Verify that the "physical" address is 32-bit and + * usable for Mali, when on a system with bus addresses + * wider than 32-bit. */ + MALI_DEBUG_ASSERT(0 == (tmp_phys >> 32)); +#endif + + *phys = (mali_dma_addr)tmp_phys; + } + } + + return ret; +} + +void mali_mem_os_release_table_page(mali_dma_addr phys, void *virt) +{ + spin_lock(&mali_mem_page_table_page_pool.lock); + if (MALI_MEM_OS_PAGE_TABLE_PAGE_POOL_SIZE > mali_mem_page_table_page_pool.count) { + u32 i = mali_mem_page_table_page_pool.count; + mali_mem_page_table_page_pool.page[i].phys = phys; + mali_mem_page_table_page_pool.page[i].mapping = virt; + + ++mali_mem_page_table_page_pool.count; + + spin_unlock(&mali_mem_page_table_page_pool.lock); + } else { + spin_unlock(&mali_mem_page_table_page_pool.lock); + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) + dma_free_attrs(&mali_platform_device->dev, + _MALI_OSK_MALI_PAGE_SIZE, virt, phys, + dma_attrs_wc); +#else + dma_free_writecombine(&mali_platform_device->dev, + _MALI_OSK_MALI_PAGE_SIZE, virt, phys); +#endif + } +} + +void mali_mem_os_free_page_node(struct mali_page_node *m_page) +{ + struct page *page = m_page->page; + MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_OS); + + if (1 == page_count(page)) { + dma_unmap_page(&mali_platform_device->dev, page_private(page), + _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE); + ClearPagePrivate(page); + } + __free_page(page); + m_page->page = NULL; + list_del(&m_page->list); + kfree(m_page); +} + +/* The maximum number of page table pool pages to free in one go. */ +#define MALI_MEM_OS_CHUNK_TO_FREE 64UL + +/* Free a certain number of pages from the page table page pool. + * The pool lock must be held when calling the function, and the lock will be + * released before returning. + */ +static void mali_mem_os_page_table_pool_free(size_t nr_to_free) +{ + mali_dma_addr phys_arr[MALI_MEM_OS_CHUNK_TO_FREE]; + void *virt_arr[MALI_MEM_OS_CHUNK_TO_FREE]; + u32 i; + + MALI_DEBUG_ASSERT(nr_to_free <= MALI_MEM_OS_CHUNK_TO_FREE); + + /* Remove nr_to_free pages from the pool and store them locally on stack. */ + for (i = 0; i < nr_to_free; i++) { + u32 pool_index = mali_mem_page_table_page_pool.count - i - 1; + + phys_arr[i] = mali_mem_page_table_page_pool.page[pool_index].phys; + virt_arr[i] = mali_mem_page_table_page_pool.page[pool_index].mapping; + } + + mali_mem_page_table_page_pool.count -= nr_to_free; + + spin_unlock(&mali_mem_page_table_page_pool.lock); + + /* After releasing the spinlock: free the pages we removed from the pool. */ + for (i = 0; i < nr_to_free; i++) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) + dma_free_attrs(&mali_platform_device->dev, _MALI_OSK_MALI_PAGE_SIZE, + virt_arr[i], (dma_addr_t)phys_arr[i], dma_attrs_wc); +#else + dma_free_writecombine(&mali_platform_device->dev, + _MALI_OSK_MALI_PAGE_SIZE, + virt_arr[i], (dma_addr_t)phys_arr[i]); +#endif + } +} + +static void mali_mem_os_trim_page_table_page_pool(void) +{ + size_t nr_to_free = 0; + size_t nr_to_keep; + + /* Keep 2 page table pages for each 1024 pages in the page cache. */ + nr_to_keep = mali_mem_os_allocator.pool_count / 512; + /* And a minimum of eight pages, to accomodate new sessions. */ + nr_to_keep += 8; + + if (0 == spin_trylock(&mali_mem_page_table_page_pool.lock)) return; + + if (nr_to_keep < mali_mem_page_table_page_pool.count) { + nr_to_free = mali_mem_page_table_page_pool.count - nr_to_keep; + nr_to_free = min((size_t)MALI_MEM_OS_CHUNK_TO_FREE, nr_to_free); + } + + /* Pool lock will be released by the callee. */ + mali_mem_os_page_table_pool_free(nr_to_free); +} + +static unsigned long mali_mem_os_shrink_count(struct shrinker *shrinker, struct shrink_control *sc) +{ + return mali_mem_os_allocator.pool_count; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 0, 0) +#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35) +static int mali_mem_os_shrink(int nr_to_scan, gfp_t gfp_mask) +#else +static int mali_mem_os_shrink(struct shrinker *shrinker, int nr_to_scan, gfp_t gfp_mask) +#endif /* Linux < 2.6.35 */ +#else +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) +static int mali_mem_os_shrink(struct shrinker *shrinker, struct shrink_control *sc) +#else +static unsigned long mali_mem_os_shrink(struct shrinker *shrinker, struct shrink_control *sc) +#endif /* Linux < 3.12.0 */ +#endif /* Linux < 3.0.0 */ +{ + struct mali_page_node *m_page, *m_tmp; + unsigned long flags; + struct list_head *le, pages; +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 0, 0) + int nr = nr_to_scan; +#else + int nr = sc->nr_to_scan; +#endif + + if (0 == nr) { + return mali_mem_os_shrink_count(shrinker, sc); + } + + if (0 == spin_trylock_irqsave(&mali_mem_os_allocator.pool_lock, flags)) { + /* Not able to lock. */ + return -1; + } + + if (0 == mali_mem_os_allocator.pool_count) { + /* No pages availble */ + spin_unlock_irqrestore(&mali_mem_os_allocator.pool_lock, flags); + return 0; + } + + /* Release from general page pool */ + nr = min((size_t)nr, mali_mem_os_allocator.pool_count); + mali_mem_os_allocator.pool_count -= nr; + list_for_each(le, &mali_mem_os_allocator.pool_pages) { + --nr; + if (0 == nr) break; + } + list_cut_position(&pages, &mali_mem_os_allocator.pool_pages, le); + spin_unlock_irqrestore(&mali_mem_os_allocator.pool_lock, flags); + + list_for_each_entry_safe(m_page, m_tmp, &pages, list) { + mali_mem_os_free_page_node(m_page); + } + + if (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES > mali_mem_os_allocator.pool_count) { + /* Pools are empty, stop timer */ + MALI_DEBUG_PRINT(5, ("Stopping timer, only %u pages on pool\n", mali_mem_os_allocator.pool_count)); + cancel_delayed_work(&mali_mem_os_allocator.timed_shrinker); + } + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) + return mali_mem_os_shrink_count(shrinker, sc); +#else + return nr; +#endif +} + +static void mali_mem_os_trim_pool(struct work_struct *data) +{ + struct mali_page_node *m_page, *m_tmp; + struct list_head *le; + LIST_HEAD(pages); + size_t nr_to_free; + + MALI_IGNORE(data); + + MALI_DEBUG_PRINT(3, ("OS Mem: Trimming pool %u\n", mali_mem_os_allocator.pool_count)); + + /* Release from general page pool */ + spin_lock(&mali_mem_os_allocator.pool_lock); + if (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES < mali_mem_os_allocator.pool_count) { + size_t count = mali_mem_os_allocator.pool_count - MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES; + const size_t min_to_free = min(64, MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES); + + /* Free half the pages on the pool above the static limit. Or 64 pages, 256KB. */ + nr_to_free = max(count / 2, min_to_free); + + mali_mem_os_allocator.pool_count -= nr_to_free; + list_for_each(le, &mali_mem_os_allocator.pool_pages) { + --nr_to_free; + if (0 == nr_to_free) break; + } + list_cut_position(&pages, &mali_mem_os_allocator.pool_pages, le); + } + spin_unlock(&mali_mem_os_allocator.pool_lock); + + list_for_each_entry_safe(m_page, m_tmp, &pages, list) { + mali_mem_os_free_page_node(m_page); + } + + /* Release some pages from page table page pool */ + mali_mem_os_trim_page_table_page_pool(); + + if (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES < mali_mem_os_allocator.pool_count) { + MALI_DEBUG_PRINT(4, ("OS Mem: Starting pool trim timer %u\n", mali_mem_os_allocator.pool_count)); + queue_delayed_work(mali_mem_os_allocator.wq, &mali_mem_os_allocator.timed_shrinker, MALI_OS_MEMORY_POOL_TRIM_JIFFIES); + } +} + +_mali_osk_errcode_t mali_mem_os_init(void) +{ + mali_mem_os_allocator.wq = alloc_workqueue("mali-mem", WQ_UNBOUND, 1); + if (NULL == mali_mem_os_allocator.wq) { + return _MALI_OSK_ERR_NOMEM; + } + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) + dma_attrs_wc |= DMA_ATTR_WRITE_COMBINE; +#endif + + register_shrinker(&mali_mem_os_allocator.shrinker); + + return _MALI_OSK_ERR_OK; +} + +void mali_mem_os_term(void) +{ + struct mali_page_node *m_page, *m_tmp; + unregister_shrinker(&mali_mem_os_allocator.shrinker); + cancel_delayed_work_sync(&mali_mem_os_allocator.timed_shrinker); + + if (NULL != mali_mem_os_allocator.wq) { + destroy_workqueue(mali_mem_os_allocator.wq); + mali_mem_os_allocator.wq = NULL; + } + + spin_lock(&mali_mem_os_allocator.pool_lock); + list_for_each_entry_safe(m_page, m_tmp, &mali_mem_os_allocator.pool_pages, list) { + mali_mem_os_free_page_node(m_page); + + --mali_mem_os_allocator.pool_count; + } + BUG_ON(mali_mem_os_allocator.pool_count); + spin_unlock(&mali_mem_os_allocator.pool_lock); + + /* Release from page table page pool */ + do { + u32 nr_to_free; + + spin_lock(&mali_mem_page_table_page_pool.lock); + + nr_to_free = min((size_t)MALI_MEM_OS_CHUNK_TO_FREE, mali_mem_page_table_page_pool.count); + + /* Pool lock will be released by the callee. */ + mali_mem_os_page_table_pool_free(nr_to_free); + } while (0 != mali_mem_page_table_page_pool.count); +} + +_mali_osk_errcode_t mali_memory_core_resource_os_memory(u32 size) +{ + mali_mem_os_allocator.allocation_limit = size; + + MALI_SUCCESS; +} + +u32 mali_mem_os_stat(void) +{ + return atomic_read(&mali_mem_os_allocator.allocated_pages) * _MALI_OSK_MALI_PAGE_SIZE; +} diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_os_alloc.h b/drivers/gpu/arm/utgard/linux/mali_memory_os_alloc.h new file mode 100644 index 00000000000000..f9ead166c45572 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_memory_os_alloc.h @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_MEMORY_OS_ALLOC_H__ +#define __MALI_MEMORY_OS_ALLOC_H__ + +#include "mali_osk.h" +#include "mali_memory_types.h" + + +/** @brief Release Mali OS memory + * + * The session memory_lock must be held when calling this function. + * + * @param mem_bkend Pointer to the mali_mem_backend to release + */ +u32 mali_mem_os_release(mali_mem_backend *mem_bkend); + +_mali_osk_errcode_t mali_mem_os_get_table_page(mali_dma_addr *phys, mali_io_address *mapping); + +void mali_mem_os_release_table_page(mali_dma_addr phys, void *virt); + +_mali_osk_errcode_t mali_mem_os_init(void); + +void mali_mem_os_term(void); + +u32 mali_mem_os_stat(void); + +void mali_mem_os_free_page_node(struct mali_page_node *m_page); + +int mali_mem_os_alloc_pages(mali_mem_os_mem *os_mem, u32 size); + +u32 mali_mem_os_free(struct list_head *os_pages, u32 pages_count, mali_bool cow_flag); + +_mali_osk_errcode_t mali_mem_os_put_page(struct page *page); + +_mali_osk_errcode_t mali_mem_os_resize_pages(mali_mem_os_mem *mem_from, mali_mem_os_mem *mem_to, u32 start_page, u32 page_count); + +_mali_osk_errcode_t mali_mem_os_mali_map(mali_mem_os_mem *os_mem, struct mali_session_data *session, u32 vaddr, u32 start_page, u32 mapping_pgae_num, u32 props); + +void mali_mem_os_mali_unmap(mali_mem_allocation *alloc); + +int mali_mem_os_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma); + +_mali_osk_errcode_t mali_mem_os_resize_cpu_map_locked(mali_mem_backend *mem_bkend, struct vm_area_struct *vma, unsigned long start_vaddr, u32 mappig_size); + +#endif /* __MALI_MEMORY_OS_ALLOC_H__ */ diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_swap_alloc.c b/drivers/gpu/arm/utgard/linux/mali_memory_swap_alloc.c new file mode 100644 index 00000000000000..1769000f0bb9a0 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_memory_swap_alloc.c @@ -0,0 +1,942 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "mali_osk.h" +#include "mali_osk_mali.h" +#include "mali_memory.h" +#include "mali_memory_manager.h" +#include "mali_memory_virtual.h" +#include "mali_memory_cow.h" +#include "mali_ukk.h" +#include "mali_kernel_utilization.h" +#include "mali_memory_swap_alloc.h" + + +static struct _mali_osk_bitmap idx_mgr; +static struct file *global_swap_file; +static struct address_space *global_swap_space; +static _mali_osk_wq_work_t *mali_mem_swap_out_workq = NULL; +static u32 mem_backend_swapped_pool_size; +#ifdef MALI_MEM_SWAP_TRACKING +static u32 mem_backend_swapped_unlock_size; +#endif +/* Lock order: mem_backend_swapped_pool_lock > each memory backend's mutex lock. + * This lock used to protect mem_backend_swapped_pool_size and mem_backend_swapped_pool. */ +static struct mutex mem_backend_swapped_pool_lock; +static struct list_head mem_backend_swapped_pool; + +extern struct mali_mem_os_allocator mali_mem_os_allocator; + +#define MALI_SWAP_LOW_MEM_DEFAULT_VALUE (60*1024*1024) +#define MALI_SWAP_INVALIDATE_MALI_ADDRESS (0) /* Used to mark the given memory cookie is invalidate. */ +#define MALI_SWAP_GLOBAL_SWAP_FILE_SIZE (0xFFFFFFFF) +#define MALI_SWAP_GLOBAL_SWAP_FILE_INDEX ((MALI_SWAP_GLOBAL_SWAP_FILE_SIZE) >> PAGE_SHIFT) +#define MALI_SWAP_GLOBAL_SWAP_FILE_INDEX_RESERVE (1 << 15) /* Reserved for CoW nonlinear swap backend memory, the space size is 128MB. */ + +unsigned int mali_mem_swap_out_threshold_value = MALI_SWAP_LOW_MEM_DEFAULT_VALUE; + +/** + * We have two situations to do shrinking things, one is we met low GPU utilization which shows GPU needn't touch too + * swappable backends in short time, and the other one is we add new swappable backends, the total pool size exceed + * the threshold value of the swapped pool size. + */ +typedef enum { + MALI_MEM_SWAP_SHRINK_WITH_LOW_UTILIZATION = 100, + MALI_MEM_SWAP_SHRINK_FOR_ADDING_NEW_BACKENDS = 257, +} _mali_mem_swap_pool_shrink_type_t; + +static void mali_mem_swap_swapped_bkend_pool_check_for_low_utilization(void *arg); + +_mali_osk_errcode_t mali_mem_swap_init(void) +{ + gfp_t flags = __GFP_NORETRY | __GFP_NOWARN; + + if (_MALI_OSK_ERR_OK != _mali_osk_bitmap_init(&idx_mgr, MALI_SWAP_GLOBAL_SWAP_FILE_INDEX, MALI_SWAP_GLOBAL_SWAP_FILE_INDEX_RESERVE)) { + return _MALI_OSK_ERR_NOMEM; + } + + global_swap_file = shmem_file_setup("mali_swap", MALI_SWAP_GLOBAL_SWAP_FILE_SIZE, VM_NORESERVE); + if (IS_ERR(global_swap_file)) { + _mali_osk_bitmap_term(&idx_mgr); + return _MALI_OSK_ERR_NOMEM; + } + + global_swap_space = global_swap_file->f_path.dentry->d_inode->i_mapping; + + mali_mem_swap_out_workq = _mali_osk_wq_create_work(mali_mem_swap_swapped_bkend_pool_check_for_low_utilization, NULL); + if (NULL == mali_mem_swap_out_workq) { + _mali_osk_bitmap_term(&idx_mgr); + fput(global_swap_file); + return _MALI_OSK_ERR_NOMEM; + } + +#if defined(CONFIG_ARM) && !defined(CONFIG_ARM_LPAE) + flags |= GFP_HIGHUSER; +#else +#ifdef CONFIG_ZONE_DMA32 + flags |= GFP_DMA32; +#else +#ifdef CONFIG_ZONE_DMA + flags |= GFP_DMA; +#else + /* arm64 utgard only work on < 4G, but the kernel + * didn't provide method to allocte memory < 4G + */ + MALI_DEBUG_ASSERT(0); +#endif +#endif +#endif + + /* When we use shmem_read_mapping_page to allocate/swap-in, it will + * use these flags to allocate new page if need.*/ + mapping_set_gfp_mask(global_swap_space, flags); + + mem_backend_swapped_pool_size = 0; +#ifdef MALI_MEM_SWAP_TRACKING + mem_backend_swapped_unlock_size = 0; +#endif + mutex_init(&mem_backend_swapped_pool_lock); + INIT_LIST_HEAD(&mem_backend_swapped_pool); + + MALI_DEBUG_PRINT(2, ("Mali SWAP: Swap out threshold vaule is %uM\n", mali_mem_swap_out_threshold_value >> 20)); + + return _MALI_OSK_ERR_OK; +} + +void mali_mem_swap_term(void) +{ + _mali_osk_bitmap_term(&idx_mgr); + + fput(global_swap_file); + + _mali_osk_wq_delete_work(mali_mem_swap_out_workq); + + MALI_DEBUG_ASSERT(list_empty(&mem_backend_swapped_pool)); + MALI_DEBUG_ASSERT(0 == mem_backend_swapped_pool_size); + + return; +} + +struct file *mali_mem_swap_get_global_swap_file(void) +{ + return global_swap_file; +} + +/* Judge if swappable backend in swapped pool. */ +static mali_bool mali_memory_swap_backend_in_swapped_pool(mali_mem_backend *mem_bkend) +{ + MALI_DEBUG_ASSERT_POINTER(mem_bkend); + + return !list_empty(&mem_bkend->list); +} + +void mali_memory_swap_list_backend_delete(mali_mem_backend *mem_bkend) +{ + MALI_DEBUG_ASSERT_POINTER(mem_bkend); + + mutex_lock(&mem_backend_swapped_pool_lock); + mutex_lock(&mem_bkend->mutex); + + if (MALI_FALSE == mali_memory_swap_backend_in_swapped_pool(mem_bkend)) { + mutex_unlock(&mem_bkend->mutex); + mutex_unlock(&mem_backend_swapped_pool_lock); + return; + } + + MALI_DEBUG_ASSERT(!list_empty(&mem_bkend->list)); + + list_del_init(&mem_bkend->list); + + mutex_unlock(&mem_bkend->mutex); + + mem_backend_swapped_pool_size -= mem_bkend->size; + + mutex_unlock(&mem_backend_swapped_pool_lock); +} + +static void mali_mem_swap_out_page_node(mali_page_node *page_node) +{ + MALI_DEBUG_ASSERT(page_node); + + dma_unmap_page(&mali_platform_device->dev, page_node->swap_it->dma_addr, + _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE); + set_page_dirty(page_node->swap_it->page); + put_page(page_node->swap_it->page); +} + +void mali_mem_swap_unlock_single_mem_backend(mali_mem_backend *mem_bkend) +{ + mali_page_node *m_page; + + MALI_DEBUG_ASSERT(1 == mutex_is_locked(&mem_bkend->mutex)); + + if (MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN == (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN)) { + return; + } + + mem_bkend->flags |= MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN; + + list_for_each_entry(m_page, &mem_bkend->swap_mem.pages, list) { + mali_mem_swap_out_page_node(m_page); + } + + return; +} + +static void mali_mem_swap_unlock_partial_locked_mem_backend(mali_mem_backend *mem_bkend, mali_page_node *page_node) +{ + mali_page_node *m_page; + + MALI_DEBUG_ASSERT(1 == mutex_is_locked(&mem_bkend->mutex)); + + list_for_each_entry(m_page, &mem_bkend->swap_mem.pages, list) { + if (m_page == page_node) { + break; + } + mali_mem_swap_out_page_node(m_page); + } +} + +static void mali_mem_swap_swapped_bkend_pool_shrink(_mali_mem_swap_pool_shrink_type_t shrink_type) +{ + mali_mem_backend *bkend, *tmp_bkend; + long system_free_size; + u32 last_gpu_utilization, gpu_utilization_threshold_value, temp_swap_out_threshold_value; + + MALI_DEBUG_ASSERT(1 == mutex_is_locked(&mem_backend_swapped_pool_lock)); + + if (MALI_MEM_SWAP_SHRINK_WITH_LOW_UTILIZATION == shrink_type) { + /** + * When we met that system memory is very low and Mali locked swappable memory size is less than + * threshold value, and at the same time, GPU load is very low and don't need high performance, + * at this condition, we can unlock more swap memory backend from swapped backends pool. + */ + gpu_utilization_threshold_value = MALI_MEM_SWAP_SHRINK_WITH_LOW_UTILIZATION; + temp_swap_out_threshold_value = (mali_mem_swap_out_threshold_value >> 2); + } else { + /* When we add swappable memory backends to swapped pool, we need to think that we couldn't + * hold too much swappable backends in Mali driver, and also we need considering performance. + * So there is a balance for swapping out memory backend, we should follow the following conditions: + * 1. Total memory size in global mem backend swapped pool is more than the defined threshold value. + * 2. System level free memory size is less than the defined threshold value. + * 3. Please note that GPU utilization problem isn't considered in this condition. + */ + gpu_utilization_threshold_value = MALI_MEM_SWAP_SHRINK_FOR_ADDING_NEW_BACKENDS; + temp_swap_out_threshold_value = mali_mem_swap_out_threshold_value; + } + + /* Get system free pages number. */ + system_free_size = global_page_state(NR_FREE_PAGES) * PAGE_SIZE; + last_gpu_utilization = _mali_ukk_utilization_gp_pp(); + + if ((last_gpu_utilization < gpu_utilization_threshold_value) + && (system_free_size < mali_mem_swap_out_threshold_value) + && (mem_backend_swapped_pool_size > temp_swap_out_threshold_value)) { + list_for_each_entry_safe(bkend, tmp_bkend, &mem_backend_swapped_pool, list) { + if (mem_backend_swapped_pool_size <= temp_swap_out_threshold_value) { + break; + } + + mutex_lock(&bkend->mutex); + + /* check if backend is in use. */ + if (0 < bkend->using_count) { + mutex_unlock(&bkend->mutex); + continue; + } + + mali_mem_swap_unlock_single_mem_backend(bkend); + list_del_init(&bkend->list); + mem_backend_swapped_pool_size -= bkend->size; +#ifdef MALI_MEM_SWAP_TRACKING + mem_backend_swapped_unlock_size += bkend->size; +#endif + mutex_unlock(&bkend->mutex); + } + } + + return; +} + +static void mali_mem_swap_swapped_bkend_pool_check_for_low_utilization(void *arg) +{ + MALI_IGNORE(arg); + + mutex_lock(&mem_backend_swapped_pool_lock); + + mali_mem_swap_swapped_bkend_pool_shrink(MALI_MEM_SWAP_SHRINK_WITH_LOW_UTILIZATION); + + mutex_unlock(&mem_backend_swapped_pool_lock); +} + +/** + * After PP job finished, we add all of swappable memory backend used by this PP + * job to the tail of the global swapped pool, and if the total size of swappable memory is more than threshold + * value, we also need to shrink the swapped pool start from the head of the list. + */ +void mali_memory_swap_list_backend_add(mali_mem_backend *mem_bkend) +{ + mutex_lock(&mem_backend_swapped_pool_lock); + mutex_lock(&mem_bkend->mutex); + + if (mali_memory_swap_backend_in_swapped_pool(mem_bkend)) { + MALI_DEBUG_ASSERT(!list_empty(&mem_bkend->list)); + + list_del_init(&mem_bkend->list); + list_add_tail(&mem_bkend->list, &mem_backend_swapped_pool); + mutex_unlock(&mem_bkend->mutex); + mutex_unlock(&mem_backend_swapped_pool_lock); + return; + } + + list_add_tail(&mem_bkend->list, &mem_backend_swapped_pool); + + mutex_unlock(&mem_bkend->mutex); + mem_backend_swapped_pool_size += mem_bkend->size; + + mali_mem_swap_swapped_bkend_pool_shrink(MALI_MEM_SWAP_SHRINK_FOR_ADDING_NEW_BACKENDS); + + mutex_unlock(&mem_backend_swapped_pool_lock); + return; +} + + +u32 mali_mem_swap_idx_alloc(void) +{ + return _mali_osk_bitmap_alloc(&idx_mgr); +} + +void mali_mem_swap_idx_free(u32 idx) +{ + _mali_osk_bitmap_free(&idx_mgr, idx); +} + +static u32 mali_mem_swap_idx_range_alloc(u32 count) +{ + u32 index; + + index = _mali_osk_bitmap_alloc_range(&idx_mgr, count); + + return index; +} + +static void mali_mem_swap_idx_range_free(u32 idx, int num) +{ + _mali_osk_bitmap_free_range(&idx_mgr, idx, num); +} + +struct mali_swap_item *mali_mem_swap_alloc_swap_item(void) +{ + mali_swap_item *swap_item; + + swap_item = kzalloc(sizeof(mali_swap_item), GFP_KERNEL); + + if (NULL == swap_item) { + return NULL; + } + + atomic_set(&swap_item->ref_count, 1); + swap_item->page = NULL; + atomic_add(1, &mali_mem_os_allocator.allocated_pages); + + return swap_item; +} + +void mali_mem_swap_free_swap_item(mali_swap_item *swap_item) +{ + struct inode *file_node; + long long start, end; + + /* If this swap item is shared, we just reduce the reference counter. */ + if (0 == atomic_dec_return(&swap_item->ref_count)) { + file_node = global_swap_file->f_path.dentry->d_inode; + start = swap_item->idx; + start = start << 12; + end = start + PAGE_SIZE; + + shmem_truncate_range(file_node, start, (end - 1)); + + mali_mem_swap_idx_free(swap_item->idx); + + atomic_sub(1, &mali_mem_os_allocator.allocated_pages); + + kfree(swap_item); + } +} + +/* Used to allocate new swap item for new memory allocation and cow page for write. */ +struct mali_page_node *_mali_mem_swap_page_node_allocate(void) +{ + struct mali_page_node *m_page; + + m_page = _mali_page_node_allocate(MALI_PAGE_NODE_SWAP); + + if (NULL == m_page) { + return NULL; + } + + m_page->swap_it = mali_mem_swap_alloc_swap_item(); + + if (NULL == m_page->swap_it) { + kfree(m_page); + return NULL; + } + + return m_page; +} + +_mali_osk_errcode_t _mali_mem_swap_put_page_node(struct mali_page_node *m_page) +{ + + mali_mem_swap_free_swap_item(m_page->swap_it); + + return _MALI_OSK_ERR_OK; +} + +void _mali_mem_swap_page_node_free(struct mali_page_node *m_page) +{ + _mali_mem_swap_put_page_node(m_page); + + kfree(m_page); + + return; +} + +u32 mali_mem_swap_free(mali_mem_swap *swap_mem) +{ + struct mali_page_node *m_page, *m_tmp; + u32 free_pages_nr = 0; + + MALI_DEBUG_ASSERT_POINTER(swap_mem); + + list_for_each_entry_safe(m_page, m_tmp, &swap_mem->pages, list) { + MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_SWAP); + + /* free the page node and release the swap item, if the ref count is 1, + * then need also free the swap item. */ + list_del(&m_page->list); + if (1 == _mali_page_node_get_ref_count(m_page)) { + free_pages_nr++; + } + + _mali_mem_swap_page_node_free(m_page); + } + + return free_pages_nr; +} + +static u32 mali_mem_swap_cow_free(mali_mem_cow *cow_mem) +{ + struct mali_page_node *m_page, *m_tmp; + u32 free_pages_nr = 0; + + MALI_DEBUG_ASSERT_POINTER(cow_mem); + + list_for_each_entry_safe(m_page, m_tmp, &cow_mem->pages, list) { + MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_SWAP); + + /* free the page node and release the swap item, if the ref count is 1, + * then need also free the swap item. */ + list_del(&m_page->list); + if (1 == _mali_page_node_get_ref_count(m_page)) { + free_pages_nr++; + } + + _mali_mem_swap_page_node_free(m_page); + } + + return free_pages_nr; +} + +u32 mali_mem_swap_release(mali_mem_backend *mem_bkend, mali_bool is_mali_mapped) +{ + mali_mem_allocation *alloc; + u32 free_pages_nr = 0; + + MALI_DEBUG_ASSERT_POINTER(mem_bkend); + alloc = mem_bkend->mali_allocation; + MALI_DEBUG_ASSERT_POINTER(alloc); + + if (is_mali_mapped) { + mali_mem_swap_mali_unmap(alloc); + } + + mali_memory_swap_list_backend_delete(mem_bkend); + + mutex_lock(&mem_bkend->mutex); + /* To make sure the given memory backend was unlocked from Mali side, + * and then free this memory block. */ + mali_mem_swap_unlock_single_mem_backend(mem_bkend); + mutex_unlock(&mem_bkend->mutex); + + if (MALI_MEM_SWAP == mem_bkend->type) { + free_pages_nr = mali_mem_swap_free(&mem_bkend->swap_mem); + } else { + free_pages_nr = mali_mem_swap_cow_free(&mem_bkend->cow_mem); + } + + return free_pages_nr; +} + +mali_bool mali_mem_swap_in_page_node(struct mali_page_node *page_node) +{ + MALI_DEBUG_ASSERT(NULL != page_node); + + page_node->swap_it->page = shmem_read_mapping_page(global_swap_space, page_node->swap_it->idx); + + if (IS_ERR(page_node->swap_it->page)) { + MALI_DEBUG_PRINT_ERROR(("SWAP Mem: failed to swap in page with index: %d.\n", page_node->swap_it->idx)); + return MALI_FALSE; + } + + /* Ensure page is flushed from CPU caches. */ + page_node->swap_it->dma_addr = dma_map_page(&mali_platform_device->dev, page_node->swap_it->page, + 0, _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE); + + return MALI_TRUE; +} + +int mali_mem_swap_alloc_pages(mali_mem_swap *swap_mem, u32 size, u32 *bkend_idx) +{ + size_t page_count = PAGE_ALIGN(size) / PAGE_SIZE; + struct mali_page_node *m_page; + long system_free_size; + u32 i, index; + mali_bool ret; + + MALI_DEBUG_ASSERT(NULL != swap_mem); + MALI_DEBUG_ASSERT(NULL != bkend_idx); + MALI_DEBUG_ASSERT(page_count <= MALI_SWAP_GLOBAL_SWAP_FILE_INDEX_RESERVE); + + if (atomic_read(&mali_mem_os_allocator.allocated_pages) * _MALI_OSK_MALI_PAGE_SIZE + size > mali_mem_os_allocator.allocation_limit) { + MALI_DEBUG_PRINT(2, ("Mali Mem: Unable to allocate %u bytes. Currently allocated: %lu, max limit %lu\n", + size, + atomic_read(&mali_mem_os_allocator.allocated_pages) * _MALI_OSK_MALI_PAGE_SIZE, + mali_mem_os_allocator.allocation_limit)); + return _MALI_OSK_ERR_NOMEM; + } + + INIT_LIST_HEAD(&swap_mem->pages); + swap_mem->count = page_count; + index = mali_mem_swap_idx_range_alloc(page_count); + + if (_MALI_OSK_BITMAP_INVALIDATE_INDEX == index) { + MALI_PRINT_ERROR(("Mali Swap: Failed to allocate continuous index for swappable Mali memory.")); + return _MALI_OSK_ERR_FAULT; + } + + for (i = 0; i < page_count; i++) { + m_page = _mali_mem_swap_page_node_allocate(); + + if (NULL == m_page) { + MALI_DEBUG_PRINT_ERROR(("SWAP Mem: Failed to allocate mali page node.")); + swap_mem->count = i; + + mali_mem_swap_free(swap_mem); + mali_mem_swap_idx_range_free(index + i, page_count - i); + return _MALI_OSK_ERR_FAULT; + } + + m_page->swap_it->idx = index + i; + + ret = mali_mem_swap_in_page_node(m_page); + + if (MALI_FALSE == ret) { + MALI_DEBUG_PRINT_ERROR(("SWAP Mem: Allocate new page from SHMEM file failed.")); + _mali_mem_swap_page_node_free(m_page); + mali_mem_swap_idx_range_free(index + i + 1, page_count - i - 1); + + swap_mem->count = i; + mali_mem_swap_free(swap_mem); + return _MALI_OSK_ERR_NOMEM; + } + + list_add_tail(&m_page->list, &swap_mem->pages); + } + + system_free_size = global_page_state(NR_FREE_PAGES) * PAGE_SIZE; + + if ((system_free_size < mali_mem_swap_out_threshold_value) + && (mem_backend_swapped_pool_size > (mali_mem_swap_out_threshold_value >> 2)) + && mali_utilization_enabled()) { + _mali_osk_wq_schedule_work(mali_mem_swap_out_workq); + } + + *bkend_idx = index; + return 0; +} + +void mali_mem_swap_mali_unmap(mali_mem_allocation *alloc) +{ + struct mali_session_data *session; + + MALI_DEBUG_ASSERT_POINTER(alloc); + session = alloc->session; + MALI_DEBUG_ASSERT_POINTER(session); + + mali_session_memory_lock(session); + mali_mem_mali_map_free(session, alloc->psize, alloc->mali_vma_node.vm_node.start, + alloc->flags); + mali_session_memory_unlock(session); +} + + +/* Insert these pages from shmem to mali page table*/ +_mali_osk_errcode_t mali_mem_swap_mali_map(mali_mem_swap *swap_mem, struct mali_session_data *session, u32 vaddr, u32 props) +{ + struct mali_page_directory *pagedir = session->page_directory; + struct mali_page_node *m_page; + dma_addr_t phys; + u32 virt = vaddr; + u32 prop = props; + + list_for_each_entry(m_page, &swap_mem->pages, list) { + MALI_DEBUG_ASSERT(NULL != m_page->swap_it->page); + phys = m_page->swap_it->dma_addr; + + mali_mmu_pagedir_update(pagedir, virt, phys, MALI_MMU_PAGE_SIZE, prop); + virt += MALI_MMU_PAGE_SIZE; + } + + return _MALI_OSK_ERR_OK; +} + +int mali_mem_swap_in_pages(struct mali_pp_job *job) +{ + u32 num_memory_cookies; + struct mali_session_data *session; + struct mali_vma_node *mali_vma_node = NULL; + mali_mem_allocation *mali_alloc = NULL; + mali_mem_backend *mem_bkend = NULL; + struct mali_page_node *m_page; + mali_bool swap_in_success = MALI_TRUE; + int i; + + MALI_DEBUG_ASSERT_POINTER(job); + + num_memory_cookies = mali_pp_job_num_memory_cookies(job); + session = mali_pp_job_get_session(job); + + MALI_DEBUG_ASSERT_POINTER(session); + + for (i = 0; i < num_memory_cookies; i++) { + + u32 mali_addr = mali_pp_job_get_memory_cookie(job, i); + + mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0); + if (NULL == mali_vma_node) { + job->memory_cookies[i] = MALI_SWAP_INVALIDATE_MALI_ADDRESS; + swap_in_success = MALI_FALSE; + MALI_PRINT_ERROR(("SWAP Mem: failed to find mali_vma_node through Mali address: 0x%08x.\n", mali_addr)); + continue; + } + + mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); + MALI_DEBUG_ASSERT(NULL != mali_alloc); + + if (MALI_MEM_SWAP != mali_alloc->type && + MALI_MEM_COW != mali_alloc->type) { + continue; + } + + /* Get backend memory & Map on GPU */ + mutex_lock(&mali_idr_mutex); + mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle); + mutex_unlock(&mali_idr_mutex); + MALI_DEBUG_ASSERT(NULL != mem_bkend); + + /* We neednot hold backend's lock here, race safe.*/ + if ((MALI_MEM_COW == mem_bkend->type) && + (!(mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED))) { + continue; + } + + mutex_lock(&mem_bkend->mutex); + + /* When swap_in_success is MALI_FALSE, it means this job has memory backend that could not be swapped in, + * and it will be aborted in mali scheduler, so here, we just mark those memory cookies which + * should not be swapped out when delete job to invalide */ + if (MALI_FALSE == swap_in_success) { + job->memory_cookies[i] = MALI_SWAP_INVALIDATE_MALI_ADDRESS; + mutex_unlock(&mem_bkend->mutex); + continue; + } + + /* Before swap in, checking if this memory backend has been swapped in by the latest flushed jobs. */ + ++mem_bkend->using_count; + + if (1 < mem_bkend->using_count) { + MALI_DEBUG_ASSERT(MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN != (MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN & mem_bkend->flags)); + mutex_unlock(&mem_bkend->mutex); + continue; + } + + if (MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN != (MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN & mem_bkend->flags)) { + mutex_unlock(&mem_bkend->mutex); + continue; + } + + + list_for_each_entry(m_page, &mem_bkend->swap_mem.pages, list) { + if (MALI_FALSE == mali_mem_swap_in_page_node(m_page)) { + /* Don't have enough memory to swap in page, so release pages have already been swapped + * in and then mark this pp job to be fail. */ + mali_mem_swap_unlock_partial_locked_mem_backend(mem_bkend, m_page); + swap_in_success = MALI_FALSE; + break; + } + } + + if (swap_in_success) { +#ifdef MALI_MEM_SWAP_TRACKING + mem_backend_swapped_unlock_size -= mem_bkend->size; +#endif + _mali_osk_mutex_wait(session->memory_lock); + mali_mem_swap_mali_map(&mem_bkend->swap_mem, session, mali_alloc->mali_mapping.addr, mali_alloc->mali_mapping.properties); + _mali_osk_mutex_signal(session->memory_lock); + + /* Remove the unlock flag from mem backend flags, mark this backend has been swapped in. */ + mem_bkend->flags &= ~(MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN); + mutex_unlock(&mem_bkend->mutex); + } else { + --mem_bkend->using_count; + /* Marking that this backend is not swapped in, need not to be processed anymore. */ + job->memory_cookies[i] = MALI_SWAP_INVALIDATE_MALI_ADDRESS; + mutex_unlock(&mem_bkend->mutex); + } + } + + job->swap_status = swap_in_success ? MALI_SWAP_IN_SUCC : MALI_SWAP_IN_FAIL; + + return _MALI_OSK_ERR_OK; +} + +int mali_mem_swap_out_pages(struct mali_pp_job *job) +{ + u32 num_memory_cookies; + struct mali_session_data *session; + struct mali_vma_node *mali_vma_node = NULL; + mali_mem_allocation *mali_alloc = NULL; + mali_mem_backend *mem_bkend = NULL; + int i; + + MALI_DEBUG_ASSERT_POINTER(job); + + num_memory_cookies = mali_pp_job_num_memory_cookies(job); + session = mali_pp_job_get_session(job); + + MALI_DEBUG_ASSERT_POINTER(session); + + + for (i = 0; i < num_memory_cookies; i++) { + u32 mali_addr = mali_pp_job_get_memory_cookie(job, i); + + if (MALI_SWAP_INVALIDATE_MALI_ADDRESS == mali_addr) { + continue; + } + + mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0); + + if (NULL == mali_vma_node) { + MALI_PRINT_ERROR(("SWAP Mem: failed to find mali_vma_node through Mali address: 0x%08x.\n", mali_addr)); + continue; + } + + mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); + MALI_DEBUG_ASSERT(NULL != mali_alloc); + + if (MALI_MEM_SWAP != mali_alloc->type && + MALI_MEM_COW != mali_alloc->type) { + continue; + } + + mutex_lock(&mali_idr_mutex); + mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle); + mutex_unlock(&mali_idr_mutex); + MALI_DEBUG_ASSERT(NULL != mem_bkend); + + /* We neednot hold backend's lock here, race safe.*/ + if ((MALI_MEM_COW == mem_bkend->type) && + (!(mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED))) { + continue; + } + + mutex_lock(&mem_bkend->mutex); + + MALI_DEBUG_ASSERT(0 < mem_bkend->using_count); + + /* Reducing the using_count of mem backend means less pp job are using this memory backend, + * if this count get to zero, it means no pp job is using it now, could put it to swap out list. */ + --mem_bkend->using_count; + + if (0 < mem_bkend->using_count) { + mutex_unlock(&mem_bkend->mutex); + continue; + } + mutex_unlock(&mem_bkend->mutex); + + mali_memory_swap_list_backend_add(mem_bkend); + } + + return _MALI_OSK_ERR_OK; +} + +int mali_mem_swap_allocate_page_on_demand(mali_mem_backend *mem_bkend, u32 offset, struct page **pagep) +{ + struct mali_page_node *m_page, *found_node = NULL; + struct page *found_page; + mali_mem_swap *swap = NULL; + mali_mem_cow *cow = NULL; + dma_addr_t dma_addr; + u32 i = 0; + + if (MALI_MEM_SWAP == mem_bkend->type) { + swap = &mem_bkend->swap_mem; + list_for_each_entry(m_page, &swap->pages, list) { + if (i == offset) { + found_node = m_page; + break; + } + i++; + } + } else { + MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_bkend->type); + MALI_DEBUG_ASSERT(MALI_MEM_BACKEND_FLAG_SWAP_COWED == (MALI_MEM_BACKEND_FLAG_SWAP_COWED & mem_bkend->flags)); + + cow = &mem_bkend->cow_mem; + list_for_each_entry(m_page, &cow->pages, list) { + if (i == offset) { + found_node = m_page; + break; + } + i++; + } + } + + if (NULL == found_node) { + return _MALI_OSK_ERR_FAULT; + } + + found_page = shmem_read_mapping_page(global_swap_space, found_node->swap_it->idx); + + if (!IS_ERR(found_page)) { + lock_page(found_page); + dma_addr = dma_map_page(&mali_platform_device->dev, found_page, + 0, _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE); + dma_unmap_page(&mali_platform_device->dev, dma_addr, + _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE); + + *pagep = found_page; + } else { + return _MALI_OSK_ERR_NOMEM; + } + + return _MALI_OSK_ERR_OK; +} + +int mali_mem_swap_cow_page_on_demand(mali_mem_backend *mem_bkend, u32 offset, struct page **pagep) +{ + struct mali_page_node *m_page, *found_node = NULL, *new_node = NULL; + mali_mem_cow *cow = NULL; + u32 i = 0; + + MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_bkend->type); + MALI_DEBUG_ASSERT(MALI_MEM_BACKEND_FLAG_SWAP_COWED == (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED)); + MALI_DEBUG_ASSERT(MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN == (MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN & mem_bkend->flags)); + MALI_DEBUG_ASSERT(!mali_memory_swap_backend_in_swapped_pool(mem_bkend)); + + cow = &mem_bkend->cow_mem; + list_for_each_entry(m_page, &cow->pages, list) { + if (i == offset) { + found_node = m_page; + break; + } + i++; + } + + if (NULL == found_node) { + return _MALI_OSK_ERR_FAULT; + } + + new_node = _mali_mem_swap_page_node_allocate(); + + if (NULL == new_node) { + return _MALI_OSK_ERR_FAULT; + } + + new_node->swap_it->idx = mali_mem_swap_idx_alloc(); + + if (_MALI_OSK_BITMAP_INVALIDATE_INDEX == new_node->swap_it->idx) { + MALI_DEBUG_PRINT(1, ("Failed to allocate swap index in swap CoW on demand.\n")); + kfree(new_node->swap_it); + kfree(new_node); + return _MALI_OSK_ERR_FAULT; + } + + if (MALI_FALSE == mali_mem_swap_in_page_node(new_node)) { + _mali_mem_swap_page_node_free(new_node); + return _MALI_OSK_ERR_FAULT; + } + + /* swap in found node for copy in kernel. */ + if (MALI_FALSE == mali_mem_swap_in_page_node(found_node)) { + mali_mem_swap_out_page_node(new_node); + _mali_mem_swap_page_node_free(new_node); + return _MALI_OSK_ERR_FAULT; + } + + _mali_mem_cow_copy_page(found_node, new_node); + + list_replace(&found_node->list, &new_node->list); + + if (1 != _mali_page_node_get_ref_count(found_node)) { + atomic_add(1, &mem_bkend->mali_allocation->session->mali_mem_allocated_pages); + if (atomic_read(&mem_bkend->mali_allocation->session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > mem_bkend->mali_allocation->session->max_mali_mem_allocated_size) { + mem_bkend->mali_allocation->session->max_mali_mem_allocated_size = atomic_read(&mem_bkend->mali_allocation->session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE; + } + mem_bkend->cow_mem.change_pages_nr++; + } + + mali_mem_swap_out_page_node(found_node); + _mali_mem_swap_page_node_free(found_node); + + /* When swap in the new page node, we have called dma_map_page for this page.\n */ + dma_unmap_page(&mali_platform_device->dev, new_node->swap_it->dma_addr, + _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE); + + lock_page(new_node->swap_it->page); + + *pagep = new_node->swap_it->page; + + return _MALI_OSK_ERR_OK; +} + +#ifdef MALI_MEM_SWAP_TRACKING +void mali_mem_swap_tracking(u32 *swap_pool_size, u32 *unlock_size) +{ + *swap_pool_size = mem_backend_swapped_pool_size; + *unlock_size = mem_backend_swapped_unlock_size; +} +#endif diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_swap_alloc.h b/drivers/gpu/arm/utgard/linux/mali_memory_swap_alloc.h new file mode 100644 index 00000000000000..a393ecce3a006c --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_memory_swap_alloc.h @@ -0,0 +1,121 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_MEMORY_SWAP_ALLOC_H__ +#define __MALI_MEMORY_SWAP_ALLOC_H__ + +#include "mali_osk.h" +#include "mali_session.h" + +#include "mali_memory_types.h" +#include "mali_pp_job.h" + +/** + * Initialize memory swapping module. + */ +_mali_osk_errcode_t mali_mem_swap_init(void); + +void mali_mem_swap_term(void); + +/** + * Return global share memory file to other modules. + */ +struct file *mali_mem_swap_get_global_swap_file(void); + +/** + * Unlock the given memory backend and pages in it could be swapped out by kernel. + */ +void mali_mem_swap_unlock_single_mem_backend(mali_mem_backend *mem_bkend); + +/** + * Remove the given memory backend from global swap list. + */ +void mali_memory_swap_list_backend_delete(mali_mem_backend *mem_bkend); + +/** + * Add the given memory backend to global swap list. + */ +void mali_memory_swap_list_backend_add(mali_mem_backend *mem_bkend); + +/** + * Allocate 1 index from bitmap used as page index in global swap file. + */ +u32 mali_mem_swap_idx_alloc(void); + +void mali_mem_swap_idx_free(u32 idx); + +/** + * Allocate a new swap item without page index. + */ +struct mali_swap_item *mali_mem_swap_alloc_swap_item(void); + +/** + * Free a swap item, truncate the corresponding space in page cache and free index of page. + */ +void mali_mem_swap_free_swap_item(mali_swap_item *swap_item); + +/** + * Allocate a page node with swap item. + */ +struct mali_page_node *_mali_mem_swap_page_node_allocate(void); + +/** + * Reduce the reference count of given page node and if return 0, just free this page node. + */ +_mali_osk_errcode_t _mali_mem_swap_put_page_node(struct mali_page_node *m_page); + +void _mali_mem_swap_page_node_free(struct mali_page_node *m_page); + +/** + * Free a swappable memory backend. + */ +u32 mali_mem_swap_free(mali_mem_swap *swap_mem); + +/** + * Ummap and free. + */ +u32 mali_mem_swap_release(mali_mem_backend *mem_bkend, mali_bool is_mali_mapped); + +/** + * Read in a page from global swap file with the pre-allcated page index. + */ +mali_bool mali_mem_swap_in_page_node(struct mali_page_node *page_node); + +int mali_mem_swap_alloc_pages(mali_mem_swap *swap_mem, u32 size, u32 *bkend_idx); + +_mali_osk_errcode_t mali_mem_swap_mali_map(mali_mem_swap *swap_mem, struct mali_session_data *session, u32 vaddr, u32 props); + +void mali_mem_swap_mali_unmap(mali_mem_allocation *alloc); + +/** + * When pp job created, we need swap in all of memory backend needed by this pp job. + */ +int mali_mem_swap_in_pages(struct mali_pp_job *job); + +/** + * Put all of memory backends used this pp job to the global swap list. + */ +int mali_mem_swap_out_pages(struct mali_pp_job *job); + +/** + * This will be called in page fault to process CPU read&write. + */ +int mali_mem_swap_allocate_page_on_demand(mali_mem_backend *mem_bkend, u32 offset, struct page **pagep) ; + +/** + * Used to process cow on demand for swappable memory backend. + */ +int mali_mem_swap_cow_page_on_demand(mali_mem_backend *mem_bkend, u32 offset, struct page **pagep); + +#ifdef MALI_MEM_SWAP_TRACKING +void mali_mem_swap_tracking(u32 *swap_pool_size, u32 *unlock_size); +#endif +#endif /* __MALI_MEMORY_SWAP_ALLOC_H__ */ + diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_types.h b/drivers/gpu/arm/utgard/linux/mali_memory_types.h new file mode 100644 index 00000000000000..82af8fed66c95d --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_memory_types.h @@ -0,0 +1,208 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_MEMORY_TYPES_H__ +#define __MALI_MEMORY_TYPES_H__ + +#include + +#if defined(CONFIG_MALI400_UMP) +#include "ump_kernel_interface.h" +#endif + +typedef u32 mali_address_t; + +typedef enum mali_mem_type { + MALI_MEM_OS, + MALI_MEM_EXTERNAL, + MALI_MEM_SWAP, + MALI_MEM_DMA_BUF, + MALI_MEM_UMP, + MALI_MEM_BLOCK, + MALI_MEM_COW, + MALI_MEM_TYPE_MAX, +} mali_mem_type; + +typedef struct mali_block_item { + /* for block type, the block_phy is alway page size align + * so use low 12bit used for ref_cout. + */ + unsigned long phy_addr; +} mali_block_item; + +/** + * idx is used to locate the given page in the address space of swap file. + * ref_count is used to mark how many memory backends are using this item. + */ +typedef struct mali_swap_item { + u32 idx; + atomic_t ref_count; + struct page *page; + dma_addr_t dma_addr; +} mali_swap_item; + +typedef enum mali_page_node_type { + MALI_PAGE_NODE_OS, + MALI_PAGE_NODE_BLOCK, + MALI_PAGE_NODE_SWAP, +} mali_page_node_type; + +typedef struct mali_page_node { + struct list_head list; + union { + struct page *page; + mali_block_item *blk_it; /*pointer to block item*/ + mali_swap_item *swap_it; + }; + + u32 type; +} mali_page_node; + +typedef struct mali_mem_os_mem { + struct list_head pages; + u32 count; +} mali_mem_os_mem; + +typedef struct mali_mem_dma_buf { +#if defined(CONFIG_DMA_SHARED_BUFFER) + struct mali_dma_buf_attachment *attachment; +#endif +} mali_mem_dma_buf; + +typedef struct mali_mem_external { + dma_addr_t phys; + u32 size; +} mali_mem_external; + +typedef struct mali_mem_ump { +#if defined(CONFIG_MALI400_UMP) + ump_dd_handle handle; +#endif +} mali_mem_ump; + +typedef struct block_allocator_allocation { + /* The list will be released in reverse order */ + struct block_info *last_allocated; + u32 mapping_length; + struct block_allocator *info; +} block_allocator_allocation; + +typedef struct mali_mem_block_mem { + struct list_head pfns; + u32 count; +} mali_mem_block_mem; + +typedef struct mali_mem_virt_mali_mapping { + mali_address_t addr; /* Virtual Mali address */ + u32 properties; /* MMU Permissions + cache, must match MMU HW */ +} mali_mem_virt_mali_mapping; + +typedef struct mali_mem_virt_cpu_mapping { + void __user *addr; + struct vm_area_struct *vma; +} mali_mem_virt_cpu_mapping; + +#define MALI_MEM_ALLOCATION_VALID_MAGIC 0xdeda110c +#define MALI_MEM_ALLOCATION_FREED_MAGIC 0x10101010 + +typedef struct mali_mm_node { + /* MALI GPU vaddr start, use u32 for mmu only support 32bit address*/ + uint32_t start; /* GPU vaddr */ + uint32_t size; /* GPU allocation virtual size */ + unsigned allocated : 1; +} mali_mm_node; + +typedef struct mali_vma_node { + struct mali_mm_node vm_node; + struct rb_node vm_rb; +} mali_vma_node; + + +typedef struct mali_mem_allocation { + MALI_DEBUG_CODE(u32 magic); + mali_mem_type type; /**< Type of memory */ + u32 flags; /**< Flags for this allocation */ + + struct mali_session_data *session; /**< Pointer to session that owns the allocation */ + + mali_mem_virt_cpu_mapping cpu_mapping; /**< CPU mapping */ + mali_mem_virt_mali_mapping mali_mapping; /**< Mali mapping */ + + /* add for new memory system */ + struct mali_vma_node mali_vma_node; + u32 vsize; /* virtual size*/ + u32 psize; /* physical backend memory size*/ + struct list_head list; + s32 backend_handle; /* idr for mem_backend */ + _mali_osk_atomic_t mem_alloc_refcount; +} mali_mem_allocation; + +struct mali_mem_os_allocator { + spinlock_t pool_lock; + struct list_head pool_pages; + size_t pool_count; + + atomic_t allocated_pages; + size_t allocation_limit; + + struct shrinker shrinker; + struct delayed_work timed_shrinker; + struct workqueue_struct *wq; +}; + +/* COW backend memory type */ +typedef struct mali_mem_cow { + struct list_head pages; /**< all pages for this cow backend allocation, + including new allocated pages for modified range*/ + u32 count; /**< number of pages */ + s32 change_pages_nr; +} mali_mem_cow; + +typedef struct mali_mem_swap { + struct list_head pages; + u32 count; +} mali_mem_swap; + +#define MALI_MEM_BACKEND_FLAG_COWED (0x1) /* COW has happen on this backend */ +#define MALI_MEM_BACKEND_FLAG_COW_CPU_NO_WRITE (0x2) /* This is an COW backend, mapped as not allowed cpu to write */ +#define MALI_MEM_BACKEND_FLAG_SWAP_COWED (0x4) /* Mark the given backend is cowed from swappable memory. */ +/* Mark this backend is not swapped_in in MALI driver, and before using it, + * we should swap it in and set up corresponding page table. */ +#define MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN (0x8) +#define MALI_MEM_BACKEND_FLAG_NOT_BINDED (0x1 << 5) /* this backend it not back with physical memory, used for defer bind */ +#define MALI_MEM_BACKEND_FLAG_BINDED (0x1 << 6) /* this backend it back with physical memory, used for defer bind */ + +typedef struct mali_mem_backend { + mali_mem_type type; /**< Type of backend memory */ + u32 flags; /**< Flags for this allocation */ + u32 size; + /* Union selected by type. */ + union { + mali_mem_os_mem os_mem; /**< MALI_MEM_OS */ + mali_mem_external ext_mem; /**< MALI_MEM_EXTERNAL */ + mali_mem_dma_buf dma_buf; /**< MALI_MEM_DMA_BUF */ + mali_mem_ump ump_mem; /**< MALI_MEM_UMP */ + mali_mem_block_mem block_mem; /**< MALI_MEM_BLOCK */ + mali_mem_cow cow_mem; + mali_mem_swap swap_mem; + }; + mali_mem_allocation *mali_allocation; + struct mutex mutex; + mali_mem_type cow_type; + + struct list_head list; /**< Used to link swappable memory backend to the global swappable list */ + int using_count; /**< Mark how many PP jobs are using this memory backend */ + u32 start_idx; /**< If the correspondign vma of this backend is linear, this value will be used to set vma->vm_pgoff */ +} mali_mem_backend; + +#define MALI_MEM_FLAG_MALI_GUARD_PAGE (_MALI_MAP_EXTERNAL_MAP_GUARD_PAGE) +#define MALI_MEM_FLAG_DONT_CPU_MAP (1 << 1) +#define MALI_MEM_FLAG_CAN_RESIZE (_MALI_MEMORY_ALLOCATE_RESIZEABLE) +#endif /* __MALI_MEMORY_TYPES__ */ diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_ump.c b/drivers/gpu/arm/utgard/linux/mali_memory_ump.c new file mode 100644 index 00000000000000..bacadc85e47e85 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_memory_ump.c @@ -0,0 +1,154 @@ +/* + * Copyright (C) 2012-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_ukk.h" +#include "mali_osk.h" +#include "mali_kernel_common.h" +#include "mali_session.h" +#include "mali_kernel_linux.h" +#include "mali_memory.h" +#include "ump_kernel_interface.h" + +static int mali_mem_ump_map(mali_mem_backend *mem_backend) +{ + ump_dd_handle ump_mem; + mali_mem_allocation *alloc; + struct mali_session_data *session; + u32 nr_blocks; + u32 i; + ump_dd_physical_block *ump_blocks; + struct mali_page_directory *pagedir; + u32 offset = 0; + _mali_osk_errcode_t err; + + MALI_DEBUG_ASSERT_POINTER(mem_backend); + MALI_DEBUG_ASSERT(MALI_MEM_UMP == mem_backend->type); + + alloc = mem_backend->mali_allocation; + MALI_DEBUG_ASSERT_POINTER(alloc); + + session = alloc->session; + MALI_DEBUG_ASSERT_POINTER(session); + + ump_mem = mem_backend->ump_mem.handle; + MALI_DEBUG_ASSERT(UMP_DD_HANDLE_INVALID != ump_mem); + + nr_blocks = ump_dd_phys_block_count_get(ump_mem); + if (nr_blocks == 0) { + MALI_DEBUG_PRINT(1, ("No block count\n")); + return -EINVAL; + } + + ump_blocks = _mali_osk_malloc(sizeof(*ump_blocks) * nr_blocks); + if (NULL == ump_blocks) { + return -ENOMEM; + } + + if (UMP_DD_INVALID == ump_dd_phys_blocks_get(ump_mem, ump_blocks, nr_blocks)) { + _mali_osk_free(ump_blocks); + return -EFAULT; + } + + pagedir = session->page_directory; + + mali_session_memory_lock(session); + + err = mali_mem_mali_map_prepare(alloc); + if (_MALI_OSK_ERR_OK != err) { + MALI_DEBUG_PRINT(1, ("Mapping of UMP memory failed\n")); + + _mali_osk_free(ump_blocks); + mali_session_memory_unlock(session); + return -ENOMEM; + } + + for (i = 0; i < nr_blocks; ++i) { + u32 virt = alloc->mali_vma_node.vm_node.start + offset; + + MALI_DEBUG_PRINT(7, ("Mapping in 0x%08x size %d\n", ump_blocks[i].addr , ump_blocks[i].size)); + + mali_mmu_pagedir_update(pagedir, virt, ump_blocks[i].addr, + ump_blocks[i].size, MALI_MMU_FLAGS_DEFAULT); + + offset += ump_blocks[i].size; + } + + if (alloc->flags & _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE) { + u32 virt = alloc->mali_vma_node.vm_node.start + offset; + + /* Map in an extra virtual guard page at the end of the VMA */ + MALI_DEBUG_PRINT(6, ("Mapping in extra guard page\n")); + + mali_mmu_pagedir_update(pagedir, virt, ump_blocks[0].addr, _MALI_OSK_MALI_PAGE_SIZE, MALI_MMU_FLAGS_DEFAULT); + + offset += _MALI_OSK_MALI_PAGE_SIZE; + } + mali_session_memory_unlock(session); + _mali_osk_free(ump_blocks); + return 0; +} + +static void mali_mem_ump_unmap(mali_mem_allocation *alloc) +{ + struct mali_session_data *session; + MALI_DEBUG_ASSERT_POINTER(alloc); + session = alloc->session; + MALI_DEBUG_ASSERT_POINTER(session); + mali_session_memory_lock(session); + mali_mem_mali_map_free(session, alloc->psize, alloc->mali_vma_node.vm_node.start, + alloc->flags); + mali_session_memory_unlock(session); +} + +int mali_mem_bind_ump_buf(mali_mem_allocation *alloc, mali_mem_backend *mem_backend, u32 secure_id, u32 flags) +{ + ump_dd_handle ump_mem; + int ret; + MALI_DEBUG_ASSERT_POINTER(alloc); + MALI_DEBUG_ASSERT_POINTER(mem_backend); + MALI_DEBUG_ASSERT(MALI_MEM_UMP == mem_backend->type); + + MALI_DEBUG_PRINT(3, + ("Requested to map ump memory with secure id %d into virtual memory 0x%08X, size 0x%08X\n", + secure_id, alloc->mali_vma_node.vm_node.start, alloc->mali_vma_node.vm_node.size)); + + ump_mem = ump_dd_handle_create_from_secure_id(secure_id); + if (UMP_DD_HANDLE_INVALID == ump_mem) MALI_ERROR(_MALI_OSK_ERR_FAULT); + alloc->flags |= MALI_MEM_FLAG_DONT_CPU_MAP; + if (flags & _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE) { + alloc->flags |= MALI_MEM_FLAG_MALI_GUARD_PAGE; + } + + mem_backend->ump_mem.handle = ump_mem; + + ret = mali_mem_ump_map(mem_backend); + if (0 != ret) { + ump_dd_reference_release(ump_mem); + return _MALI_OSK_ERR_FAULT; + } + MALI_DEBUG_PRINT(3, ("Returning from UMP bind\n")); + return _MALI_OSK_ERR_OK; +} + +void mali_mem_unbind_ump_buf(mali_mem_backend *mem_backend) +{ + ump_dd_handle ump_mem; + mali_mem_allocation *alloc; + MALI_DEBUG_ASSERT_POINTER(mem_backend); + MALI_DEBUG_ASSERT(MALI_MEM_UMP == mem_backend->type); + ump_mem = mem_backend->ump_mem.handle; + MALI_DEBUG_ASSERT(UMP_DD_HANDLE_INVALID != ump_mem); + + alloc = mem_backend->mali_allocation; + MALI_DEBUG_ASSERT_POINTER(alloc); + mali_mem_ump_unmap(alloc); + ump_dd_reference_release(ump_mem); +} + diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_ump.h b/drivers/gpu/arm/utgard/linux/mali_memory_ump.h new file mode 100644 index 00000000000000..4adb70df884e4d --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_memory_ump.h @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_MEMORY_UMP_BUF_H__ +#define __MALI_MEMORY_UMP_BUF_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "mali_uk_types.h" +#include "mali_osk.h" +#include "mali_memory.h" + +int mali_mem_bind_ump_buf(mali_mem_allocation *alloc, mali_mem_backend *mem_backend, u32 secure_id, u32 flags); +void mali_mem_unbind_ump_buf(mali_mem_backend *mem_backend); + +#ifdef __cplusplus +} +#endif + +#endif /* __MALI_MEMORY_DMA_BUF_H__ */ diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_util.c b/drivers/gpu/arm/utgard/linux/mali_memory_util.c new file mode 100644 index 00000000000000..bcca43c09f0bb6 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_memory_util.c @@ -0,0 +1,149 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "mali_osk.h" +#include "mali_osk_mali.h" +#include "mali_kernel_linux.h" +#include "mali_scheduler.h" + +#include "mali_memory.h" +#include "mali_memory_os_alloc.h" +#if defined(CONFIG_DMA_SHARED_BUFFER) +#include "mali_memory_dma_buf.h" +#endif +#if defined(CONFIG_MALI400_UMP) +#include "mali_memory_ump.h" +#endif +#include "mali_memory_external.h" +#include "mali_memory_manager.h" +#include "mali_memory_virtual.h" +#include "mali_memory_cow.h" +#include "mali_memory_block_alloc.h" +#include "mali_memory_swap_alloc.h" + + + +/** +*function @_mali_free_allocation_mem - free a memory allocation +*/ +static u32 _mali_free_allocation_mem(mali_mem_allocation *mali_alloc) +{ + mali_mem_backend *mem_bkend = NULL; + u32 free_pages_nr = 0; + + struct mali_session_data *session = mali_alloc->session; + MALI_DEBUG_PRINT(4, (" _mali_free_allocation_mem, psize =0x%x! \n", mali_alloc->psize)); + if (0 == mali_alloc->psize) + goto out; + + /* Get backend memory & Map on CPU */ + mutex_lock(&mali_idr_mutex); + mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle); + mutex_unlock(&mali_idr_mutex); + MALI_DEBUG_ASSERT(NULL != mem_bkend); + + switch (mem_bkend->type) { + case MALI_MEM_OS: + free_pages_nr = mali_mem_os_release(mem_bkend); + atomic_sub(free_pages_nr, &session->mali_mem_allocated_pages); + break; + case MALI_MEM_UMP: +#if defined(CONFIG_MALI400_UMP) + mali_mem_unbind_ump_buf(mem_bkend); + atomic_sub(mem_bkend->size / MALI_MMU_PAGE_SIZE, &session->mali_mem_array[mem_bkend->type]); +#else + MALI_DEBUG_PRINT(2, ("UMP not supported\n")); +#endif + break; + case MALI_MEM_DMA_BUF: +#if defined(CONFIG_DMA_SHARED_BUFFER) + mali_mem_unbind_dma_buf(mem_bkend); + atomic_sub(mem_bkend->size / MALI_MMU_PAGE_SIZE, &session->mali_mem_array[mem_bkend->type]); +#else + MALI_DEBUG_PRINT(2, ("DMA not supported\n")); +#endif + break; + case MALI_MEM_EXTERNAL: + mali_mem_unbind_ext_buf(mem_bkend); + atomic_sub(mem_bkend->size / MALI_MMU_PAGE_SIZE, &session->mali_mem_array[mem_bkend->type]); + break; + + case MALI_MEM_BLOCK: + free_pages_nr = mali_mem_block_release(mem_bkend); + atomic_sub(free_pages_nr, &session->mali_mem_allocated_pages); + break; + + case MALI_MEM_COW: + if (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED) { + free_pages_nr = mali_mem_swap_release(mem_bkend, MALI_TRUE); + } else { + free_pages_nr = mali_mem_cow_release(mem_bkend, MALI_TRUE); + } + atomic_sub(free_pages_nr, &session->mali_mem_allocated_pages); + break; + case MALI_MEM_SWAP: + free_pages_nr = mali_mem_swap_release(mem_bkend, MALI_TRUE); + atomic_sub(free_pages_nr, &session->mali_mem_allocated_pages); + atomic_sub(free_pages_nr, &session->mali_mem_array[mem_bkend->type]); + break; + default: + MALI_DEBUG_PRINT(1, ("mem type %d is not in the mali_mem_type enum.\n", mem_bkend->type)); + break; + } + + /*Remove backend memory idex */ + mutex_lock(&mali_idr_mutex); + idr_remove(&mali_backend_idr, mali_alloc->backend_handle); + mutex_unlock(&mali_idr_mutex); + kfree(mem_bkend); +out: + /* remove memory allocation */ + mali_vma_offset_remove(&session->allocation_mgr, &mali_alloc->mali_vma_node); + mali_mem_allocation_struct_destory(mali_alloc); + return free_pages_nr; +} + +/** +* ref_count for allocation +*/ +u32 mali_allocation_unref(struct mali_mem_allocation **alloc) +{ + u32 free_pages_nr = 0; + mali_mem_allocation *mali_alloc = *alloc; + *alloc = NULL; + if (0 == _mali_osk_atomic_dec_return(&mali_alloc->mem_alloc_refcount)) { + free_pages_nr = _mali_free_allocation_mem(mali_alloc); + } + return free_pages_nr; +} + +void mali_allocation_ref(struct mali_mem_allocation *alloc) +{ + _mali_osk_atomic_inc(&alloc->mem_alloc_refcount); +} + +void mali_free_session_allocations(struct mali_session_data *session) +{ + struct mali_mem_allocation *entry, *next; + + MALI_DEBUG_PRINT(4, (" mali_free_session_allocations! \n")); + + list_for_each_entry_safe(entry, next, &session->allocation_mgr.head, list) { + mali_allocation_unref(&entry); + } +} diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_util.h b/drivers/gpu/arm/utgard/linux/mali_memory_util.h new file mode 100644 index 00000000000000..0d686979d7a8a2 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_memory_util.h @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_MEMORY_UTIL_H__ +#define __MALI_MEMORY_UTIL_H__ + +u32 mali_allocation_unref(struct mali_mem_allocation **alloc); + +void mali_allocation_ref(struct mali_mem_allocation *alloc); + +void mali_free_session_allocations(struct mali_session_data *session); + +#endif diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_virtual.c b/drivers/gpu/arm/utgard/linux/mali_memory_virtual.c new file mode 100644 index 00000000000000..35f8f90a444dee --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_memory_virtual.c @@ -0,0 +1,127 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "mali_osk.h" +#include "mali_osk_mali.h" +#include "mali_kernel_linux.h" +#include "mali_scheduler.h" +#include "mali_memory_os_alloc.h" +#include "mali_memory_manager.h" +#include "mali_memory_virtual.h" + + +/** +*internal helper to link node into the rb-tree +*/ +static inline void _mali_vma_offset_add_rb(struct mali_allocation_manager *mgr, + struct mali_vma_node *node) +{ + struct rb_node **iter = &mgr->allocation_mgr_rb.rb_node; + struct rb_node *parent = NULL; + struct mali_vma_node *iter_node; + + while (likely(*iter)) { + parent = *iter; + iter_node = rb_entry(*iter, struct mali_vma_node, vm_rb); + + if (node->vm_node.start < iter_node->vm_node.start) + iter = &(*iter)->rb_left; + else if (node->vm_node.start > iter_node->vm_node.start) + iter = &(*iter)->rb_right; + else + MALI_DEBUG_ASSERT(0); + } + + rb_link_node(&node->vm_rb, parent, iter); + rb_insert_color(&node->vm_rb, &mgr->allocation_mgr_rb); +} + +/** + * mali_vma_offset_add() - Add offset node to RB Tree + */ +int mali_vma_offset_add(struct mali_allocation_manager *mgr, + struct mali_vma_node *node) +{ + int ret = 0; + write_lock(&mgr->vm_lock); + + if (node->vm_node.allocated) { + goto out; + } + + _mali_vma_offset_add_rb(mgr, node); + /* set to allocated */ + node->vm_node.allocated = 1; + +out: + write_unlock(&mgr->vm_lock); + return ret; +} + +/** + * mali_vma_offset_remove() - Remove offset node from RB tree + */ +void mali_vma_offset_remove(struct mali_allocation_manager *mgr, + struct mali_vma_node *node) +{ + write_lock(&mgr->vm_lock); + + if (node->vm_node.allocated) { + rb_erase(&node->vm_rb, &mgr->allocation_mgr_rb); + memset(&node->vm_node, 0, sizeof(node->vm_node)); + } + write_unlock(&mgr->vm_lock); +} + +/** +* mali_vma_offset_search - Search the node in RB tree +*/ +struct mali_vma_node *mali_vma_offset_search(struct mali_allocation_manager *mgr, + unsigned long start, unsigned long pages) +{ + struct mali_vma_node *node, *best; + struct rb_node *iter; + unsigned long offset; + read_lock(&mgr->vm_lock); + + iter = mgr->allocation_mgr_rb.rb_node; + best = NULL; + + while (likely(iter)) { + node = rb_entry(iter, struct mali_vma_node, vm_rb); + offset = node->vm_node.start; + if (start >= offset) { + iter = iter->rb_right; + best = node; + if (start == offset) + break; + } else { + iter = iter->rb_left; + } + } + + if (best) { + offset = best->vm_node.start + best->vm_node.size; + if (offset <= start + pages) + best = NULL; + } + read_unlock(&mgr->vm_lock); + + return best; +} + diff --git a/drivers/gpu/arm/utgard/linux/mali_memory_virtual.h b/drivers/gpu/arm/utgard/linux/mali_memory_virtual.h new file mode 100644 index 00000000000000..3e2d48f44e28cb --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_memory_virtual.h @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#ifndef __MALI_GPU_VMEM_H__ +#define __MALI_GPU_VMEM_H__ + +#include "mali_osk.h" +#include "mali_session.h" +#include +#include +#include +#include +#include +#include "mali_memory_types.h" +#include "mali_memory_os_alloc.h" +#include "mali_memory_manager.h" + + + +int mali_vma_offset_add(struct mali_allocation_manager *mgr, + struct mali_vma_node *node); + +void mali_vma_offset_remove(struct mali_allocation_manager *mgr, + struct mali_vma_node *node); + +struct mali_vma_node *mali_vma_offset_search(struct mali_allocation_manager *mgr, + unsigned long start, unsigned long pages); + +#endif diff --git a/drivers/gpu/arm/utgard/linux/mali_osk_atomics.c b/drivers/gpu/arm/utgard/linux/mali_osk_atomics.c new file mode 100644 index 00000000000000..ba630e2730cf33 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_osk_atomics.c @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2010, 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_atomics.c + * Implementation of the OS abstraction layer for the kernel device driver + */ + +#include "mali_osk.h" +#include +#include "mali_kernel_common.h" + +void _mali_osk_atomic_dec(_mali_osk_atomic_t *atom) +{ + atomic_dec((atomic_t *)&atom->u.val); +} + +u32 _mali_osk_atomic_dec_return(_mali_osk_atomic_t *atom) +{ + return atomic_dec_return((atomic_t *)&atom->u.val); +} + +void _mali_osk_atomic_inc(_mali_osk_atomic_t *atom) +{ + atomic_inc((atomic_t *)&atom->u.val); +} + +u32 _mali_osk_atomic_inc_return(_mali_osk_atomic_t *atom) +{ + return atomic_inc_return((atomic_t *)&atom->u.val); +} + +void _mali_osk_atomic_init(_mali_osk_atomic_t *atom, u32 val) +{ + MALI_DEBUG_ASSERT_POINTER(atom); + atomic_set((atomic_t *)&atom->u.val, val); +} + +u32 _mali_osk_atomic_read(_mali_osk_atomic_t *atom) +{ + return atomic_read((atomic_t *)&atom->u.val); +} + +void _mali_osk_atomic_term(_mali_osk_atomic_t *atom) +{ + MALI_IGNORE(atom); +} + +u32 _mali_osk_atomic_xchg(_mali_osk_atomic_t *atom, u32 val) +{ + return atomic_xchg((atomic_t *)&atom->u.val, val); +} diff --git a/drivers/gpu/arm/utgard/linux/mali_osk_bitmap.c b/drivers/gpu/arm/utgard/linux/mali_osk_bitmap.c new file mode 100644 index 00000000000000..01ca38235b201f --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_osk_bitmap.c @@ -0,0 +1,152 @@ +/* + * Copyright (C) 2010, 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_bitmap.c + * Implementation of the OS abstraction layer for the kernel device driver + */ + +#include +#include +#include +#include +#include +#include "common/mali_kernel_common.h" +#include "mali_osk_types.h" +#include "mali_osk.h" + +u32 _mali_osk_bitmap_alloc(struct _mali_osk_bitmap *bitmap) +{ + u32 obj; + + MALI_DEBUG_ASSERT_POINTER(bitmap); + + _mali_osk_spinlock_lock(bitmap->lock); + + obj = find_next_zero_bit(bitmap->table, bitmap->max, bitmap->reserve); + + if (obj < bitmap->max) { + set_bit(obj, bitmap->table); + } else { + obj = -1; + } + + if (obj != -1) + --bitmap->avail; + _mali_osk_spinlock_unlock(bitmap->lock); + + return obj; +} + +void _mali_osk_bitmap_free(struct _mali_osk_bitmap *bitmap, u32 obj) +{ + MALI_DEBUG_ASSERT_POINTER(bitmap); + + _mali_osk_bitmap_free_range(bitmap, obj, 1); +} + +u32 _mali_osk_bitmap_alloc_range(struct _mali_osk_bitmap *bitmap, int cnt) +{ + u32 obj; + + MALI_DEBUG_ASSERT_POINTER(bitmap); + + if (0 >= cnt) { + return -1; + } + + if (1 == cnt) { + return _mali_osk_bitmap_alloc(bitmap); + } + + _mali_osk_spinlock_lock(bitmap->lock); + obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max, + bitmap->last, cnt, 0); + + if (obj >= bitmap->max) { + obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max, + bitmap->reserve, cnt, 0); + } + + if (obj < bitmap->max) { + bitmap_set(bitmap->table, obj, cnt); + + bitmap->last = (obj + cnt); + if (bitmap->last >= bitmap->max) { + bitmap->last = bitmap->reserve; + } + } else { + obj = -1; + } + + if (obj != -1) { + bitmap->avail -= cnt; + } + + _mali_osk_spinlock_unlock(bitmap->lock); + + return obj; +} + +u32 _mali_osk_bitmap_avail(struct _mali_osk_bitmap *bitmap) +{ + MALI_DEBUG_ASSERT_POINTER(bitmap); + + return bitmap->avail; +} + +void _mali_osk_bitmap_free_range(struct _mali_osk_bitmap *bitmap, u32 obj, int cnt) +{ + MALI_DEBUG_ASSERT_POINTER(bitmap); + + _mali_osk_spinlock_lock(bitmap->lock); + bitmap_clear(bitmap->table, obj, cnt); + bitmap->last = min(bitmap->last, obj); + + bitmap->avail += cnt; + _mali_osk_spinlock_unlock(bitmap->lock); +} + +int _mali_osk_bitmap_init(struct _mali_osk_bitmap *bitmap, u32 num, u32 reserve) +{ + MALI_DEBUG_ASSERT_POINTER(bitmap); + MALI_DEBUG_ASSERT(reserve <= num); + + bitmap->reserve = reserve; + bitmap->last = reserve; + bitmap->max = num; + bitmap->avail = num - reserve; + bitmap->lock = _mali_osk_spinlock_init(_MALI_OSK_LOCKFLAG_UNORDERED, _MALI_OSK_LOCK_ORDER_FIRST); + if (!bitmap->lock) { + return _MALI_OSK_ERR_NOMEM; + } + bitmap->table = kzalloc(BITS_TO_LONGS(bitmap->max) * + sizeof(long), GFP_KERNEL); + if (!bitmap->table) { + _mali_osk_spinlock_term(bitmap->lock); + return _MALI_OSK_ERR_NOMEM; + } + + return _MALI_OSK_ERR_OK; +} + +void _mali_osk_bitmap_term(struct _mali_osk_bitmap *bitmap) +{ + MALI_DEBUG_ASSERT_POINTER(bitmap); + + if (NULL != bitmap->lock) { + _mali_osk_spinlock_term(bitmap->lock); + } + + if (NULL != bitmap->table) { + kfree(bitmap->table); + } +} + diff --git a/drivers/gpu/arm/utgard/linux/mali_osk_irq.c b/drivers/gpu/arm/utgard/linux/mali_osk_irq.c new file mode 100644 index 00000000000000..539832d9125af8 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_osk_irq.c @@ -0,0 +1,200 @@ +/* + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_irq.c + * Implementation of the OS abstraction layer for the kernel device driver + */ + +#include /* For memory allocation */ +#include +#include +#include + +#include "mali_osk.h" +#include "mali_kernel_common.h" + +typedef struct _mali_osk_irq_t_struct { + u32 irqnum; + void *data; + _mali_osk_irq_uhandler_t uhandler; +} mali_osk_irq_object_t; + +typedef irqreturn_t (*irq_handler_func_t)(int, void *, struct pt_regs *); +static irqreturn_t irq_handler_upper_half(int port_name, void *dev_id); /* , struct pt_regs *regs*/ + +#if defined(DEBUG) + +struct test_interrupt_data { + _mali_osk_irq_ack_t ack_func; + void *probe_data; + mali_bool interrupt_received; + wait_queue_head_t wq; +}; + +static irqreturn_t test_interrupt_upper_half(int port_name, void *dev_id) +{ + irqreturn_t ret = IRQ_NONE; + struct test_interrupt_data *data = (struct test_interrupt_data *)dev_id; + + if (_MALI_OSK_ERR_OK == data->ack_func(data->probe_data)) { + data->interrupt_received = MALI_TRUE; + wake_up(&data->wq); + ret = IRQ_HANDLED; + } + + return ret; +} + +static _mali_osk_errcode_t test_interrupt(u32 irqnum, + _mali_osk_irq_trigger_t trigger_func, + _mali_osk_irq_ack_t ack_func, + void *probe_data, + const char *description) +{ + unsigned long irq_flags = 0; + struct test_interrupt_data data = { + .ack_func = ack_func, + .probe_data = probe_data, + .interrupt_received = MALI_FALSE, + }; + +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + irq_flags |= IRQF_SHARED; +#endif /* defined(CONFIG_MALI_SHARED_INTERRUPTS) */ + + if (0 != request_irq(irqnum, test_interrupt_upper_half, irq_flags, description, &data)) { + MALI_DEBUG_PRINT(2, ("Unable to install test IRQ handler for core '%s'\n", description)); + return _MALI_OSK_ERR_FAULT; + } + + init_waitqueue_head(&data.wq); + + trigger_func(probe_data); + wait_event_timeout(data.wq, data.interrupt_received, 100); + + free_irq(irqnum, &data); + + if (data.interrupt_received) { + MALI_DEBUG_PRINT(3, ("%s: Interrupt test OK\n", description)); + return _MALI_OSK_ERR_OK; + } else { + MALI_PRINT_ERROR(("%s: Failed interrupt test on %u\n", description, irqnum)); + return _MALI_OSK_ERR_FAULT; + } +} + +#endif /* defined(DEBUG) */ + +_mali_osk_irq_t *_mali_osk_irq_init(u32 irqnum, _mali_osk_irq_uhandler_t uhandler, void *int_data, _mali_osk_irq_trigger_t trigger_func, _mali_osk_irq_ack_t ack_func, void *probe_data, const char *description) +{ + mali_osk_irq_object_t *irq_object; + unsigned long irq_flags = 0; + +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + irq_flags |= IRQF_SHARED; +#endif /* defined(CONFIG_MALI_SHARED_INTERRUPTS) */ + + irq_object = kmalloc(sizeof(mali_osk_irq_object_t), GFP_KERNEL); + if (NULL == irq_object) { + return NULL; + } + + if (-1 == irqnum) { + /* Probe for IRQ */ + if ((NULL != trigger_func) && (NULL != ack_func)) { + unsigned long probe_count = 3; + _mali_osk_errcode_t err; + int irq; + + MALI_DEBUG_PRINT(2, ("Probing for irq\n")); + + do { + unsigned long mask; + + mask = probe_irq_on(); + trigger_func(probe_data); + + _mali_osk_time_ubusydelay(5); + + irq = probe_irq_off(mask); + err = ack_func(probe_data); + } while (irq < 0 && (err == _MALI_OSK_ERR_OK) && probe_count--); + + if (irq < 0 || (_MALI_OSK_ERR_OK != err)) irqnum = -1; + else irqnum = irq; + } else irqnum = -1; /* no probe functions, fault */ + + if (-1 != irqnum) { + /* found an irq */ + MALI_DEBUG_PRINT(2, ("Found irq %d\n", irqnum)); + } else { + MALI_DEBUG_PRINT(2, ("Probe for irq failed\n")); + } + } + + irq_object->irqnum = irqnum; + irq_object->uhandler = uhandler; + irq_object->data = int_data; + + if (-1 == irqnum) { + MALI_DEBUG_PRINT(2, ("No IRQ for core '%s' found during probe\n", description)); + kfree(irq_object); + return NULL; + } + +#if defined(DEBUG) + /* Verify that the configured interrupt settings are working */ + if (_MALI_OSK_ERR_OK != test_interrupt(irqnum, trigger_func, ack_func, probe_data, description)) { + MALI_DEBUG_PRINT(2, ("Test of IRQ(%d) handler for core '%s' failed\n", irqnum, description)); + kfree(irq_object); + return NULL; + } +#endif + + if (0 != request_irq(irqnum, irq_handler_upper_half, irq_flags, description, irq_object)) { + MALI_DEBUG_PRINT(2, ("Unable to install IRQ handler for core '%s'\n", description)); + kfree(irq_object); + return NULL; + } + + return irq_object; +} + +void _mali_osk_irq_term(_mali_osk_irq_t *irq) +{ + mali_osk_irq_object_t *irq_object = (mali_osk_irq_object_t *)irq; + free_irq(irq_object->irqnum, irq_object); + kfree(irq_object); +} + + +/** This function is called directly in interrupt context from the OS just after + * the CPU get the hw-irq from mali, or other devices on the same IRQ-channel. + * It is registered one of these function for each mali core. When an interrupt + * arrives this function will be called equal times as registered mali cores. + * That means that we only check one mali core in one function call, and the + * core we check for each turn is given by the \a dev_id variable. + * If we detect an pending interrupt on the given core, we mask the interrupt + * out by settging the core's IRQ_MASK register to zero. + * Then we schedule the mali_core_irq_handler_bottom_half to run as high priority + * work queue job. + */ +static irqreturn_t irq_handler_upper_half(int port_name, void *dev_id) /* , struct pt_regs *regs*/ +{ + irqreturn_t ret = IRQ_NONE; + mali_osk_irq_object_t *irq_object = (mali_osk_irq_object_t *)dev_id; + + if (_MALI_OSK_ERR_OK == irq_object->uhandler(irq_object->data)) { + ret = IRQ_HANDLED; + } + + return ret; +} diff --git a/drivers/gpu/arm/utgard/linux/mali_osk_locks.c b/drivers/gpu/arm/utgard/linux/mali_osk_locks.c new file mode 100644 index 00000000000000..50c0a9d238190f --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_osk_locks.c @@ -0,0 +1,287 @@ +/* + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_locks.c + * Implemenation of the OS abstraction layer for the kernel device driver + */ + +#include "mali_osk_locks.h" +#include "mali_kernel_common.h" +#include "mali_osk.h" + + +#ifdef DEBUG +#ifdef LOCK_ORDER_CHECKING +static DEFINE_SPINLOCK(lock_tracking_lock); +static mali_bool add_lock_to_log_and_check(struct _mali_osk_lock_debug_s *lock, uint32_t tid); +static void remove_lock_from_log(struct _mali_osk_lock_debug_s *lock, uint32_t tid); +static const char *const lock_order_to_string(_mali_osk_lock_order_t order); +#endif /* LOCK_ORDER_CHECKING */ + +void _mali_osk_locks_debug_init(struct _mali_osk_lock_debug_s *checker, _mali_osk_lock_flags_t flags, _mali_osk_lock_order_t order) +{ + checker->orig_flags = flags; + checker->owner = 0; + +#ifdef LOCK_ORDER_CHECKING + checker->order = order; + checker->next = NULL; +#endif +} + +void _mali_osk_locks_debug_add(struct _mali_osk_lock_debug_s *checker) +{ + checker->owner = _mali_osk_get_tid(); + +#ifdef LOCK_ORDER_CHECKING + if (!(checker->orig_flags & _MALI_OSK_LOCKFLAG_UNORDERED)) { + if (!add_lock_to_log_and_check(checker, _mali_osk_get_tid())) { + printk(KERN_ERR "%d: ERROR lock %p taken while holding a lock of a higher order.\n", + _mali_osk_get_tid(), checker); + dump_stack(); + } + } +#endif +} + +void _mali_osk_locks_debug_remove(struct _mali_osk_lock_debug_s *checker) +{ + +#ifdef LOCK_ORDER_CHECKING + if (!(checker->orig_flags & _MALI_OSK_LOCKFLAG_UNORDERED)) { + remove_lock_from_log(checker, _mali_osk_get_tid()); + } +#endif + checker->owner = 0; +} + + +#ifdef LOCK_ORDER_CHECKING +/* Lock order checking + * ------------------- + * + * To assure that lock ordering scheme defined by _mali_osk_lock_order_t is strictly adhered to, the + * following function will, together with a linked list and some extra members in _mali_osk_lock_debug_s, + * make sure that a lock that is taken has a higher order than the current highest-order lock a + * thread holds. + * + * This is done in the following manner: + * - A linked list keeps track of locks held by a thread. + * - A `next' pointer is added to each lock. This is used to chain the locks together. + * - When taking a lock, the `add_lock_to_log_and_check' makes sure that taking + * the given lock is legal. It will follow the linked list to find the last + * lock taken by this thread. If the last lock's order was lower than the + * lock that is to be taken, it appends the new lock to the list and returns + * true, if not, it return false. This return value is assert()'ed on in + * _mali_osk_lock_wait(). + */ + +static struct _mali_osk_lock_debug_s *lock_lookup_list; + +static void dump_lock_tracking_list(void) +{ + struct _mali_osk_lock_debug_s *l; + u32 n = 1; + + /* print list for debugging purposes */ + l = lock_lookup_list; + + while (NULL != l) { + printk(" [lock: %p, tid_owner: %d, order: %d] ->", l, l->owner, l->order); + l = l->next; + MALI_DEBUG_ASSERT(n++ < 100); + } + printk(" NULL\n"); +} + +static int tracking_list_length(void) +{ + struct _mali_osk_lock_debug_s *l; + u32 n = 0; + l = lock_lookup_list; + + while (NULL != l) { + l = l->next; + n++; + MALI_DEBUG_ASSERT(n < 100); + } + return n; +} + +static mali_bool add_lock_to_log_and_check(struct _mali_osk_lock_debug_s *lock, uint32_t tid) +{ + mali_bool ret = MALI_FALSE; + _mali_osk_lock_order_t highest_order_for_tid = _MALI_OSK_LOCK_ORDER_FIRST; + struct _mali_osk_lock_debug_s *highest_order_lock = (struct _mali_osk_lock_debug_s *)0xbeefbabe; + struct _mali_osk_lock_debug_s *l; + unsigned long local_lock_flag; + u32 len; + + spin_lock_irqsave(&lock_tracking_lock, local_lock_flag); + len = tracking_list_length(); + + l = lock_lookup_list; + if (NULL == l) { /* This is the first lock taken by this thread -- record and return true */ + lock_lookup_list = lock; + spin_unlock_irqrestore(&lock_tracking_lock, local_lock_flag); + return MALI_TRUE; + } else { + /* Traverse the locks taken and find the lock of the highest order. + * Since several threads may hold locks, each lock's owner must be + * checked so that locks not owned by this thread can be ignored. */ + for (;;) { + MALI_DEBUG_ASSERT_POINTER(l); + if (tid == l->owner && l->order >= highest_order_for_tid) { + highest_order_for_tid = l->order; + highest_order_lock = l; + } + + if (NULL != l->next) { + l = l->next; + } else { + break; + } + } + + l->next = lock; + l->next = NULL; + } + + /* We have now found the highest order lock currently held by this thread and can see if it is + * legal to take the requested lock. */ + ret = highest_order_for_tid < lock->order; + + if (!ret) { + printk(KERN_ERR "Took lock of order %d (%s) while holding lock of order %d (%s)\n", + lock->order, lock_order_to_string(lock->order), + highest_order_for_tid, lock_order_to_string(highest_order_for_tid)); + dump_lock_tracking_list(); + } + + if (len + 1 != tracking_list_length()) { + printk(KERN_ERR "************ lock: %p\n", lock); + printk(KERN_ERR "************ before: %d *** after: %d ****\n", len, tracking_list_length()); + dump_lock_tracking_list(); + MALI_DEBUG_ASSERT_POINTER(NULL); + } + + spin_unlock_irqrestore(&lock_tracking_lock, local_lock_flag); + return ret; +} + +static void remove_lock_from_log(struct _mali_osk_lock_debug_s *lock, uint32_t tid) +{ + struct _mali_osk_lock_debug_s *curr; + struct _mali_osk_lock_debug_s *prev = NULL; + unsigned long local_lock_flag; + u32 len; + u32 n = 0; + + spin_lock_irqsave(&lock_tracking_lock, local_lock_flag); + len = tracking_list_length(); + curr = lock_lookup_list; + + if (NULL == curr) { + printk(KERN_ERR "Error: Lock tracking list was empty on call to remove_lock_from_log\n"); + dump_lock_tracking_list(); + } + + MALI_DEBUG_ASSERT_POINTER(curr); + + + while (lock != curr) { + prev = curr; + + MALI_DEBUG_ASSERT_POINTER(curr); + curr = curr->next; + MALI_DEBUG_ASSERT(n++ < 100); + } + + if (NULL == prev) { + lock_lookup_list = curr->next; + } else { + MALI_DEBUG_ASSERT_POINTER(curr); + MALI_DEBUG_ASSERT_POINTER(prev); + prev->next = curr->next; + } + + lock->next = NULL; + + if (len - 1 != tracking_list_length()) { + printk(KERN_ERR "************ lock: %p\n", lock); + printk(KERN_ERR "************ before: %d *** after: %d ****\n", len, tracking_list_length()); + dump_lock_tracking_list(); + MALI_DEBUG_ASSERT_POINTER(NULL); + } + + spin_unlock_irqrestore(&lock_tracking_lock, local_lock_flag); +} + +static const char *const lock_order_to_string(_mali_osk_lock_order_t order) +{ + switch (order) { + case _MALI_OSK_LOCK_ORDER_SESSIONS: + return "_MALI_OSK_LOCK_ORDER_SESSIONS"; + break; + case _MALI_OSK_LOCK_ORDER_MEM_SESSION: + return "_MALI_OSK_LOCK_ORDER_MEM_SESSION"; + break; + case _MALI_OSK_LOCK_ORDER_MEM_INFO: + return "_MALI_OSK_LOCK_ORDER_MEM_INFO"; + break; + case _MALI_OSK_LOCK_ORDER_MEM_PT_CACHE: + return "_MALI_OSK_LOCK_ORDER_MEM_PT_CACHE"; + break; + case _MALI_OSK_LOCK_ORDER_DESCRIPTOR_MAP: + return "_MALI_OSK_LOCK_ORDER_DESCRIPTOR_MAP"; + break; + case _MALI_OSK_LOCK_ORDER_PM_EXECUTION: + return "_MALI_OSK_LOCK_ORDER_PM_EXECUTION"; + break; + case _MALI_OSK_LOCK_ORDER_EXECUTOR: + return "_MALI_OSK_LOCK_ORDER_EXECUTOR"; + break; + case _MALI_OSK_LOCK_ORDER_TIMELINE_SYSTEM: + return "_MALI_OSK_LOCK_ORDER_TIMELINE_SYSTEM"; + break; + case _MALI_OSK_LOCK_ORDER_SCHEDULER: + return "_MALI_OSK_LOCK_ORDER_SCHEDULER"; + break; + case _MALI_OSK_LOCK_ORDER_SCHEDULER_DEFERRED: + return "_MALI_OSK_LOCK_ORDER_SCHEDULER_DEFERRED"; + break; + case _MALI_OSK_LOCK_ORDER_DMA_COMMAND: + return "_MALI_OSK_LOCK_ORDER_DMA_COMMAND"; + break; + case _MALI_OSK_LOCK_ORDER_PROFILING: + return "_MALI_OSK_LOCK_ORDER_PROFILING"; + break; + case _MALI_OSK_LOCK_ORDER_L2: + return "_MALI_OSK_LOCK_ORDER_L2"; + break; + case _MALI_OSK_LOCK_ORDER_L2_COMMAND: + return "_MALI_OSK_LOCK_ORDER_L2_COMMAND"; + break; + case _MALI_OSK_LOCK_ORDER_UTILIZATION: + return "_MALI_OSK_LOCK_ORDER_UTILIZATION"; + break; + case _MALI_OSK_LOCK_ORDER_SESSION_PENDING_JOBS: + return "_MALI_OSK_LOCK_ORDER_SESSION_PENDING_JOBS"; + break; + case _MALI_OSK_LOCK_ORDER_PM_STATE: + return "_MALI_OSK_LOCK_ORDER_PM_STATE"; + break; + default: + return ""; + } +} +#endif /* LOCK_ORDER_CHECKING */ +#endif /* DEBUG */ diff --git a/drivers/gpu/arm/utgard/linux/mali_osk_locks.h b/drivers/gpu/arm/utgard/linux/mali_osk_locks.h new file mode 100644 index 00000000000000..aa32a81e7496bd --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_osk_locks.h @@ -0,0 +1,326 @@ +/* + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_locks.h + * Defines OS abstraction of lock and mutex + */ +#ifndef _MALI_OSK_LOCKS_H +#define _MALI_OSK_LOCKS_H + +#include +#include +#include + +#include + +#include "mali_osk_types.h" + +#ifdef _cplusplus +extern "C" { +#endif + + /* When DEBUG is enabled, this struct will be used to track owner, mode and order checking */ +#ifdef DEBUG + struct _mali_osk_lock_debug_s { + u32 owner; + _mali_osk_lock_flags_t orig_flags; + _mali_osk_lock_order_t order; + struct _mali_osk_lock_debug_s *next; + }; +#endif + + /* Anstraction of spinlock_t */ + struct _mali_osk_spinlock_s { +#ifdef DEBUG + struct _mali_osk_lock_debug_s checker; +#endif + spinlock_t spinlock; + }; + + /* Abstration of spinlock_t and lock flag which is used to store register's state before locking */ + struct _mali_osk_spinlock_irq_s { +#ifdef DEBUG + struct _mali_osk_lock_debug_s checker; +#endif + + spinlock_t spinlock; + unsigned long flags; + }; + + /* Abstraction of rw_semaphore in OS */ + struct _mali_osk_mutex_rw_s { +#ifdef DEBUG + struct _mali_osk_lock_debug_s checker; + _mali_osk_lock_mode_t mode; +#endif + + struct rw_semaphore rw_sema; + }; + + /* Mutex and mutex_interruptible functions share the same osk mutex struct */ + struct _mali_osk_mutex_s { +#ifdef DEBUG + struct _mali_osk_lock_debug_s checker; +#endif + struct mutex mutex; + }; + +#ifdef DEBUG + /** @brief _mali_osk_locks_debug_init/add/remove() functions are declared when DEBUG is enabled and + * defined in file mali_osk_locks.c. When LOCK_ORDER_CHECKING is enabled, calling these functions when we + * init/lock/unlock a lock/mutex, we could track lock order of a given tid. */ + void _mali_osk_locks_debug_init(struct _mali_osk_lock_debug_s *checker, _mali_osk_lock_flags_t flags, _mali_osk_lock_order_t order); + void _mali_osk_locks_debug_add(struct _mali_osk_lock_debug_s *checker); + void _mali_osk_locks_debug_remove(struct _mali_osk_lock_debug_s *checker); + + /** @brief This function can return a given lock's owner when DEBUG is enabled. */ + static inline u32 _mali_osk_lock_get_owner(struct _mali_osk_lock_debug_s *lock) + { + return lock->owner; + } +#else +#define _mali_osk_locks_debug_init(x, y, z) do {} while (0) +#define _mali_osk_locks_debug_add(x) do {} while (0) +#define _mali_osk_locks_debug_remove(x) do {} while (0) +#endif + + /** @brief Before use _mali_osk_spin_lock, init function should be used to allocate memory and initial spinlock*/ + static inline _mali_osk_spinlock_t *_mali_osk_spinlock_init(_mali_osk_lock_flags_t flags, _mali_osk_lock_order_t order) + { + _mali_osk_spinlock_t *lock = NULL; + + lock = kmalloc(sizeof(_mali_osk_spinlock_t), GFP_KERNEL); + if (NULL == lock) { + return NULL; + } + spin_lock_init(&lock->spinlock); + _mali_osk_locks_debug_init((struct _mali_osk_lock_debug_s *)lock, flags, order); + return lock; + } + + /** @brief Lock a spinlock */ + static inline void _mali_osk_spinlock_lock(_mali_osk_spinlock_t *lock) + { + BUG_ON(NULL == lock); + spin_lock(&lock->spinlock); + _mali_osk_locks_debug_add((struct _mali_osk_lock_debug_s *)lock); + } + + /** @brief Unlock a spinlock */ + static inline void _mali_osk_spinlock_unlock(_mali_osk_spinlock_t *lock) + { + BUG_ON(NULL == lock); + _mali_osk_locks_debug_remove((struct _mali_osk_lock_debug_s *)lock); + spin_unlock(&lock->spinlock); + } + + /** @brief Free a memory block which the argument lock pointed to and its type must be + * _mali_osk_spinlock_t *. */ + static inline void _mali_osk_spinlock_term(_mali_osk_spinlock_t *lock) + { + /* Parameter validation */ + BUG_ON(NULL == lock); + + /* Linux requires no explicit termination of spinlocks, semaphores, or rw_semaphores */ + kfree(lock); + } + + /** @brief Before _mali_osk_spinlock_irq_lock/unlock/term() is called, init function should be + * called to initial spinlock and flags in struct _mali_osk_spinlock_irq_t. */ + static inline _mali_osk_spinlock_irq_t *_mali_osk_spinlock_irq_init(_mali_osk_lock_flags_t flags, _mali_osk_lock_order_t order) + { + _mali_osk_spinlock_irq_t *lock = NULL; + lock = kmalloc(sizeof(_mali_osk_spinlock_irq_t), GFP_KERNEL); + + if (NULL == lock) { + return NULL; + } + + lock->flags = 0; + spin_lock_init(&lock->spinlock); + _mali_osk_locks_debug_init((struct _mali_osk_lock_debug_s *)lock, flags, order); + return lock; + } + + /** @brief Lock spinlock and save the register's state */ + static inline void _mali_osk_spinlock_irq_lock(_mali_osk_spinlock_irq_t *lock) + { + unsigned long tmp_flags; + + BUG_ON(NULL == lock); + spin_lock_irqsave(&lock->spinlock, tmp_flags); + lock->flags = tmp_flags; + _mali_osk_locks_debug_add((struct _mali_osk_lock_debug_s *)lock); + } + + /** @brief Unlock spinlock with saved register's state */ + static inline void _mali_osk_spinlock_irq_unlock(_mali_osk_spinlock_irq_t *lock) + { + BUG_ON(NULL == lock); + _mali_osk_locks_debug_remove((struct _mali_osk_lock_debug_s *)lock); + spin_unlock_irqrestore(&lock->spinlock, lock->flags); + } + + /** @brief Destroy a given memory block which lock pointed to, and the lock type must be + * _mali_osk_spinlock_irq_t *. */ + static inline void _mali_osk_spinlock_irq_term(_mali_osk_spinlock_irq_t *lock) + { + /* Parameter validation */ + BUG_ON(NULL == lock); + + /* Linux requires no explicit termination of spinlocks, semaphores, or rw_semaphores */ + kfree(lock); + } + + /** @brief Before _mali_osk_mutex_rw_wait/signal/term() is called, we should call + * _mali_osk_mutex_rw_init() to kmalloc a memory block and initial part of elements in it. */ + static inline _mali_osk_mutex_rw_t *_mali_osk_mutex_rw_init(_mali_osk_lock_flags_t flags, _mali_osk_lock_order_t order) + { + _mali_osk_mutex_rw_t *lock = NULL; + + lock = kmalloc(sizeof(_mali_osk_mutex_rw_t), GFP_KERNEL); + + if (NULL == lock) { + return NULL; + } + + init_rwsem(&lock->rw_sema); + _mali_osk_locks_debug_init((struct _mali_osk_lock_debug_s *)lock, flags, order); + return lock; + } + + /** @brief When call _mali_osk_mutex_rw_wait/signal() functions, the second argument mode + * should be assigned with value _MALI_OSK_LOCKMODE_RO or _MALI_OSK_LOCKMODE_RW */ + static inline void _mali_osk_mutex_rw_wait(_mali_osk_mutex_rw_t *lock, _mali_osk_lock_mode_t mode) + { + BUG_ON(NULL == lock); + BUG_ON(!(_MALI_OSK_LOCKMODE_RO == mode || _MALI_OSK_LOCKMODE_RW == mode)); + + if (mode == _MALI_OSK_LOCKMODE_RO) { + down_read(&lock->rw_sema); + } else { + down_write(&lock->rw_sema); + } + +#ifdef DEBUG + if (mode == _MALI_OSK_LOCKMODE_RW) { + lock->mode = mode; + } else { /* mode == _MALI_OSK_LOCKMODE_RO */ + lock->mode = mode; + } + _mali_osk_locks_debug_add((struct _mali_osk_lock_debug_s *)lock); +#endif + } + + /** @brief Up lock->rw_sema with up_read/write() accordinf argument mode's value. */ + static inline void _mali_osk_mutex_rw_signal(_mali_osk_mutex_rw_t *lock, _mali_osk_lock_mode_t mode) + { + BUG_ON(NULL == lock); + BUG_ON(!(_MALI_OSK_LOCKMODE_RO == mode || _MALI_OSK_LOCKMODE_RW == mode)); +#ifdef DEBUG + /* make sure the thread releasing the lock actually was the owner */ + if (mode == _MALI_OSK_LOCKMODE_RW) { + _mali_osk_locks_debug_remove((struct _mali_osk_lock_debug_s *)lock); + /* This lock now has no owner */ + lock->checker.owner = 0; + } +#endif + + if (mode == _MALI_OSK_LOCKMODE_RO) { + up_read(&lock->rw_sema); + } else { + up_write(&lock->rw_sema); + } + } + + /** @brief Free a given memory block which lock pointed to and its type must be + * _mali_sok_mutex_rw_t *. */ + static inline void _mali_osk_mutex_rw_term(_mali_osk_mutex_rw_t *lock) + { + /* Parameter validation */ + BUG_ON(NULL == lock); + + /* Linux requires no explicit termination of spinlocks, semaphores, or rw_semaphores */ + kfree(lock); + } + + /** @brief Mutex & mutex_interruptible share the same init and term function, because they have the + * same osk mutex struct, and the difference between them is which locking function they use */ + static inline _mali_osk_mutex_t *_mali_osk_mutex_init(_mali_osk_lock_flags_t flags, _mali_osk_lock_order_t order) + { + _mali_osk_mutex_t *lock = NULL; + + lock = kmalloc(sizeof(_mali_osk_mutex_t), GFP_KERNEL); + + if (NULL == lock) { + return NULL; + } + mutex_init(&lock->mutex); + + _mali_osk_locks_debug_init((struct _mali_osk_lock_debug_s *)lock, flags, order); + return lock; + } + + /** @brief Lock the lock->mutex with mutex_lock_interruptible function */ + static inline _mali_osk_errcode_t _mali_osk_mutex_wait_interruptible(_mali_osk_mutex_t *lock) + { + _mali_osk_errcode_t err = _MALI_OSK_ERR_OK; + + BUG_ON(NULL == lock); + + if (mutex_lock_interruptible(&lock->mutex)) { + printk(KERN_WARNING "Mali: Can not lock mutex\n"); + err = _MALI_OSK_ERR_RESTARTSYSCALL; + } + + _mali_osk_locks_debug_add((struct _mali_osk_lock_debug_s *)lock); + return err; + } + + /** @brief Unlock the lock->mutex which is locked with mutex_lock_interruptible() function. */ + static inline void _mali_osk_mutex_signal_interruptible(_mali_osk_mutex_t *lock) + { + BUG_ON(NULL == lock); + _mali_osk_locks_debug_remove((struct _mali_osk_lock_debug_s *)lock); + mutex_unlock(&lock->mutex); + } + + /** @brief Lock the lock->mutex just with mutex_lock() function which could not be interruptted. */ + static inline void _mali_osk_mutex_wait(_mali_osk_mutex_t *lock) + { + BUG_ON(NULL == lock); + mutex_lock(&lock->mutex); + _mali_osk_locks_debug_add((struct _mali_osk_lock_debug_s *)lock); + } + + /** @brief Unlock the lock->mutex which is locked with mutex_lock() function. */ + static inline void _mali_osk_mutex_signal(_mali_osk_mutex_t *lock) + { + BUG_ON(NULL == lock); + _mali_osk_locks_debug_remove((struct _mali_osk_lock_debug_s *)lock); + mutex_unlock(&lock->mutex); + } + + /** @brief Free a given memory block which lock point. */ + static inline void _mali_osk_mutex_term(_mali_osk_mutex_t *lock) + { + /* Parameter validation */ + BUG_ON(NULL == lock); + + /* Linux requires no explicit termination of spinlocks, semaphores, or rw_semaphores */ + kfree(lock); + } + +#ifdef _cplusplus +} +#endif + +#endif diff --git a/drivers/gpu/arm/utgard/linux/mali_osk_low_level_mem.c b/drivers/gpu/arm/utgard/linux/mali_osk_low_level_mem.c new file mode 100644 index 00000000000000..c14e872a7d6391 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_osk_low_level_mem.c @@ -0,0 +1,146 @@ +/* + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_low_level_mem.c + * Implementation of the OS abstraction layer for the kernel device driver + */ + +#include +#include +#include + +#include "mali_kernel_common.h" +#include "mali_osk.h" +#include "mali_ukk.h" + +void _mali_osk_mem_barrier(void) +{ + mb(); +} + +void _mali_osk_write_mem_barrier(void) +{ + wmb(); +} + +mali_io_address _mali_osk_mem_mapioregion(uintptr_t phys, u32 size, const char *description) +{ + return (mali_io_address)ioremap_nocache(phys, size); +} + +void _mali_osk_mem_unmapioregion(uintptr_t phys, u32 size, mali_io_address virt) +{ + iounmap((void *)virt); +} + +_mali_osk_errcode_t inline _mali_osk_mem_reqregion(uintptr_t phys, u32 size, const char *description) +{ +#if MALI_LICENSE_IS_GPL + return _MALI_OSK_ERR_OK; /* GPL driver gets the mem region for the resources registered automatically */ +#else + return ((NULL == request_mem_region(phys, size, description)) ? _MALI_OSK_ERR_NOMEM : _MALI_OSK_ERR_OK); +#endif +} + +void inline _mali_osk_mem_unreqregion(uintptr_t phys, u32 size) +{ +#if !MALI_LICENSE_IS_GPL + release_mem_region(phys, size); +#endif +} + +void inline _mali_osk_mem_iowrite32_relaxed(volatile mali_io_address addr, u32 offset, u32 val) +{ + __raw_writel(cpu_to_le32(val), ((u8 *)addr) + offset); +} + +u32 inline _mali_osk_mem_ioread32(volatile mali_io_address addr, u32 offset) +{ + return ioread32(((u8 *)addr) + offset); +} + +void inline _mali_osk_mem_iowrite32(volatile mali_io_address addr, u32 offset, u32 val) +{ + iowrite32(val, ((u8 *)addr) + offset); +} + +void _mali_osk_cache_flushall(void) +{ + /** @note Cached memory is not currently supported in this implementation */ +} + +void _mali_osk_cache_ensure_uncached_range_flushed(void *uncached_mapping, u32 offset, u32 size) +{ + _mali_osk_write_mem_barrier(); +} + +u32 _mali_osk_mem_write_safe(void __user *dest, const void __user *src, u32 size) +{ +#define MALI_MEM_SAFE_COPY_BLOCK_SIZE 4096 + u32 retval = 0; + void *temp_buf; + + temp_buf = kmalloc(MALI_MEM_SAFE_COPY_BLOCK_SIZE, GFP_KERNEL); + if (NULL != temp_buf) { + u32 bytes_left_to_copy = size; + u32 i; + for (i = 0; i < size; i += MALI_MEM_SAFE_COPY_BLOCK_SIZE) { + u32 size_to_copy; + u32 size_copied; + u32 bytes_left; + + if (bytes_left_to_copy > MALI_MEM_SAFE_COPY_BLOCK_SIZE) { + size_to_copy = MALI_MEM_SAFE_COPY_BLOCK_SIZE; + } else { + size_to_copy = bytes_left_to_copy; + } + + bytes_left = copy_from_user(temp_buf, ((char *)src) + i, size_to_copy); + size_copied = size_to_copy - bytes_left; + + bytes_left = copy_to_user(((char *)dest) + i, temp_buf, size_copied); + size_copied -= bytes_left; + + bytes_left_to_copy -= size_copied; + retval += size_copied; + + if (size_copied != size_to_copy) { + break; /* Early out, we was not able to copy this entire block */ + } + } + + kfree(temp_buf); + } + + return retval; +} + +_mali_osk_errcode_t _mali_ukk_mem_write_safe(_mali_uk_mem_write_safe_s *args) +{ + void __user *src; + void __user *dst; + struct mali_session_data *session; + + MALI_DEBUG_ASSERT_POINTER(args); + + session = (struct mali_session_data *)(uintptr_t)args->ctx; + + if (NULL == session) { + return _MALI_OSK_ERR_INVALID_ARGS; + } + + src = (void __user *)(uintptr_t)args->src; + dst = (void __user *)(uintptr_t)args->dest; + + /* Return number of bytes actually copied */ + args->size = _mali_osk_mem_write_safe(dst, src, args->size); + return _MALI_OSK_ERR_OK; +} diff --git a/drivers/gpu/arm/utgard/linux/mali_osk_mali.c b/drivers/gpu/arm/utgard/linux/mali_osk_mali.c new file mode 100644 index 00000000000000..a3749d8057b450 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_osk_mali.c @@ -0,0 +1,390 @@ +/* + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_mali.c + * Implementation of the OS abstraction layer which is specific for the Mali kernel device driver + */ +#include +#include +#include +#include +#include +#include + +#include "mali_osk_mali.h" +#include "mali_kernel_common.h" /* MALI_xxx macros */ +#include "mali_osk.h" /* kernel side OS functions */ +#include "mali_kernel_linux.h" + + +#if defined(CONFIG_MALI_DT) && !defined(CONFIG_MALI_PLAT_SPECIFIC_DT) + +#define MALI_OSK_INVALID_RESOURCE_ADDRESS 0xFFFFFFFF + +/** + * Define the max number of resource we could have. + */ +#define MALI_OSK_MAX_RESOURCE_NUMBER 27 + +/** + * Define the max number of resource with interrupts, and they are + * the first 20 elements in array mali_osk_resource_bank. + */ +#define MALI_OSK_RESOURCE_WITH_IRQ_NUMBER 20 + +/** + * pp core start and end location in mali_osk_resource_bank array. + */ +#define MALI_OSK_RESOURCE_PP_LOCATION_START 2 +#define MALI_OSK_RESOURCE_PP_LOCATION_END 17 + +/** + * L2 cache start and end location in mali_osk_resource_bank array. + */ +#define MALI_OSK_RESOURCE_L2_LOCATION_START 20 +#define MALI_OSK_RESOURCE_l2_LOCATION_END 22 + +/** + * DMA unit location. + */ +#define MALI_OSK_RESOURCE_DMA_LOCATION 26 + +static _mali_osk_resource_t mali_osk_resource_bank[MALI_OSK_MAX_RESOURCE_NUMBER] = { + {.description = "Mali_GP", .base = MALI_OFFSET_GP, .irq_name = "IRQGP",}, + {.description = "Mali_GP_MMU", .base = MALI_OFFSET_GP_MMU, .irq_name = "IRQGPMMU",}, + {.description = "Mali_PP0", .base = MALI_OFFSET_PP0, .irq_name = "IRQPP0",}, + {.description = "Mali_PP0_MMU", .base = MALI_OFFSET_PP0_MMU, .irq_name = "IRQPPMMU0",}, + {.description = "Mali_PP1", .base = MALI_OFFSET_PP1, .irq_name = "IRQPP1",}, + {.description = "Mali_PP1_MMU", .base = MALI_OFFSET_PP1_MMU, .irq_name = "IRQPPMMU1",}, + {.description = "Mali_PP2", .base = MALI_OFFSET_PP2, .irq_name = "IRQPP2",}, + {.description = "Mali_PP2_MMU", .base = MALI_OFFSET_PP2_MMU, .irq_name = "IRQPPMMU2",}, + {.description = "Mali_PP3", .base = MALI_OFFSET_PP3, .irq_name = "IRQPP3",}, + {.description = "Mali_PP3_MMU", .base = MALI_OFFSET_PP3_MMU, .irq_name = "IRQPPMMU3",}, + {.description = "Mali_PP4", .base = MALI_OFFSET_PP4, .irq_name = "IRQPP4",}, + {.description = "Mali_PP4_MMU", .base = MALI_OFFSET_PP4_MMU, .irq_name = "IRQPPMMU4",}, + {.description = "Mali_PP5", .base = MALI_OFFSET_PP5, .irq_name = "IRQPP5",}, + {.description = "Mali_PP5_MMU", .base = MALI_OFFSET_PP5_MMU, .irq_name = "IRQPPMMU5",}, + {.description = "Mali_PP6", .base = MALI_OFFSET_PP6, .irq_name = "IRQPP6",}, + {.description = "Mali_PP6_MMU", .base = MALI_OFFSET_PP6_MMU, .irq_name = "IRQPPMMU6",}, + {.description = "Mali_PP7", .base = MALI_OFFSET_PP7, .irq_name = "IRQPP7",}, + {.description = "Mali_PP7_MMU", .base = MALI_OFFSET_PP7_MMU, .irq_name = "IRQPPMMU",}, + {.description = "Mali_PP_Broadcast", .base = MALI_OFFSET_PP_BCAST, .irq_name = "IRQPP",}, + {.description = "Mali_PMU", .base = MALI_OFFSET_PMU, .irq_name = "IRQPMU",}, + {.description = "Mali_L2", .base = MALI_OFFSET_L2_RESOURCE0,}, + {.description = "Mali_L2", .base = MALI_OFFSET_L2_RESOURCE1,}, + {.description = "Mali_L2", .base = MALI_OFFSET_L2_RESOURCE2,}, + {.description = "Mali_PP_MMU_Broadcast", .base = MALI_OFFSET_PP_BCAST_MMU,}, + {.description = "Mali_Broadcast", .base = MALI_OFFSET_BCAST,}, + {.description = "Mali_DLBU", .base = MALI_OFFSET_DLBU,}, + {.description = "Mali_DMA", .base = MALI_OFFSET_DMA,}, +}; + +static int _mali_osk_get_compatible_name(const char **out_string) +{ + struct device_node *node = mali_platform_device->dev.of_node; + + MALI_DEBUG_ASSERT(NULL != node); + + return of_property_read_string(node, "compatible", out_string); +} + +_mali_osk_errcode_t _mali_osk_resource_initialize(void) +{ + mali_bool mali_is_450 = MALI_FALSE, mali_is_470 = MALI_FALSE; + int i, pp_core_num = 0, l2_core_num = 0; + struct resource *res; + const char *compatible_name = NULL; + + if (0 == _mali_osk_get_compatible_name(&compatible_name)) { + if (0 == strncmp(compatible_name, "arm,mali-450", strlen("arm,mali-450"))) { + mali_is_450 = MALI_TRUE; + MALI_DEBUG_PRINT(2, ("mali-450 device tree detected.")); + } else if (0 == strncmp(compatible_name, "arm,mali-470", strlen("arm,mali-470"))) { + mali_is_470 = MALI_TRUE; + MALI_DEBUG_PRINT(2, ("mali-470 device tree detected.")); + } + } + + for (i = 0; i < MALI_OSK_RESOURCE_WITH_IRQ_NUMBER; i++) { + res = platform_get_resource_byname(mali_platform_device, IORESOURCE_IRQ, mali_osk_resource_bank[i].irq_name); + if (res) { + mali_osk_resource_bank[i].irq = res->start; + } else { + mali_osk_resource_bank[i].base = MALI_OSK_INVALID_RESOURCE_ADDRESS; + } + } + + for (i = MALI_OSK_RESOURCE_PP_LOCATION_START; i <= MALI_OSK_RESOURCE_PP_LOCATION_END; i++) { + if (MALI_OSK_INVALID_RESOURCE_ADDRESS != mali_osk_resource_bank[i].base) { + pp_core_num++; + } + } + + /* We have to divide by 2, because we caculate twice for only one pp(pp_core and pp_mmu_core). */ + if (0 != pp_core_num % 2) { + MALI_DEBUG_PRINT(2, ("The value of pp core number isn't normal.")); + return _MALI_OSK_ERR_FAULT; + } + + pp_core_num /= 2; + + /** + * we can caculate the number of l2 cache core according the number of pp core number + * and device type(mali400/mali450/mali470). + */ + l2_core_num = 1; + if (mali_is_450) { + if (pp_core_num > 4) { + l2_core_num = 3; + } else if (pp_core_num <= 4) { + l2_core_num = 2; + } + } + + for (i = MALI_OSK_RESOURCE_l2_LOCATION_END; i > MALI_OSK_RESOURCE_L2_LOCATION_START + l2_core_num - 1; i--) { + mali_osk_resource_bank[i].base = MALI_OSK_INVALID_RESOURCE_ADDRESS; + } + + /* If device is not mali-450 type, we have to remove related resource from resource bank. */ + if (!(mali_is_450 || mali_is_470)) { + for (i = MALI_OSK_RESOURCE_l2_LOCATION_END + 1; i < MALI_OSK_MAX_RESOURCE_NUMBER; i++) { + mali_osk_resource_bank[i].base = MALI_OSK_INVALID_RESOURCE_ADDRESS; + } + } + + if (mali_is_470) + mali_osk_resource_bank[MALI_OSK_RESOURCE_DMA_LOCATION].base = MALI_OSK_INVALID_RESOURCE_ADDRESS; + + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t _mali_osk_resource_find(u32 addr, _mali_osk_resource_t *res) +{ + int i; + + if (NULL == mali_platform_device) { + return _MALI_OSK_ERR_ITEM_NOT_FOUND; + } + + /* Traverse all of resources in resources bank to find the matching one. */ + for (i = 0; i < MALI_OSK_MAX_RESOURCE_NUMBER; i++) { + if (mali_osk_resource_bank[i].base == addr) { + if (NULL != res) { + res->base = addr + _mali_osk_resource_base_address(); + res->description = mali_osk_resource_bank[i].description; + res->irq = mali_osk_resource_bank[i].irq; + } + return _MALI_OSK_ERR_OK; + } + } + + return _MALI_OSK_ERR_ITEM_NOT_FOUND; +} + +uintptr_t _mali_osk_resource_base_address(void) +{ + struct resource *reg_res = NULL; + uintptr_t ret = 0; + + reg_res = platform_get_resource(mali_platform_device, IORESOURCE_MEM, 0); + + if (NULL != reg_res) { + ret = reg_res->start; + } + + return ret; +} + +void _mali_osk_device_data_pmu_config_get(u16 *domain_config_array, int array_size) +{ + struct device_node *node = mali_platform_device->dev.of_node; + struct property *prop; + const __be32 *p; + int length = 0, i = 0; + u32 u; + + MALI_DEBUG_PRINT(2, ("Get pmu config from device tree configuration.\n")); + + MALI_DEBUG_ASSERT(NULL != node); + + if (!of_get_property(node, "pmu_domain_config", &length)) { + return; + } + + if (array_size != length / sizeof(u32)) { + MALI_PRINT_ERROR(("Wrong pmu domain config in device tree.")); + return; + } + + of_property_for_each_u32(node, "pmu_domain_config", prop, p, u) { + domain_config_array[i] = (u16)u; + i++; + } + + return; +} + +u32 _mali_osk_get_pmu_switch_delay(void) +{ + struct device_node *node = mali_platform_device->dev.of_node; + u32 switch_delay; + + MALI_DEBUG_ASSERT(NULL != node); + + if (0 == of_property_read_u32(node, "pmu_switch_delay", &switch_delay)) { + return switch_delay; + } else { + MALI_DEBUG_PRINT(2, ("Couldn't find pmu_switch_delay in device tree configuration.\n")); + } + + return 0; +} + +#else /* CONFIG_MALI_DT && !CONFIG_MALI_PLAT_SPECIFIC_DT */ + +_mali_osk_errcode_t _mali_osk_resource_find(u32 addr, _mali_osk_resource_t *res) +{ + int i; + uintptr_t phys_addr; + + if (NULL == mali_platform_device) { + /* Not connected to a device */ + return _MALI_OSK_ERR_ITEM_NOT_FOUND; + } + + phys_addr = addr + _mali_osk_resource_base_address(); + for (i = 0; i < mali_platform_device->num_resources; i++) { + if (IORESOURCE_MEM == resource_type(&(mali_platform_device->resource[i])) && + mali_platform_device->resource[i].start == phys_addr) { + if (NULL != res) { + res->base = phys_addr; + res->description = mali_platform_device->resource[i].name; + + /* Any (optional) IRQ resource belonging to this resource will follow */ + if ((i + 1) < mali_platform_device->num_resources && + IORESOURCE_IRQ == resource_type(&(mali_platform_device->resource[i + 1]))) { + res->irq = mali_platform_device->resource[i + 1].start; + } else { + res->irq = -1; + } + } + return _MALI_OSK_ERR_OK; + } + } + + return _MALI_OSK_ERR_ITEM_NOT_FOUND; +} + +uintptr_t _mali_osk_resource_base_address(void) +{ + uintptr_t lowest_addr = (uintptr_t)(0 - 1); + uintptr_t ret = 0; + + if (NULL != mali_platform_device) { + int i; + for (i = 0; i < mali_platform_device->num_resources; i++) { + if (mali_platform_device->resource[i].flags & IORESOURCE_MEM && + mali_platform_device->resource[i].start < lowest_addr) { + lowest_addr = mali_platform_device->resource[i].start; + ret = lowest_addr; + } + } + } + + return ret; +} + +void _mali_osk_device_data_pmu_config_get(u16 *domain_config_array, int array_size) +{ + _mali_osk_device_data data = { 0, }; + + MALI_DEBUG_PRINT(2, ("Get pmu config from platform device data.\n")); + if (_MALI_OSK_ERR_OK == _mali_osk_device_data_get(&data)) { + /* Copy the custom customer power domain config */ + _mali_osk_memcpy(domain_config_array, data.pmu_domain_config, sizeof(data.pmu_domain_config)); + } + + return; +} + +u32 _mali_osk_get_pmu_switch_delay(void) +{ + _mali_osk_errcode_t err; + _mali_osk_device_data data = { 0, }; + + err = _mali_osk_device_data_get(&data); + + if (_MALI_OSK_ERR_OK == err) { + return data.pmu_switch_delay; + } + + return 0; +} +#endif /* CONFIG_MALI_DT */ + +_mali_osk_errcode_t _mali_osk_device_data_get(_mali_osk_device_data *data) +{ + MALI_DEBUG_ASSERT_POINTER(data); + + if (NULL != mali_platform_device) { + struct mali_gpu_device_data *os_data = NULL; + + os_data = (struct mali_gpu_device_data *)mali_platform_device->dev.platform_data; + if (NULL != os_data) { + /* Copy data from OS dependant struct to Mali neutral struct (identical!) */ + BUILD_BUG_ON(sizeof(*os_data) != sizeof(*data)); + _mali_osk_memcpy(data, os_data, sizeof(*os_data)); + + return _MALI_OSK_ERR_OK; + } + } + + return _MALI_OSK_ERR_ITEM_NOT_FOUND; +} + +u32 _mali_osk_identify_gpu_resource(void) +{ + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI_OFFSET_L2_RESOURCE1, NULL)) + /* Mali 450 */ + return 0x450; + + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI_OFFSET_DLBU, NULL)) + /* Mali 470 */ + return 0x470; + + /* Mali 400 */ + return 0x400; +} + +mali_bool _mali_osk_shared_interrupts(void) +{ + u32 irqs[128]; + u32 i, j, irq, num_irqs_found = 0; + + MALI_DEBUG_ASSERT_POINTER(mali_platform_device); + MALI_DEBUG_ASSERT(128 >= mali_platform_device->num_resources); + + for (i = 0; i < mali_platform_device->num_resources; i++) { + if (IORESOURCE_IRQ & mali_platform_device->resource[i].flags) { + irq = mali_platform_device->resource[i].start; + + for (j = 0; j < num_irqs_found; ++j) { + if (irq == irqs[j]) { + return MALI_TRUE; + } + } + + irqs[num_irqs_found++] = irq; + } + } + + return MALI_FALSE; +} diff --git a/drivers/gpu/arm/utgard/linux/mali_osk_math.c b/drivers/gpu/arm/utgard/linux/mali_osk_math.c new file mode 100644 index 00000000000000..085ce76f7665f8 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_osk_math.c @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2010, 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_math.c + * Implementation of the OS abstraction layer for the kernel device driver + */ + +#include "mali_osk.h" +#include + +u32 _mali_osk_clz(u32 input) +{ + return 32 - fls(input); +} + +u32 _mali_osk_fls(u32 input) +{ + return fls(input); +} diff --git a/drivers/gpu/arm/utgard/linux/mali_osk_memory.c b/drivers/gpu/arm/utgard/linux/mali_osk_memory.c new file mode 100644 index 00000000000000..390e613e186da5 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_osk_memory.c @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2010-2011, 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_memory.c + * Implementation of the OS abstraction layer for the kernel device driver + */ + +#include "mali_osk.h" +#include +#include + +void inline *_mali_osk_calloc(u32 n, u32 size) +{ + return kcalloc(n, size, GFP_KERNEL); +} + +void inline *_mali_osk_malloc(u32 size) +{ + return kmalloc(size, GFP_KERNEL); +} + +void inline _mali_osk_free(void *ptr) +{ + kfree(ptr); +} + +void inline *_mali_osk_valloc(u32 size) +{ + return vmalloc(size); +} + +void inline _mali_osk_vfree(void *ptr) +{ + vfree(ptr); +} + +void inline *_mali_osk_memcpy(void *dst, const void *src, u32 len) +{ + return memcpy(dst, src, len); +} + +void inline *_mali_osk_memset(void *s, u32 c, u32 n) +{ + return memset(s, c, n); +} + +mali_bool _mali_osk_mem_check_allocated(u32 max_allocated) +{ + /* No need to prevent an out-of-memory dialogue appearing on Linux, + * so we always return MALI_TRUE. + */ + return MALI_TRUE; +} diff --git a/drivers/gpu/arm/utgard/linux/mali_osk_misc.c b/drivers/gpu/arm/utgard/linux/mali_osk_misc.c new file mode 100644 index 00000000000000..0a619e3fc27e24 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_osk_misc.c @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_misc.c + * Implementation of the OS abstraction layer for the kernel device driver + */ +#include +#include +#include +#include +#include +#include +#include "mali_osk.h" + +#if !defined(CONFIG_MALI_QUIET) +void _mali_osk_dbgmsg(const char *fmt, ...) +{ + va_list args; + va_start(args, fmt); + vprintk(fmt, args); + va_end(args); +} +#endif /* !defined(CONFIG_MALI_QUIET) */ + +u32 _mali_osk_snprintf(char *buf, u32 size, const char *fmt, ...) +{ + int res; + va_list args; + va_start(args, fmt); + + res = vscnprintf(buf, (size_t)size, fmt, args); + + va_end(args); + return res; +} + +void _mali_osk_ctxprintf(_mali_osk_print_ctx *print_ctx, const char *fmt, ...) +{ + va_list args; + char buf[512]; + + va_start(args, fmt); + vscnprintf(buf, 512, fmt, args); + seq_printf(print_ctx, buf); + va_end(args); +} + +void _mali_osk_abort(void) +{ + /* make a simple fault by dereferencing a NULL pointer */ + dump_stack(); + *(int *)0 = 0; +} + +void _mali_osk_break(void) +{ + _mali_osk_abort(); +} + +u32 _mali_osk_get_pid(void) +{ + /* Thread group ID is the process ID on Linux */ + return (u32)current->tgid; +} + +char *_mali_osk_get_comm(void) +{ + return (char *)current->comm; +} + + +u32 _mali_osk_get_tid(void) +{ + /* pid is actually identifying the thread on Linux */ + u32 tid = current->pid; + + /* If the pid is 0 the core was idle. Instead of returning 0 we return a special number + * identifying which core we are on. */ + if (0 == tid) { + tid = -(1 + raw_smp_processor_id()); + } + + return tid; +} diff --git a/drivers/gpu/arm/utgard/linux/mali_osk_notification.c b/drivers/gpu/arm/utgard/linux/mali_osk_notification.c new file mode 100644 index 00000000000000..e66fe83f35571d --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_osk_notification.c @@ -0,0 +1,182 @@ +/* + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_notification.c + * Implementation of the OS abstraction layer for the kernel device driver + */ + +#include "mali_osk.h" +#include "mali_kernel_common.h" + +#include +#include +#include + +/** + * Declaration of the notification queue object type + * Contains a linked list of notification pending delivery to user space. + * It also contains a wait queue of exclusive waiters blocked in the ioctl + * When a new notification is posted a single thread is resumed. + */ +struct _mali_osk_notification_queue_t_struct { + spinlock_t mutex; /**< Mutex protecting the list */ + wait_queue_head_t receive_queue; /**< Threads waiting for new entries to the queue */ + struct list_head head; /**< List of notifications waiting to be picked up */ +}; + +typedef struct _mali_osk_notification_wrapper_t_struct { + struct list_head list; /**< Internal linked list variable */ + _mali_osk_notification_t data; /**< Notification data */ +} _mali_osk_notification_wrapper_t; + +_mali_osk_notification_queue_t *_mali_osk_notification_queue_init(void) +{ + _mali_osk_notification_queue_t *result; + + result = (_mali_osk_notification_queue_t *)kmalloc(sizeof(_mali_osk_notification_queue_t), GFP_KERNEL); + if (NULL == result) return NULL; + + spin_lock_init(&result->mutex); + init_waitqueue_head(&result->receive_queue); + INIT_LIST_HEAD(&result->head); + + return result; +} + +_mali_osk_notification_t *_mali_osk_notification_create(u32 type, u32 size) +{ + /* OPT Recycling of notification objects */ + _mali_osk_notification_wrapper_t *notification; + + notification = (_mali_osk_notification_wrapper_t *)kmalloc(sizeof(_mali_osk_notification_wrapper_t) + size, + GFP_KERNEL | __GFP_HIGH | __GFP_REPEAT); + if (NULL == notification) { + MALI_DEBUG_PRINT(1, ("Failed to create a notification object\n")); + return NULL; + } + + /* Init the list */ + INIT_LIST_HEAD(¬ification->list); + + if (0 != size) { + notification->data.result_buffer = ((u8 *)notification) + sizeof(_mali_osk_notification_wrapper_t); + } else { + notification->data.result_buffer = NULL; + } + + /* set up the non-allocating fields */ + notification->data.notification_type = type; + notification->data.result_buffer_size = size; + + /* all ok */ + return &(notification->data); +} + +void _mali_osk_notification_delete(_mali_osk_notification_t *object) +{ + _mali_osk_notification_wrapper_t *notification; + MALI_DEBUG_ASSERT_POINTER(object); + + notification = container_of(object, _mali_osk_notification_wrapper_t, data); + + /* Free the container */ + kfree(notification); +} + +void _mali_osk_notification_queue_term(_mali_osk_notification_queue_t *queue) +{ + _mali_osk_notification_t *result; + MALI_DEBUG_ASSERT_POINTER(queue); + + while (_MALI_OSK_ERR_OK == _mali_osk_notification_queue_dequeue(queue, &result)) { + _mali_osk_notification_delete(result); + } + + /* not much to do, just free the memory */ + kfree(queue); +} +void _mali_osk_notification_queue_send(_mali_osk_notification_queue_t *queue, _mali_osk_notification_t *object) +{ +#if defined(MALI_UPPER_HALF_SCHEDULING) + unsigned long irq_flags; +#endif + + _mali_osk_notification_wrapper_t *notification; + MALI_DEBUG_ASSERT_POINTER(queue); + MALI_DEBUG_ASSERT_POINTER(object); + + notification = container_of(object, _mali_osk_notification_wrapper_t, data); + +#if defined(MALI_UPPER_HALF_SCHEDULING) + spin_lock_irqsave(&queue->mutex, irq_flags); +#else + spin_lock(&queue->mutex); +#endif + + list_add_tail(¬ification->list, &queue->head); + +#if defined(MALI_UPPER_HALF_SCHEDULING) + spin_unlock_irqrestore(&queue->mutex, irq_flags); +#else + spin_unlock(&queue->mutex); +#endif + + /* and wake up one possible exclusive waiter */ + wake_up(&queue->receive_queue); +} + +_mali_osk_errcode_t _mali_osk_notification_queue_dequeue(_mali_osk_notification_queue_t *queue, _mali_osk_notification_t **result) +{ +#if defined(MALI_UPPER_HALF_SCHEDULING) + unsigned long irq_flags; +#endif + + _mali_osk_errcode_t ret = _MALI_OSK_ERR_ITEM_NOT_FOUND; + _mali_osk_notification_wrapper_t *wrapper_object; + +#if defined(MALI_UPPER_HALF_SCHEDULING) + spin_lock_irqsave(&queue->mutex, irq_flags); +#else + spin_lock(&queue->mutex); +#endif + + if (!list_empty(&queue->head)) { + wrapper_object = list_entry(queue->head.next, _mali_osk_notification_wrapper_t, list); + *result = &(wrapper_object->data); + list_del_init(&wrapper_object->list); + ret = _MALI_OSK_ERR_OK; + } + +#if defined(MALI_UPPER_HALF_SCHEDULING) + spin_unlock_irqrestore(&queue->mutex, irq_flags); +#else + spin_unlock(&queue->mutex); +#endif + + return ret; +} + +_mali_osk_errcode_t _mali_osk_notification_queue_receive(_mali_osk_notification_queue_t *queue, _mali_osk_notification_t **result) +{ + /* check input */ + MALI_DEBUG_ASSERT_POINTER(queue); + MALI_DEBUG_ASSERT_POINTER(result); + + /* default result */ + *result = NULL; + + if (wait_event_interruptible(queue->receive_queue, + _MALI_OSK_ERR_OK == _mali_osk_notification_queue_dequeue(queue, result))) { + return _MALI_OSK_ERR_RESTARTSYSCALL; + } + + return _MALI_OSK_ERR_OK; /* all ok */ +} diff --git a/drivers/gpu/arm/utgard/linux/mali_osk_pm.c b/drivers/gpu/arm/utgard/linux/mali_osk_pm.c new file mode 100644 index 00000000000000..21180d33fe7571 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_osk_pm.c @@ -0,0 +1,83 @@ +/** + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_pm.c + * Implementation of the callback functions from common power management + */ + +#include + +#ifdef CONFIG_PM_RUNTIME +#include +#endif /* CONFIG_PM_RUNTIME */ +#include +#include +#include "mali_osk.h" +#include "mali_kernel_common.h" +#include "mali_kernel_linux.h" + +/* Can NOT run in atomic context */ +_mali_osk_errcode_t _mali_osk_pm_dev_ref_get_sync(void) +{ +#ifdef CONFIG_PM_RUNTIME + int err; + MALI_DEBUG_ASSERT_POINTER(mali_platform_device); + err = pm_runtime_get_sync(&(mali_platform_device->dev)); +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) + pm_runtime_mark_last_busy(&(mali_platform_device->dev)); +#endif + if (0 > err) { + MALI_PRINT_ERROR(("Mali OSK PM: pm_runtime_get_sync() returned error code %d\n", err)); + return _MALI_OSK_ERR_FAULT; + } +#endif + return _MALI_OSK_ERR_OK; +} + +/* Can run in atomic context */ +_mali_osk_errcode_t _mali_osk_pm_dev_ref_get_async(void) +{ +#ifdef CONFIG_PM_RUNTIME + int err; + MALI_DEBUG_ASSERT_POINTER(mali_platform_device); + err = pm_runtime_get(&(mali_platform_device->dev)); +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) + pm_runtime_mark_last_busy(&(mali_platform_device->dev)); +#endif + if (0 > err && -EINPROGRESS != err) { + MALI_PRINT_ERROR(("Mali OSK PM: pm_runtime_get() returned error code %d\n", err)); + return _MALI_OSK_ERR_FAULT; + } +#endif + return _MALI_OSK_ERR_OK; +} + + +/* Can run in atomic context */ +void _mali_osk_pm_dev_ref_put(void) +{ +#ifdef CONFIG_PM_RUNTIME + MALI_DEBUG_ASSERT_POINTER(mali_platform_device); +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) + pm_runtime_mark_last_busy(&(mali_platform_device->dev)); + pm_runtime_put_autosuspend(&(mali_platform_device->dev)); +#else + pm_runtime_put(&(mali_platform_device->dev)); +#endif +#endif +} + +void _mali_osk_pm_dev_barrier(void) +{ +#ifdef CONFIG_PM_RUNTIME + pm_runtime_barrier(&(mali_platform_device->dev)); +#endif +} diff --git a/drivers/gpu/arm/utgard/linux/mali_osk_profiling.c b/drivers/gpu/arm/utgard/linux/mali_osk_profiling.c new file mode 100644 index 00000000000000..cc09748e7316ad --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_osk_profiling.c @@ -0,0 +1,1272 @@ +/* + * Copyright (C) 2012-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#include +#include +#include +#include +#include +#include + +#include +#include "mali_kernel_common.h" +#include "mali_osk.h" +#include "mali_ukk.h" +#include "mali_uk_types.h" +#include "mali_osk_profiling.h" +#include "mali_linux_trace.h" +#include "mali_gp.h" +#include "mali_pp.h" +#include "mali_l2_cache.h" +#include "mali_user_settings_db.h" +#include "mali_executor.h" +#include "mali_memory_manager.h" + +#define MALI_PROFILING_STREAM_DATA_DEFAULT_SIZE 100 +#define MALI_PROFILING_STREAM_HOLD_TIME 1000000 /*1 ms */ + +#define MALI_PROFILING_STREAM_BUFFER_SIZE (1 << 12) +#define MALI_PROFILING_STREAM_BUFFER_NUM 100 + +/** + * Define the mali profiling stream struct. + */ +typedef struct mali_profiling_stream { + u8 data[MALI_PROFILING_STREAM_BUFFER_SIZE]; + u32 used_size; + struct list_head list; +} mali_profiling_stream; + +typedef struct mali_profiling_stream_list { + spinlock_t spin_lock; + struct list_head free_list; + struct list_head queue_list; +} mali_profiling_stream_list; + +static const char mali_name[] = "4xx"; +static const char utgard_setup_version[] = "ANNOTATE_SETUP 1\n"; + +static u32 profiling_sample_rate = 0; +static u32 first_sw_counter_index = 0; + +static mali_bool l2_cache_counter_if_enabled = MALI_FALSE; +static u32 num_counters_enabled = 0; +static u32 mem_counters_enabled = 0; + +static _mali_osk_atomic_t stream_fd_if_used; + +static wait_queue_head_t stream_fd_wait_queue; +static mali_profiling_counter *global_mali_profiling_counters = NULL; +static u32 num_global_mali_profiling_counters = 0; + +static mali_profiling_stream_list *global_mali_stream_list = NULL; +static mali_profiling_stream *mali_counter_stream = NULL; +static mali_profiling_stream *mali_core_activity_stream = NULL; +static u64 mali_core_activity_stream_dequeue_time = 0; +static spinlock_t mali_activity_lock; +static u32 mali_activity_cores_num = 0; +static struct hrtimer profiling_sampling_timer; + +const char *_mali_mem_counter_descriptions[] = _MALI_MEM_COUTNER_DESCRIPTIONS; +const char *_mali_special_counter_descriptions[] = _MALI_SPCIAL_COUNTER_DESCRIPTIONS; + +static u32 current_profiling_pid = 0; + +static void _mali_profiling_stream_list_destory(mali_profiling_stream_list *profiling_stream_list) +{ + mali_profiling_stream *profiling_stream, *tmp_profiling_stream; + MALI_DEBUG_ASSERT_POINTER(profiling_stream_list); + + list_for_each_entry_safe(profiling_stream, tmp_profiling_stream, &profiling_stream_list->free_list, list) { + list_del(&profiling_stream->list); + kfree(profiling_stream); + } + + list_for_each_entry_safe(profiling_stream, tmp_profiling_stream, &profiling_stream_list->queue_list, list) { + list_del(&profiling_stream->list); + kfree(profiling_stream); + } + + kfree(profiling_stream_list); +} + +static void _mali_profiling_global_stream_list_free(void) +{ + mali_profiling_stream *profiling_stream, *tmp_profiling_stream; + unsigned long irq_flags; + + MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list); + spin_lock_irqsave(&global_mali_stream_list->spin_lock, irq_flags); + list_for_each_entry_safe(profiling_stream, tmp_profiling_stream, &global_mali_stream_list->queue_list, list) { + profiling_stream->used_size = 0; + list_move(&profiling_stream->list, &global_mali_stream_list->free_list); + } + spin_unlock_irqrestore(&global_mali_stream_list->spin_lock, irq_flags); +} + +static _mali_osk_errcode_t _mali_profiling_global_stream_list_dequeue(struct list_head *stream_list, mali_profiling_stream **new_mali_profiling_stream) +{ + unsigned long irq_flags; + _mali_osk_errcode_t ret = _MALI_OSK_ERR_OK; + MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list); + MALI_DEBUG_ASSERT_POINTER(stream_list); + + spin_lock_irqsave(&global_mali_stream_list->spin_lock, irq_flags); + + if (!list_empty(stream_list)) { + *new_mali_profiling_stream = list_entry(stream_list->next, mali_profiling_stream, list); + list_del_init(&(*new_mali_profiling_stream)->list); + } else { + ret = _MALI_OSK_ERR_NOMEM; + } + + spin_unlock_irqrestore(&global_mali_stream_list->spin_lock, irq_flags); + + return ret; +} + +static void _mali_profiling_global_stream_list_queue(struct list_head *stream_list, mali_profiling_stream *current_mali_profiling_stream) +{ + unsigned long irq_flags; + MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list); + MALI_DEBUG_ASSERT_POINTER(stream_list); + + spin_lock_irqsave(&global_mali_stream_list->spin_lock, irq_flags); + list_add_tail(¤t_mali_profiling_stream->list, stream_list); + spin_unlock_irqrestore(&global_mali_stream_list->spin_lock, irq_flags); +} + +static mali_bool _mali_profiling_global_stream_queue_list_if_empty(void) +{ + MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list); + return list_empty(&global_mali_stream_list->queue_list); +} + +static u32 _mali_profiling_global_stream_queue_list_next_size(void) +{ + unsigned long irq_flags; + u32 size = 0; + MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list); + + spin_lock_irqsave(&global_mali_stream_list->spin_lock, irq_flags); + if (!list_empty(&global_mali_stream_list->queue_list)) { + mali_profiling_stream *next_mali_profiling_stream = + list_entry(global_mali_stream_list->queue_list.next, mali_profiling_stream, list); + size = next_mali_profiling_stream->used_size; + } + spin_unlock_irqrestore(&global_mali_stream_list->spin_lock, irq_flags); + return size; +} + +/* The mali profiling stream file operations functions. */ +static ssize_t _mali_profiling_stream_read( + struct file *filp, + char __user *buffer, + size_t size, + loff_t *f_pos); + +static unsigned int _mali_profiling_stream_poll(struct file *filp, poll_table *wait); + +static int _mali_profiling_stream_release(struct inode *inode, struct file *filp); + +/* The timeline stream file operations structure. */ +static const struct file_operations mali_profiling_stream_fops = { + .release = _mali_profiling_stream_release, + .read = _mali_profiling_stream_read, + .poll = _mali_profiling_stream_poll, +}; + +static ssize_t _mali_profiling_stream_read( + struct file *filp, + char __user *buffer, + size_t size, + loff_t *f_pos) +{ + u32 copy_len = 0; + mali_profiling_stream *current_mali_profiling_stream; + u32 used_size; + MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list); + + while (!_mali_profiling_global_stream_queue_list_if_empty()) { + used_size = _mali_profiling_global_stream_queue_list_next_size(); + if (used_size <= ((u32)size - copy_len)) { + current_mali_profiling_stream = NULL; + _mali_profiling_global_stream_list_dequeue(&global_mali_stream_list->queue_list, + ¤t_mali_profiling_stream); + MALI_DEBUG_ASSERT_POINTER(current_mali_profiling_stream); + if (copy_to_user(&buffer[copy_len], current_mali_profiling_stream->data, current_mali_profiling_stream->used_size)) { + current_mali_profiling_stream->used_size = 0; + _mali_profiling_global_stream_list_queue(&global_mali_stream_list->free_list, current_mali_profiling_stream); + return -EFAULT; + } + copy_len += current_mali_profiling_stream->used_size; + current_mali_profiling_stream->used_size = 0; + _mali_profiling_global_stream_list_queue(&global_mali_stream_list->free_list, current_mali_profiling_stream); + } else { + break; + } + } + return (ssize_t)copy_len; +} + +static unsigned int _mali_profiling_stream_poll(struct file *filp, poll_table *wait) +{ + poll_wait(filp, &stream_fd_wait_queue, wait); + if (!_mali_profiling_global_stream_queue_list_if_empty()) + return POLLIN; + return 0; +} + +static int _mali_profiling_stream_release(struct inode *inode, struct file *filp) +{ + _mali_osk_atomic_init(&stream_fd_if_used, 0); + return 0; +} + +/* The funs for control packet and stream data.*/ +static void _mali_profiling_set_packet_size(unsigned char *const buf, const u32 size) +{ + u32 i; + + for (i = 0; i < sizeof(size); ++i) + buf[i] = (size >> 8 * i) & 0xFF; +} + +static u32 _mali_profiling_get_packet_size(unsigned char *const buf) +{ + u32 i; + u32 size = 0; + for (i = 0; i < sizeof(size); ++i) + size |= (u32)buf[i] << 8 * i; + return size; +} + +static u32 _mali_profiling_read_packet_int(unsigned char *const buf, u32 *const pos, u32 const packet_size) +{ + u64 int_value = 0; + u8 shift = 0; + u8 byte_value = ~0; + + while ((byte_value & 0x80) != 0) { + MALI_DEBUG_ASSERT((*pos) < packet_size); + byte_value = buf[*pos]; + *pos += 1; + int_value |= (u32)(byte_value & 0x7f) << shift; + shift += 7; + } + + if (shift < 8 * sizeof(int_value) && (byte_value & 0x40) != 0) { + int_value |= -(1 << shift); + } + + return int_value; +} + +static u32 _mali_profiling_pack_int(u8 *const buf, u32 const buf_size, u32 const pos, s32 value) +{ + u32 add_bytes = 0; + int more = 1; + while (more) { + /* low order 7 bits of val */ + char byte_value = value & 0x7f; + value >>= 7; + + if ((value == 0 && (byte_value & 0x40) == 0) || (value == -1 && (byte_value & 0x40) != 0)) { + more = 0; + } else { + byte_value |= 0x80; + } + + MALI_DEBUG_ASSERT((pos + add_bytes) < buf_size); + buf[pos + add_bytes] = byte_value; + add_bytes++; + } + + return add_bytes; +} + +static int _mali_profiling_pack_long(uint8_t *const buf, u32 const buf_size, u32 const pos, s64 val) +{ + int add_bytes = 0; + int more = 1; + while (more) { + /* low order 7 bits of x */ + char byte_value = val & 0x7f; + val >>= 7; + + if ((val == 0 && (byte_value & 0x40) == 0) || (val == -1 && (byte_value & 0x40) != 0)) { + more = 0; + } else { + byte_value |= 0x80; + } + + MALI_DEBUG_ASSERT((pos + add_bytes) < buf_size); + buf[pos + add_bytes] = byte_value; + add_bytes++; + } + + return add_bytes; +} + +static void _mali_profiling_stream_add_counter(mali_profiling_stream *profiling_stream, s64 current_time, u32 key, u32 counter_value) +{ + u32 add_size = STREAM_HEADER_SIZE; + MALI_DEBUG_ASSERT_POINTER(profiling_stream); + MALI_DEBUG_ASSERT((profiling_stream->used_size) < MALI_PROFILING_STREAM_BUFFER_SIZE); + + profiling_stream->data[profiling_stream->used_size] = STREAM_HEADER_COUNTER_VALUE; + + add_size += _mali_profiling_pack_long(profiling_stream->data, MALI_PROFILING_STREAM_BUFFER_SIZE, + profiling_stream->used_size + add_size, current_time); + add_size += _mali_profiling_pack_int(profiling_stream->data, MALI_PROFILING_STREAM_BUFFER_SIZE, + profiling_stream->used_size + add_size, (s32)0); + add_size += _mali_profiling_pack_int(profiling_stream->data, MALI_PROFILING_STREAM_BUFFER_SIZE, + profiling_stream->used_size + add_size, (s32)key); + add_size += _mali_profiling_pack_int(profiling_stream->data, MALI_PROFILING_STREAM_BUFFER_SIZE, + profiling_stream->used_size + add_size, (s32)counter_value); + + _mali_profiling_set_packet_size(profiling_stream->data + profiling_stream->used_size + 1, + add_size - STREAM_HEADER_SIZE); + + profiling_stream->used_size += add_size; +} + +/* The callback function for sampling timer.*/ +static enum hrtimer_restart _mali_profiling_sampling_counters(struct hrtimer *timer) +{ + u32 counter_index; + s64 current_time; + MALI_DEBUG_ASSERT_POINTER(global_mali_profiling_counters); + MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list); + + MALI_DEBUG_ASSERT(NULL == mali_counter_stream); + if (_MALI_OSK_ERR_OK == _mali_profiling_global_stream_list_dequeue( + &global_mali_stream_list->free_list, &mali_counter_stream)) { + + MALI_DEBUG_ASSERT_POINTER(mali_counter_stream); + MALI_DEBUG_ASSERT(0 == mali_counter_stream->used_size); + + /* Capture l2 cache counter values if enabled */ + if (MALI_TRUE == l2_cache_counter_if_enabled) { + int i, j = 0; + _mali_profiling_l2_counter_values l2_counters_values; + _mali_profiling_get_l2_counters(&l2_counters_values); + + for (i = COUNTER_L2_0_C0; i <= COUNTER_L2_2_C1; i++) { + if (0 == (j % 2)) + _mali_osk_profiling_record_global_counters(i, l2_counters_values.cores[j / 2].value0); + else + _mali_osk_profiling_record_global_counters(i, l2_counters_values.cores[j / 2].value1); + j++; + } + } + + current_time = (s64)_mali_osk_boot_time_get_ns(); + + /* Add all enabled counter values into stream */ + for (counter_index = 0; counter_index < num_global_mali_profiling_counters; counter_index++) { + /* No need to sample these couners here. */ + if (global_mali_profiling_counters[counter_index].enabled) { + if ((global_mali_profiling_counters[counter_index].counter_id >= FIRST_MEM_COUNTER && + global_mali_profiling_counters[counter_index].counter_id <= LAST_MEM_COUNTER) + || (global_mali_profiling_counters[counter_index].counter_id == COUNTER_VP_ACTIVITY) + || (global_mali_profiling_counters[counter_index].counter_id == COUNTER_FP_ACTIVITY) + || (global_mali_profiling_counters[counter_index].counter_id == COUNTER_FILMSTRIP)) { + + continue; + } + + if (global_mali_profiling_counters[counter_index].counter_id >= COUNTER_L2_0_C0 && + global_mali_profiling_counters[counter_index].counter_id <= COUNTER_L2_2_C1) { + + u32 prev_val = global_mali_profiling_counters[counter_index].prev_counter_value; + + _mali_profiling_stream_add_counter(mali_counter_stream, current_time, global_mali_profiling_counters[counter_index].key, + global_mali_profiling_counters[counter_index].current_counter_value - prev_val); + + prev_val = global_mali_profiling_counters[counter_index].current_counter_value; + + global_mali_profiling_counters[counter_index].prev_counter_value = prev_val; + } else { + + if (global_mali_profiling_counters[counter_index].counter_id == COUNTER_TOTAL_ALLOC_PAGES) { + u32 total_alloc_mem = _mali_ukk_report_memory_usage(); + global_mali_profiling_counters[counter_index].current_counter_value = total_alloc_mem / _MALI_OSK_MALI_PAGE_SIZE; + } + _mali_profiling_stream_add_counter(mali_counter_stream, current_time, global_mali_profiling_counters[counter_index].key, + global_mali_profiling_counters[counter_index].current_counter_value); + if (global_mali_profiling_counters[counter_index].counter_id < FIRST_SPECIAL_COUNTER) + global_mali_profiling_counters[counter_index].current_counter_value = 0; + } + } + } + _mali_profiling_global_stream_list_queue(&global_mali_stream_list->queue_list, mali_counter_stream); + mali_counter_stream = NULL; + } else { + MALI_DEBUG_PRINT(1, ("Not enough mali profiling stream buffer!\n")); + } + + wake_up_interruptible(&stream_fd_wait_queue); + + /*Enable the sampling timer again*/ + if (0 != num_counters_enabled && 0 != profiling_sample_rate) { + hrtimer_forward_now(&profiling_sampling_timer, ns_to_ktime(profiling_sample_rate)); + return HRTIMER_RESTART; + } + return HRTIMER_NORESTART; +} + +static void _mali_profiling_sampling_core_activity_switch(int counter_id, int core, u32 activity, u32 pid) +{ + unsigned long irq_flags; + + spin_lock_irqsave(&mali_activity_lock, irq_flags); + if (activity == 0) + mali_activity_cores_num--; + else + mali_activity_cores_num++; + spin_unlock_irqrestore(&mali_activity_lock, irq_flags); + + if (NULL != global_mali_profiling_counters) { + int i ; + for (i = 0; i < num_global_mali_profiling_counters; i++) { + if (counter_id == global_mali_profiling_counters[i].counter_id && global_mali_profiling_counters[i].enabled) { + u64 current_time = _mali_osk_boot_time_get_ns(); + u32 add_size = STREAM_HEADER_SIZE; + + if (NULL != mali_core_activity_stream) { + if ((mali_core_activity_stream_dequeue_time + MALI_PROFILING_STREAM_HOLD_TIME < current_time) || + (MALI_PROFILING_STREAM_DATA_DEFAULT_SIZE > MALI_PROFILING_STREAM_BUFFER_SIZE + - mali_core_activity_stream->used_size)) { + _mali_profiling_global_stream_list_queue(&global_mali_stream_list->queue_list, mali_core_activity_stream); + mali_core_activity_stream = NULL; + wake_up_interruptible(&stream_fd_wait_queue); + } + } + + if (NULL == mali_core_activity_stream) { + if (_MALI_OSK_ERR_OK == _mali_profiling_global_stream_list_dequeue( + &global_mali_stream_list->free_list, &mali_core_activity_stream)) { + mali_core_activity_stream_dequeue_time = current_time; + } else { + MALI_DEBUG_PRINT(1, ("Not enough mali profiling stream buffer!\n")); + wake_up_interruptible(&stream_fd_wait_queue); + break; + } + + } + + mali_core_activity_stream->data[mali_core_activity_stream->used_size] = STREAM_HEADER_CORE_ACTIVITY; + + add_size += _mali_profiling_pack_long(mali_core_activity_stream->data, + MALI_PROFILING_STREAM_BUFFER_SIZE, mali_core_activity_stream->used_size + add_size, (s64)current_time); + add_size += _mali_profiling_pack_int(mali_core_activity_stream->data, + MALI_PROFILING_STREAM_BUFFER_SIZE, mali_core_activity_stream->used_size + add_size, core); + add_size += _mali_profiling_pack_int(mali_core_activity_stream->data, + MALI_PROFILING_STREAM_BUFFER_SIZE, mali_core_activity_stream->used_size + add_size, (s32)global_mali_profiling_counters[i].key); + add_size += _mali_profiling_pack_int(mali_core_activity_stream->data, + MALI_PROFILING_STREAM_BUFFER_SIZE, mali_core_activity_stream->used_size + add_size, activity); + add_size += _mali_profiling_pack_int(mali_core_activity_stream->data, + MALI_PROFILING_STREAM_BUFFER_SIZE, mali_core_activity_stream->used_size + add_size, pid); + + _mali_profiling_set_packet_size(mali_core_activity_stream->data + mali_core_activity_stream->used_size + 1, + add_size - STREAM_HEADER_SIZE); + + mali_core_activity_stream->used_size += add_size; + + if (0 == mali_activity_cores_num) { + _mali_profiling_global_stream_list_queue(&global_mali_stream_list->queue_list, mali_core_activity_stream); + mali_core_activity_stream = NULL; + wake_up_interruptible(&stream_fd_wait_queue); + } + + break; + } + } + } +} + +static mali_bool _mali_profiling_global_counters_init(void) +{ + int core_id, counter_index, counter_number, counter_id; + u32 num_l2_cache_cores; + u32 num_pp_cores; + u32 num_gp_cores = 1; + + MALI_DEBUG_ASSERT(NULL == global_mali_profiling_counters); + num_pp_cores = mali_pp_get_glob_num_pp_cores(); + num_l2_cache_cores = mali_l2_cache_core_get_glob_num_l2_cores(); + + num_global_mali_profiling_counters = 3 * (num_gp_cores + num_pp_cores) + 2 * num_l2_cache_cores + + MALI_PROFILING_SW_COUNTERS_NUM + + MALI_PROFILING_SPECIAL_COUNTERS_NUM + + MALI_PROFILING_MEM_COUNTERS_NUM; + global_mali_profiling_counters = _mali_osk_calloc(num_global_mali_profiling_counters, sizeof(mali_profiling_counter)); + + if (NULL == global_mali_profiling_counters) + return MALI_FALSE; + + counter_index = 0; + /*Vertex processor counters */ + for (core_id = 0; core_id < num_gp_cores; core_id ++) { + global_mali_profiling_counters[counter_index].counter_id = ACTIVITY_VP_0 + core_id; + _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, + sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_VP_%d_active", mali_name, core_id); + + for (counter_number = 0; counter_number < 2; counter_number++) { + counter_index++; + global_mali_profiling_counters[counter_index].counter_id = COUNTER_VP_0_C0 + (2 * core_id) + counter_number; + _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, + sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_VP_%d_cnt%d", mali_name, core_id, counter_number); + } + } + + /* Fragment processors' counters */ + for (core_id = 0; core_id < num_pp_cores; core_id++) { + counter_index++; + global_mali_profiling_counters[counter_index].counter_id = ACTIVITY_FP_0 + core_id; + _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, + sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_FP_%d_active", mali_name, core_id); + + for (counter_number = 0; counter_number < 2; counter_number++) { + counter_index++; + global_mali_profiling_counters[counter_index].counter_id = COUNTER_FP_0_C0 + (2 * core_id) + counter_number; + _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, + sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_FP_%d_cnt%d", mali_name, core_id, counter_number); + } + } + + /* L2 Cache counters */ + for (core_id = 0; core_id < num_l2_cache_cores; core_id++) { + for (counter_number = 0; counter_number < 2; counter_number++) { + counter_index++; + global_mali_profiling_counters[counter_index].counter_id = COUNTER_L2_0_C0 + (2 * core_id) + counter_number; + _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, + sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_L2_%d_cnt%d", mali_name, core_id, counter_number); + } + } + + /* Now set up the software counter entries */ + for (counter_id = FIRST_SW_COUNTER; counter_id <= LAST_SW_COUNTER; counter_id++) { + counter_index++; + + if (0 == first_sw_counter_index) + first_sw_counter_index = counter_index; + + global_mali_profiling_counters[counter_index].counter_id = counter_id; + _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, + sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_SW_%d", mali_name, counter_id - FIRST_SW_COUNTER); + } + + /* Now set up the special counter entries */ + for (counter_id = FIRST_SPECIAL_COUNTER; counter_id <= LAST_SPECIAL_COUNTER; counter_id++) { + + counter_index++; + _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, + sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_%s", + mali_name, _mali_special_counter_descriptions[counter_id - FIRST_SPECIAL_COUNTER]); + + global_mali_profiling_counters[counter_index].counter_id = counter_id; + } + + /* Now set up the mem counter entries*/ + for (counter_id = FIRST_MEM_COUNTER; counter_id <= LAST_MEM_COUNTER; counter_id++) { + + counter_index++; + _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, + sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_%s", + mali_name, _mali_mem_counter_descriptions[counter_id - FIRST_MEM_COUNTER]); + + global_mali_profiling_counters[counter_index].counter_id = counter_id; + } + + MALI_DEBUG_ASSERT((counter_index + 1) == num_global_mali_profiling_counters); + + return MALI_TRUE; +} + +void _mali_profiling_notification_mem_counter(struct mali_session_data *session, u32 counter_id, u32 key, int enable) +{ + + MALI_DEBUG_ASSERT_POINTER(session); + + if (NULL != session) { + _mali_osk_notification_t *notification; + _mali_osk_notification_queue_t *queue; + + queue = session->ioctl_queue; + MALI_DEBUG_ASSERT(NULL != queue); + + notification = _mali_osk_notification_create(_MALI_NOTIFICATION_ANNOTATE_PROFILING_MEM_COUNTER, + sizeof(_mali_uk_annotate_profiling_mem_counter_s)); + + if (NULL != notification) { + _mali_uk_annotate_profiling_mem_counter_s *data = notification->result_buffer; + data->counter_id = counter_id; + data->key = key; + data->enable = enable; + + _mali_osk_notification_queue_send(queue, notification); + } else { + MALI_PRINT_ERROR(("Failed to create notification object!\n")); + } + } else { + MALI_PRINT_ERROR(("Failed to find the right session!\n")); + } +} + +void _mali_profiling_notification_enable(struct mali_session_data *session, u32 sampling_rate, int enable) +{ + MALI_DEBUG_ASSERT_POINTER(session); + + if (NULL != session) { + _mali_osk_notification_t *notification; + _mali_osk_notification_queue_t *queue; + + queue = session->ioctl_queue; + MALI_DEBUG_ASSERT(NULL != queue); + + notification = _mali_osk_notification_create(_MALI_NOTIFICATION_ANNOTATE_PROFILING_ENABLE, + sizeof(_mali_uk_annotate_profiling_enable_s)); + + if (NULL != notification) { + _mali_uk_annotate_profiling_enable_s *data = notification->result_buffer; + data->sampling_rate = sampling_rate; + data->enable = enable; + + _mali_osk_notification_queue_send(queue, notification); + } else { + MALI_PRINT_ERROR(("Failed to create notification object!\n")); + } + } else { + MALI_PRINT_ERROR(("Failed to find the right session!\n")); + } +} + + +_mali_osk_errcode_t _mali_osk_profiling_init(mali_bool auto_start) +{ + int i; + mali_profiling_stream *new_mali_profiling_stream = NULL; + mali_profiling_stream_list *new_mali_profiling_stream_list = NULL; + if (MALI_TRUE == auto_start) { + mali_set_user_setting(_MALI_UK_USER_SETTING_SW_EVENTS_ENABLE, MALI_TRUE); + } + + /*Init the global_mali_stream_list*/ + MALI_DEBUG_ASSERT(NULL == global_mali_stream_list); + new_mali_profiling_stream_list = (mali_profiling_stream_list *)kmalloc(sizeof(mali_profiling_stream_list), GFP_KERNEL); + + if (NULL == new_mali_profiling_stream_list) { + return _MALI_OSK_ERR_NOMEM; + } + + spin_lock_init(&new_mali_profiling_stream_list->spin_lock); + INIT_LIST_HEAD(&new_mali_profiling_stream_list->free_list); + INIT_LIST_HEAD(&new_mali_profiling_stream_list->queue_list); + + spin_lock_init(&mali_activity_lock); + mali_activity_cores_num = 0; + + for (i = 0; i < MALI_PROFILING_STREAM_BUFFER_NUM; i++) { + new_mali_profiling_stream = (mali_profiling_stream *)kmalloc(sizeof(mali_profiling_stream), GFP_KERNEL); + if (NULL == new_mali_profiling_stream) { + _mali_profiling_stream_list_destory(new_mali_profiling_stream_list); + return _MALI_OSK_ERR_NOMEM; + } + + INIT_LIST_HEAD(&new_mali_profiling_stream->list); + new_mali_profiling_stream->used_size = 0; + list_add_tail(&new_mali_profiling_stream->list, &new_mali_profiling_stream_list->free_list); + + } + + _mali_osk_atomic_init(&stream_fd_if_used, 0); + init_waitqueue_head(&stream_fd_wait_queue); + + hrtimer_init(&profiling_sampling_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + + profiling_sampling_timer.function = _mali_profiling_sampling_counters; + + global_mali_stream_list = new_mali_profiling_stream_list; + + return _MALI_OSK_ERR_OK; +} + +void _mali_osk_profiling_term(void) +{ + if (0 != profiling_sample_rate) { + hrtimer_cancel(&profiling_sampling_timer); + profiling_sample_rate = 0; + } + _mali_osk_atomic_term(&stream_fd_if_used); + + if (NULL != global_mali_profiling_counters) { + _mali_osk_free(global_mali_profiling_counters); + global_mali_profiling_counters = NULL; + num_global_mali_profiling_counters = 0; + } + + if (NULL != global_mali_stream_list) { + _mali_profiling_stream_list_destory(global_mali_stream_list); + global_mali_stream_list = NULL; + } + +} + +void _mali_osk_profiling_stop_sampling(u32 pid) +{ + if (pid == current_profiling_pid) { + + int i; + /* Reset all counter states when closing connection.*/ + for (i = 0; i < num_global_mali_profiling_counters; ++i) { + _mali_profiling_set_event(global_mali_profiling_counters[i].counter_id, MALI_HW_CORE_NO_COUNTER); + global_mali_profiling_counters[i].enabled = 0; + global_mali_profiling_counters[i].prev_counter_value = 0; + global_mali_profiling_counters[i].current_counter_value = 0; + } + l2_cache_counter_if_enabled = MALI_FALSE; + num_counters_enabled = 0; + mem_counters_enabled = 0; + _mali_profiling_control(FBDUMP_CONTROL_ENABLE, 0); + _mali_profiling_control(SW_COUNTER_ENABLE, 0); + /* Delete sampling timer when closing connection. */ + if (0 != profiling_sample_rate) { + hrtimer_cancel(&profiling_sampling_timer); + profiling_sample_rate = 0; + } + current_profiling_pid = 0; + } +} + +void _mali_osk_profiling_add_event(u32 event_id, u32 data0, u32 data1, u32 data2, u32 data3, u32 data4) +{ + /*Record the freq & volt to global_mali_profiling_counters here. */ + if (0 != profiling_sample_rate) { + u32 channel; + u32 state; + channel = (event_id >> 16) & 0xFF; + state = ((event_id >> 24) & 0xF) << 24; + + switch (state) { + case MALI_PROFILING_EVENT_TYPE_SINGLE: + if ((MALI_PROFILING_EVENT_CHANNEL_GPU >> 16) == channel) { + u32 reason = (event_id & 0xFFFF); + if (MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE == reason) { + _mali_osk_profiling_record_global_counters(COUNTER_FREQUENCY, data0); + _mali_osk_profiling_record_global_counters(COUNTER_VOLTAGE, data1); + } + } + break; + case MALI_PROFILING_EVENT_TYPE_START: + if ((MALI_PROFILING_EVENT_CHANNEL_GP0 >> 16) == channel) { + _mali_profiling_sampling_core_activity_switch(COUNTER_VP_ACTIVITY, 0, 1, data1); + } else if (channel >= (MALI_PROFILING_EVENT_CHANNEL_PP0 >> 16) && + (MALI_PROFILING_EVENT_CHANNEL_PP7 >> 16) >= channel) { + u32 core_id = channel - (MALI_PROFILING_EVENT_CHANNEL_PP0 >> 16); + _mali_profiling_sampling_core_activity_switch(COUNTER_FP_ACTIVITY, core_id, 1, data1); + } + break; + case MALI_PROFILING_EVENT_TYPE_STOP: + if ((MALI_PROFILING_EVENT_CHANNEL_GP0 >> 16) == channel) { + _mali_profiling_sampling_core_activity_switch(COUNTER_VP_ACTIVITY, 0, 0, 0); + } else if (channel >= (MALI_PROFILING_EVENT_CHANNEL_PP0 >> 16) && + (MALI_PROFILING_EVENT_CHANNEL_PP7 >> 16) >= channel) { + u32 core_id = channel - (MALI_PROFILING_EVENT_CHANNEL_PP0 >> 16); + _mali_profiling_sampling_core_activity_switch(COUNTER_FP_ACTIVITY, core_id, 0, 0); + } + break; + default: + break; + } + } + trace_mali_timeline_event(event_id, data0, data1, data2, data3, data4); +} + +void _mali_osk_profiling_report_sw_counters(u32 *counters) +{ + trace_mali_sw_counters(_mali_osk_get_pid(), _mali_osk_get_tid(), NULL, counters); +} + +void _mali_osk_profiling_record_global_counters(int counter_id, u32 value) +{ + if (NULL != global_mali_profiling_counters) { + int i ; + for (i = 0; i < num_global_mali_profiling_counters; i++) { + if (counter_id == global_mali_profiling_counters[i].counter_id && global_mali_profiling_counters[i].enabled) { + global_mali_profiling_counters[i].current_counter_value = value; + break; + } + } + } +} + +_mali_osk_errcode_t _mali_ukk_profiling_add_event(_mali_uk_profiling_add_event_s *args) +{ + /* Always add process and thread identificator in the first two data elements for events from user space */ + _mali_osk_profiling_add_event(args->event_id, _mali_osk_get_pid(), _mali_osk_get_tid(), args->data[2], args->data[3], args->data[4]); + + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t _mali_ukk_sw_counters_report(_mali_uk_sw_counters_report_s *args) +{ + u32 *counters = (u32 *)(uintptr_t)args->counters; + + _mali_osk_profiling_report_sw_counters(counters); + + if (NULL != global_mali_profiling_counters) { + int i; + for (i = 0; i < MALI_PROFILING_SW_COUNTERS_NUM; i ++) { + if (global_mali_profiling_counters[first_sw_counter_index + i].enabled) { + global_mali_profiling_counters[first_sw_counter_index + i].current_counter_value = *(counters + i); + } + } + } + + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t _mali_ukk_profiling_stream_fd_get(_mali_uk_profiling_stream_fd_get_s *args) +{ + struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; + MALI_DEBUG_ASSERT_POINTER(session); + + if (1 == _mali_osk_atomic_inc_return(&stream_fd_if_used)) { + + s32 fd = anon_inode_getfd("[mali_profiling_stream]", &mali_profiling_stream_fops, + session, + O_RDONLY | O_CLOEXEC); + + args->stream_fd = fd; + if (0 > fd) { + _mali_osk_atomic_dec(&stream_fd_if_used); + return _MALI_OSK_ERR_FAULT; + } + args->stream_fd = fd; + } else { + _mali_osk_atomic_dec(&stream_fd_if_used); + args->stream_fd = -1; + return _MALI_OSK_ERR_BUSY; + } + + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t _mali_ukk_profiling_control_set(_mali_uk_profiling_control_set_s *args) +{ + u32 control_packet_size; + u32 output_buffer_size; + + struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; + MALI_DEBUG_ASSERT_POINTER(session); + + if (NULL == global_mali_profiling_counters && MALI_FALSE == _mali_profiling_global_counters_init()) { + MALI_PRINT_ERROR(("Failed to create global_mali_profiling_counters.\n")); + return _MALI_OSK_ERR_FAULT; + } + + control_packet_size = args->control_packet_size; + output_buffer_size = args->response_packet_size; + + if (0 != control_packet_size) { + u8 control_type; + u8 *control_packet_data; + u8 *response_packet_data; + u32 version_length = sizeof(utgard_setup_version) - 1; + + control_packet_data = (u8 *)(uintptr_t)args->control_packet_data; + MALI_DEBUG_ASSERT_POINTER(control_packet_data); + response_packet_data = (u8 *)(uintptr_t)args->response_packet_data; + MALI_DEBUG_ASSERT_POINTER(response_packet_data); + + /*Decide if need to ignore Utgard setup version.*/ + if (control_packet_size >= version_length) { + if (0 == memcmp(control_packet_data, utgard_setup_version, version_length)) { + if (control_packet_size == version_length) { + args->response_packet_size = 0; + return _MALI_OSK_ERR_OK; + } else { + control_packet_data += version_length; + control_packet_size -= version_length; + } + } + } + + current_profiling_pid = _mali_osk_get_pid(); + + control_type = control_packet_data[0]; + switch (control_type) { + case PACKET_HEADER_COUNTERS_REQUEST: { + int i; + + if (PACKET_HEADER_SIZE > control_packet_size || + control_packet_size != _mali_profiling_get_packet_size(control_packet_data + 1)) { + MALI_PRINT_ERROR(("Wrong control packet size, type 0x%x,size 0x%x.\n", control_packet_data[0], control_packet_size)); + return _MALI_OSK_ERR_FAULT; + } + + /* Send supported counters */ + *response_packet_data = PACKET_HEADER_COUNTERS_ACK; + args->response_packet_size = PACKET_HEADER_SIZE; + + for (i = 0; i < num_global_mali_profiling_counters; ++i) { + u32 name_size = strlen(global_mali_profiling_counters[i].counter_name); + + if ((args->response_packet_size + name_size + 1) > output_buffer_size) { + MALI_PRINT_ERROR(("Response packet data is too large..\n")); + return _MALI_OSK_ERR_FAULT; + } + + memcpy(response_packet_data + args->response_packet_size, + global_mali_profiling_counters[i].counter_name, name_size + 1); + + args->response_packet_size += (name_size + 1); + + if (global_mali_profiling_counters[i].counter_id == COUNTER_VP_ACTIVITY) { + args->response_packet_size += _mali_profiling_pack_int(response_packet_data, + output_buffer_size, args->response_packet_size, (s32)1); + } else if (global_mali_profiling_counters[i].counter_id == COUNTER_FP_ACTIVITY) { + args->response_packet_size += _mali_profiling_pack_int(response_packet_data, + output_buffer_size, args->response_packet_size, (s32)mali_pp_get_glob_num_pp_cores()); + } else { + args->response_packet_size += _mali_profiling_pack_int(response_packet_data, + output_buffer_size, args->response_packet_size, (s32) - 1); + } + } + + _mali_profiling_set_packet_size(response_packet_data + 1, args->response_packet_size); + break; + } + + case PACKET_HEADER_COUNTERS_ENABLE: { + int i; + u32 request_pos = PACKET_HEADER_SIZE; + mali_bool sw_counter_if_enabled = MALI_FALSE; + + if (PACKET_HEADER_SIZE > control_packet_size || + control_packet_size != _mali_profiling_get_packet_size(control_packet_data + 1)) { + MALI_PRINT_ERROR(("Wrong control packet size , type 0x%x,size 0x%x.\n", control_packet_data[0], control_packet_size)); + return _MALI_OSK_ERR_FAULT; + } + + /* Init all counter states before enable requested counters.*/ + for (i = 0; i < num_global_mali_profiling_counters; ++i) { + _mali_profiling_set_event(global_mali_profiling_counters[i].counter_id, MALI_HW_CORE_NO_COUNTER); + global_mali_profiling_counters[i].enabled = 0; + global_mali_profiling_counters[i].prev_counter_value = 0; + global_mali_profiling_counters[i].current_counter_value = 0; + + if (global_mali_profiling_counters[i].counter_id >= FIRST_MEM_COUNTER && + global_mali_profiling_counters[i].counter_id <= LAST_MEM_COUNTER) { + _mali_profiling_notification_mem_counter(session, global_mali_profiling_counters[i].counter_id, 0, 0); + } + } + + l2_cache_counter_if_enabled = MALI_FALSE; + num_counters_enabled = 0; + mem_counters_enabled = 0; + _mali_profiling_control(FBDUMP_CONTROL_ENABLE, 0); + _mali_profiling_control(SW_COUNTER_ENABLE, 0); + _mali_profiling_notification_enable(session, 0, 0); + + /* Enable requested counters */ + while (request_pos < control_packet_size) { + u32 begin = request_pos; + u32 event; + u32 key; + + while (request_pos < control_packet_size && control_packet_data[request_pos] != '\0') { + ++request_pos; + } + + ++request_pos; + event = _mali_profiling_read_packet_int(control_packet_data, &request_pos, control_packet_size); + key = _mali_profiling_read_packet_int(control_packet_data, &request_pos, control_packet_size); + + for (i = 0; i < num_global_mali_profiling_counters; ++i) { + u32 name_size = strlen((char *)(control_packet_data + begin)); + if (strncmp(global_mali_profiling_counters[i].counter_name, (char *)(control_packet_data + begin), name_size) == 0) { + if (!sw_counter_if_enabled && (FIRST_SW_COUNTER <= global_mali_profiling_counters[i].counter_id + && global_mali_profiling_counters[i].counter_id <= LAST_SW_COUNTER)) { + sw_counter_if_enabled = MALI_TRUE; + _mali_profiling_control(SW_COUNTER_ENABLE, 1); + } + + if (COUNTER_FILMSTRIP == global_mali_profiling_counters[i].counter_id) { + _mali_profiling_control(FBDUMP_CONTROL_ENABLE, 1); + _mali_profiling_control(FBDUMP_CONTROL_RATE, event & 0xff); + _mali_profiling_control(FBDUMP_CONTROL_RESIZE_FACTOR, (event >> 8) & 0xff); + } + + if (global_mali_profiling_counters[i].counter_id >= FIRST_MEM_COUNTER && + global_mali_profiling_counters[i].counter_id <= LAST_MEM_COUNTER) { + _mali_profiling_notification_mem_counter(session, global_mali_profiling_counters[i].counter_id, + key, 1); + mem_counters_enabled++; + } + + global_mali_profiling_counters[i].counter_event = event; + global_mali_profiling_counters[i].key = key; + global_mali_profiling_counters[i].enabled = 1; + + _mali_profiling_set_event(global_mali_profiling_counters[i].counter_id, + global_mali_profiling_counters[i].counter_event); + num_counters_enabled++; + break; + } + } + + if (i == num_global_mali_profiling_counters) { + MALI_PRINT_ERROR(("Counter name does not match for type %u.\n", control_type)); + return _MALI_OSK_ERR_FAULT; + } + } + + if (PACKET_HEADER_SIZE <= output_buffer_size) { + *response_packet_data = PACKET_HEADER_ACK; + _mali_profiling_set_packet_size(response_packet_data + 1, PACKET_HEADER_SIZE); + args->response_packet_size = PACKET_HEADER_SIZE; + } else { + return _MALI_OSK_ERR_FAULT; + } + + break; + } + + case PACKET_HEADER_START_CAPTURE_VALUE: { + u32 live_rate; + u32 request_pos = PACKET_HEADER_SIZE; + + if (PACKET_HEADER_SIZE > control_packet_size || + control_packet_size != _mali_profiling_get_packet_size(control_packet_data + 1)) { + MALI_PRINT_ERROR(("Wrong control packet size , type 0x%x,size 0x%x.\n", control_packet_data[0], control_packet_size)); + return _MALI_OSK_ERR_FAULT; + } + + /* Read samping rate in nanoseconds and live rate, start capture.*/ + profiling_sample_rate = _mali_profiling_read_packet_int(control_packet_data, + &request_pos, control_packet_size); + + live_rate = _mali_profiling_read_packet_int(control_packet_data, &request_pos, control_packet_size); + + if (PACKET_HEADER_SIZE <= output_buffer_size) { + *response_packet_data = PACKET_HEADER_ACK; + _mali_profiling_set_packet_size(response_packet_data + 1, PACKET_HEADER_SIZE); + args->response_packet_size = PACKET_HEADER_SIZE; + } else { + return _MALI_OSK_ERR_FAULT; + } + + if (0 != num_counters_enabled && 0 != profiling_sample_rate) { + _mali_profiling_global_stream_list_free(); + if (mem_counters_enabled > 0) { + _mali_profiling_notification_enable(session, profiling_sample_rate, 1); + } + hrtimer_start(&profiling_sampling_timer, + ktime_set(profiling_sample_rate / 1000000000, profiling_sample_rate % 1000000000), + HRTIMER_MODE_REL_PINNED); + } + + break; + } + default: + MALI_PRINT_ERROR(("Unsupported profiling packet header type %u.\n", control_type)); + args->response_packet_size = 0; + return _MALI_OSK_ERR_FAULT; + } + } else { + _mali_osk_profiling_stop_sampling(current_profiling_pid); + _mali_profiling_notification_enable(session, 0, 0); + } + + return _MALI_OSK_ERR_OK; +} + +/** + * Called by gator.ko to set HW counters + * + * @param counter_id The counter ID. + * @param event_id Event ID that the counter should count (HW counter value from TRM). + * + * @return 1 on success, 0 on failure. + */ +int _mali_profiling_set_event(u32 counter_id, s32 event_id) +{ + if (COUNTER_VP_0_C0 == counter_id) { + mali_gp_job_set_gp_counter_src0(event_id); + } else if (COUNTER_VP_0_C1 == counter_id) { + mali_gp_job_set_gp_counter_src1(event_id); + } else if (COUNTER_FP_0_C0 <= counter_id && COUNTER_FP_7_C1 >= counter_id) { + /* + * Two compatibility notes for this function: + * + * 1) Previously the DDK allowed per core counters. + * + * This did not make much sense on Mali-450 with the "virtual PP core" concept, + * so this option was removed, and only the same pair of HW counters was allowed on all cores, + * beginning with r3p2 release. + * + * Starting with r4p0, it is now possible to set different HW counters for the different sub jobs. + * This should be almost the same, since sub job 0 is designed to run on core 0, + * sub job 1 on core 1, and so on. + * + * The scheduling of PP sub jobs is not predictable, and this often led to situations where core 0 ran 2 + * sub jobs, while for instance core 1 ran zero. Having the counters set per sub job would thus increase + * the predictability of the returned data (as you would be guaranteed data for all the selected HW counters). + * + * PS: Core scaling needs to be disabled in order to use this reliably (goes for both solutions). + * + * The framework/#defines with Gator still indicates that the counter is for a particular core, + * but this is internally used as a sub job ID instead (no translation needed). + * + * 2) Global/default vs per sub job counters + * + * Releases before r3p2 had only per PP core counters. + * r3p2 releases had only one set of default/global counters which applied to all PP cores + * Starting with r4p0, we have both a set of default/global counters, + * and individual counters per sub job (equal to per core). + * + * To keep compatibility with Gator/DS-5/streamline, the following scheme is used: + * + * r3p2 release; only counters set for core 0 is handled, + * this is applied as the default/global set of counters, and will thus affect all cores. + * + * r4p0 release; counters set for core 0 is applied as both the global/default set of counters, + * and counters for sub job 0. + * Counters set for core 1-7 is only applied for the corresponding sub job. + * + * This should allow the DS-5/Streamline GUI to have a simple mode where it only allows setting the + * values for core 0, and thus this will be applied to all PP sub jobs/cores. + * Advanced mode will also be supported, where individual pairs of HW counters can be selected. + * + * The GUI will (until it is updated) still refer to cores instead of sub jobs, but this is probably + * something we can live with! + * + * Mali-450 note: Each job is not divided into a deterministic number of sub jobs, as the HW DLBU + * automatically distributes the load between whatever number of cores is available at this particular time. + * A normal PP job on Mali-450 is thus considered a single (virtual) job, and it will thus only be possible + * to use a single pair of HW counters (even if the job ran on multiple PP cores). + * In other words, only the global/default pair of PP HW counters will be used for normal Mali-450 jobs. + */ + u32 sub_job = (counter_id - COUNTER_FP_0_C0) >> 1; + u32 counter_src = (counter_id - COUNTER_FP_0_C0) & 1; + if (0 == counter_src) { + mali_pp_job_set_pp_counter_sub_job_src0(sub_job, event_id); + if (0 == sub_job) { + mali_pp_job_set_pp_counter_global_src0(event_id); + } + } else { + mali_pp_job_set_pp_counter_sub_job_src1(sub_job, event_id); + if (0 == sub_job) { + mali_pp_job_set_pp_counter_global_src1(event_id); + } + } + } else if (COUNTER_L2_0_C0 <= counter_id && COUNTER_L2_2_C1 >= counter_id) { + u32 core_id = (counter_id - COUNTER_L2_0_C0) >> 1; + struct mali_l2_cache_core *l2_cache_core = mali_l2_cache_core_get_glob_l2_core(core_id); + + if (NULL != l2_cache_core) { + u32 counter_src = (counter_id - COUNTER_L2_0_C0) & 1; + mali_l2_cache_core_set_counter_src(l2_cache_core, + counter_src, event_id); + l2_cache_counter_if_enabled = MALI_TRUE; + } + } else { + return 0; /* Failure, unknown event */ + } + + return 1; /* success */ +} + +/** + * Called by gator.ko to retrieve the L2 cache counter values for all L2 cache cores. + * The L2 cache counters are unique in that they are polled by gator, rather than being + * transmitted via the tracepoint mechanism. + * + * @param values Pointer to a _mali_profiling_l2_counter_values structure where + * the counter sources and values will be output + * @return 0 if all went well; otherwise, return the mask with the bits set for the powered off cores + */ +u32 _mali_profiling_get_l2_counters(_mali_profiling_l2_counter_values *values) +{ + u32 l2_cores_num = mali_l2_cache_core_get_glob_num_l2_cores(); + u32 i; + + MALI_DEBUG_ASSERT(l2_cores_num <= 3); + + for (i = 0; i < l2_cores_num; i++) { + struct mali_l2_cache_core *l2_cache = mali_l2_cache_core_get_glob_l2_core(i); + + if (NULL == l2_cache) { + continue; + } + + mali_l2_cache_core_get_counter_values(l2_cache, + &values->cores[i].source0, + &values->cores[i].value0, + &values->cores[i].source1, + &values->cores[i].value1); + } + + return 0; +} + +/** + * Called by gator to control the production of profiling information at runtime. + */ +void _mali_profiling_control(u32 action, u32 value) +{ + switch (action) { + case FBDUMP_CONTROL_ENABLE: + mali_set_user_setting(_MALI_UK_USER_SETTING_COLORBUFFER_CAPTURE_ENABLED, (value == 0 ? MALI_FALSE : MALI_TRUE)); + break; + case FBDUMP_CONTROL_RATE: + mali_set_user_setting(_MALI_UK_USER_SETTING_BUFFER_CAPTURE_N_FRAMES, value); + break; + case SW_COUNTER_ENABLE: + mali_set_user_setting(_MALI_UK_USER_SETTING_SW_COUNTER_ENABLED, value); + break; + case FBDUMP_CONTROL_RESIZE_FACTOR: + mali_set_user_setting(_MALI_UK_USER_SETTING_BUFFER_CAPTURE_RESIZE_FACTOR, value); + break; + default: + break; /* Ignore unimplemented actions */ + } +} + +/** + * Called by gator to get mali api version. + */ +u32 _mali_profiling_get_api_version(void) +{ + return MALI_PROFILING_API_VERSION; +} + +/** +* Called by gator to get the data about Mali instance in use: +* product id, version, number of cores +*/ +void _mali_profiling_get_mali_version(struct _mali_profiling_mali_version *values) +{ + values->mali_product_id = (u32)mali_kernel_core_get_product_id(); + values->mali_version_major = mali_kernel_core_get_gpu_major_version(); + values->mali_version_minor = mali_kernel_core_get_gpu_minor_version(); + values->num_of_l2_cores = mali_l2_cache_core_get_glob_num_l2_cores(); + values->num_of_fp_cores = mali_executor_get_num_cores_total(); + values->num_of_vp_cores = 1; +} + + +EXPORT_SYMBOL(_mali_profiling_set_event); +EXPORT_SYMBOL(_mali_profiling_get_l2_counters); +EXPORT_SYMBOL(_mali_profiling_control); +EXPORT_SYMBOL(_mali_profiling_get_api_version); +EXPORT_SYMBOL(_mali_profiling_get_mali_version); diff --git a/drivers/gpu/arm/utgard/linux/mali_osk_specific.h b/drivers/gpu/arm/utgard/linux/mali_osk_specific.h new file mode 100644 index 00000000000000..db034a5b3c7072 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_osk_specific.h @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2010, 2012-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_specific.h + * Defines per-OS Kernel level specifics, such as unusual workarounds for + * certain OSs. + */ + +#ifndef __MALI_OSK_SPECIFIC_H__ +#define __MALI_OSK_SPECIFIC_H__ + +#include +#include +#include +#include + + +#include "mali_osk_types.h" +#include "mali_kernel_linux.h" + +#define MALI_STATIC_INLINE static inline +#define MALI_NON_STATIC_INLINE inline + +typedef struct dma_pool *mali_dma_pool; + +typedef u32 mali_dma_addr; + +#if MALI_ENABLE_CPU_CYCLES +/* Reads out the clock cycle performance counter of the current cpu. + It is useful for cost-free (2 cycle) measuring of the time spent + in a code path. Sample before and after, the diff number of cycles. + When the CPU is idle it will not increase this clock counter. + It means that the counter is accurate if only spin-locks are used, + but mutexes may lead to too low values since the cpu might "idle" + waiting for the mutex to become available. + The clock source is configured on the CPU during mali module load, + but will not give useful output after a CPU has been power cycled. + It is therefore important to configure the system to not turn of + the cpu cores when using this functionallity.*/ +static inline unsigned int mali_get_cpu_cyclecount(void) +{ + unsigned int value; + /* Reading the CCNT Register - CPU clock counter */ + asm volatile("MRC p15, 0, %0, c9, c13, 0\t\n": "=r"(value)); + return value; +} + +void mali_init_cpu_time_counters(int reset, int enable_divide_by_64); +#endif + + +MALI_STATIC_INLINE u32 _mali_osk_copy_from_user(void *to, void *from, u32 n) +{ + return (u32)copy_from_user(to, from, (unsigned long)n); +} + +MALI_STATIC_INLINE mali_bool _mali_osk_in_atomic(void) +{ + return in_atomic(); +} + +#define _mali_osk_put_user(x, ptr) put_user(x, ptr) + +#endif /* __MALI_OSK_SPECIFIC_H__ */ diff --git a/drivers/gpu/arm/utgard/linux/mali_osk_time.c b/drivers/gpu/arm/utgard/linux/mali_osk_time.c new file mode 100644 index 00000000000000..4deaa101e48f9c --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_osk_time.c @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2010, 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_time.c + * Implementation of the OS abstraction layer for the kernel device driver + */ + +#include "mali_osk.h" +#include +#include +#include + +mali_bool _mali_osk_time_after_eq(unsigned long ticka, unsigned long tickb) +{ + return time_after_eq(ticka, tickb) ? + MALI_TRUE : MALI_FALSE; +} + +unsigned long _mali_osk_time_mstoticks(u32 ms) +{ + return msecs_to_jiffies(ms); +} + +u32 _mali_osk_time_tickstoms(unsigned long ticks) +{ + return jiffies_to_msecs(ticks); +} + +unsigned long _mali_osk_time_tickcount(void) +{ + return jiffies; +} + +void _mali_osk_time_ubusydelay(u32 usecs) +{ + udelay(usecs); +} + +u64 _mali_osk_time_get_ns(void) +{ + struct timespec tsval; + getnstimeofday(&tsval); + return (u64)timespec_to_ns(&tsval); +} + +u64 _mali_osk_boot_time_get_ns(void) +{ + struct timespec tsval; + get_monotonic_boottime(&tsval); + return (u64)timespec_to_ns(&tsval); +} diff --git a/drivers/gpu/arm/utgard/linux/mali_osk_timers.c b/drivers/gpu/arm/utgard/linux/mali_osk_timers.c new file mode 100644 index 00000000000000..6bbaee749d64d7 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_osk_timers.c @@ -0,0 +1,76 @@ +/* + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_timers.c + * Implementation of the OS abstraction layer for the kernel device driver + */ + +#include +#include +#include "mali_osk.h" +#include "mali_kernel_common.h" + +struct _mali_osk_timer_t_struct { + struct timer_list timer; +}; + +typedef void (*timer_timeout_function_t)(unsigned long); + +_mali_osk_timer_t *_mali_osk_timer_init(void) +{ + _mali_osk_timer_t *t = (_mali_osk_timer_t *)kmalloc(sizeof(_mali_osk_timer_t), GFP_KERNEL); + if (NULL != t) init_timer(&t->timer); + return t; +} + +void _mali_osk_timer_add(_mali_osk_timer_t *tim, unsigned long ticks_to_expire) +{ + MALI_DEBUG_ASSERT_POINTER(tim); + tim->timer.expires = jiffies + ticks_to_expire; + add_timer(&(tim->timer)); +} + +void _mali_osk_timer_mod(_mali_osk_timer_t *tim, unsigned long ticks_to_expire) +{ + MALI_DEBUG_ASSERT_POINTER(tim); + mod_timer(&(tim->timer), jiffies + ticks_to_expire); +} + +void _mali_osk_timer_del(_mali_osk_timer_t *tim) +{ + MALI_DEBUG_ASSERT_POINTER(tim); + del_timer_sync(&(tim->timer)); +} + +void _mali_osk_timer_del_async(_mali_osk_timer_t *tim) +{ + MALI_DEBUG_ASSERT_POINTER(tim); + del_timer(&(tim->timer)); +} + +mali_bool _mali_osk_timer_pending(_mali_osk_timer_t *tim) +{ + MALI_DEBUG_ASSERT_POINTER(tim); + return 1 == timer_pending(&(tim->timer)); +} + +void _mali_osk_timer_setcallback(_mali_osk_timer_t *tim, _mali_osk_timer_callback_t callback, void *data) +{ + MALI_DEBUG_ASSERT_POINTER(tim); + tim->timer.data = (unsigned long)data; + tim->timer.function = (timer_timeout_function_t)callback; +} + +void _mali_osk_timer_term(_mali_osk_timer_t *tim) +{ + MALI_DEBUG_ASSERT_POINTER(tim); + kfree(tim); +} diff --git a/drivers/gpu/arm/utgard/linux/mali_osk_wait_queue.c b/drivers/gpu/arm/utgard/linux/mali_osk_wait_queue.c new file mode 100644 index 00000000000000..15d5ce250eb199 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_osk_wait_queue.c @@ -0,0 +1,78 @@ +/* + * Copyright (C) 2012-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_wait_queue.c + * Implemenation of the OS abstraction layer for the kernel device driver + */ + +#include +#include +#include + +#include "mali_osk.h" +#include "mali_kernel_common.h" + +struct _mali_osk_wait_queue_t_struct { + wait_queue_head_t wait_queue; +}; + +_mali_osk_wait_queue_t *_mali_osk_wait_queue_init(void) +{ + _mali_osk_wait_queue_t *ret = NULL; + + ret = kmalloc(sizeof(_mali_osk_wait_queue_t), GFP_KERNEL); + + if (NULL == ret) { + return ret; + } + + init_waitqueue_head(&ret->wait_queue); + MALI_DEBUG_ASSERT(!waitqueue_active(&ret->wait_queue)); + + return ret; +} + +void _mali_osk_wait_queue_wait_event(_mali_osk_wait_queue_t *queue, mali_bool(*condition)(void *), void *data) +{ + MALI_DEBUG_ASSERT_POINTER(queue); + MALI_DEBUG_PRINT(6, ("Adding to wait queue %p\n", queue)); + wait_event(queue->wait_queue, condition(data)); +} + +void _mali_osk_wait_queue_wait_event_timeout(_mali_osk_wait_queue_t *queue, mali_bool(*condition)(void *), void *data, u32 timeout) +{ + MALI_DEBUG_ASSERT_POINTER(queue); + MALI_DEBUG_PRINT(6, ("Adding to wait queue %p\n", queue)); + wait_event_timeout(queue->wait_queue, condition(data), _mali_osk_time_mstoticks(timeout)); +} + +void _mali_osk_wait_queue_wake_up(_mali_osk_wait_queue_t *queue) +{ + MALI_DEBUG_ASSERT_POINTER(queue); + + /* if queue is empty, don't attempt to wake up its elements */ + if (!waitqueue_active(&queue->wait_queue)) return; + + MALI_DEBUG_PRINT(6, ("Waking up elements in wait queue %p ....\n", queue)); + + wake_up_all(&queue->wait_queue); + + MALI_DEBUG_PRINT(6, ("... elements in wait queue %p woken up\n", queue)); +} + +void _mali_osk_wait_queue_term(_mali_osk_wait_queue_t *queue) +{ + /* Parameter validation */ + MALI_DEBUG_ASSERT_POINTER(queue); + + /* Linux requires no explicit termination of wait queues */ + kfree(queue); +} diff --git a/drivers/gpu/arm/utgard/linux/mali_osk_wq.c b/drivers/gpu/arm/utgard/linux/mali_osk_wq.c new file mode 100644 index 00000000000000..2c34c91a7922d5 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_osk_wq.c @@ -0,0 +1,240 @@ +/* + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_wq.c + * Implementation of the OS abstraction layer for the kernel device driver + */ + +#include /* For memory allocation */ +#include +#include +#include + +#include "mali_osk.h" +#include "mali_kernel_common.h" +#include "mali_kernel_license.h" +#include "mali_kernel_linux.h" + +typedef struct _mali_osk_wq_work_s { + _mali_osk_wq_work_handler_t handler; + void *data; + mali_bool high_pri; + struct work_struct work_handle; +} mali_osk_wq_work_object_t; + +typedef struct _mali_osk_wq_delayed_work_s { + _mali_osk_wq_work_handler_t handler; + void *data; + struct delayed_work work; +} mali_osk_wq_delayed_work_object_t; + +#if MALI_LICENSE_IS_GPL +static struct workqueue_struct *mali_wq_normal = NULL; +static struct workqueue_struct *mali_wq_high = NULL; +#endif + +static void _mali_osk_wq_work_func(struct work_struct *work); + +_mali_osk_errcode_t _mali_osk_wq_init(void) +{ +#if MALI_LICENSE_IS_GPL + MALI_DEBUG_ASSERT(NULL == mali_wq_normal); + MALI_DEBUG_ASSERT(NULL == mali_wq_high); + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 36) + mali_wq_normal = alloc_workqueue("mali", WQ_UNBOUND, 0); + mali_wq_high = alloc_workqueue("mali_high_pri", WQ_HIGHPRI | WQ_UNBOUND, 0); +#else + mali_wq_normal = create_workqueue("mali"); + mali_wq_high = create_workqueue("mali_high_pri"); +#endif + if (NULL == mali_wq_normal || NULL == mali_wq_high) { + MALI_PRINT_ERROR(("Unable to create Mali workqueues\n")); + + if (mali_wq_normal) destroy_workqueue(mali_wq_normal); + if (mali_wq_high) destroy_workqueue(mali_wq_high); + + mali_wq_normal = NULL; + mali_wq_high = NULL; + + return _MALI_OSK_ERR_FAULT; + } +#endif /* MALI_LICENSE_IS_GPL */ + + return _MALI_OSK_ERR_OK; +} + +void _mali_osk_wq_flush(void) +{ +#if MALI_LICENSE_IS_GPL + flush_workqueue(mali_wq_high); + flush_workqueue(mali_wq_normal); +#else + flush_scheduled_work(); +#endif +} + +void _mali_osk_wq_term(void) +{ +#if MALI_LICENSE_IS_GPL + MALI_DEBUG_ASSERT(NULL != mali_wq_normal); + MALI_DEBUG_ASSERT(NULL != mali_wq_high); + + flush_workqueue(mali_wq_normal); + destroy_workqueue(mali_wq_normal); + + flush_workqueue(mali_wq_high); + destroy_workqueue(mali_wq_high); + + mali_wq_normal = NULL; + mali_wq_high = NULL; +#else + flush_scheduled_work(); +#endif +} + +_mali_osk_wq_work_t *_mali_osk_wq_create_work(_mali_osk_wq_work_handler_t handler, void *data) +{ + mali_osk_wq_work_object_t *work = kmalloc(sizeof(mali_osk_wq_work_object_t), GFP_KERNEL); + + if (NULL == work) return NULL; + + work->handler = handler; + work->data = data; + work->high_pri = MALI_FALSE; + + INIT_WORK(&work->work_handle, _mali_osk_wq_work_func); + + return work; +} + +_mali_osk_wq_work_t *_mali_osk_wq_create_work_high_pri(_mali_osk_wq_work_handler_t handler, void *data) +{ + mali_osk_wq_work_object_t *work = kmalloc(sizeof(mali_osk_wq_work_object_t), GFP_KERNEL); + + if (NULL == work) return NULL; + + work->handler = handler; + work->data = data; + work->high_pri = MALI_TRUE; + + INIT_WORK(&work->work_handle, _mali_osk_wq_work_func); + + return work; +} + +void _mali_osk_wq_delete_work(_mali_osk_wq_work_t *work) +{ + mali_osk_wq_work_object_t *work_object = (mali_osk_wq_work_object_t *)work; + _mali_osk_wq_flush(); + kfree(work_object); +} + +void _mali_osk_wq_delete_work_nonflush(_mali_osk_wq_work_t *work) +{ + mali_osk_wq_work_object_t *work_object = (mali_osk_wq_work_object_t *)work; + kfree(work_object); +} + +void _mali_osk_wq_schedule_work(_mali_osk_wq_work_t *work) +{ + mali_osk_wq_work_object_t *work_object = (mali_osk_wq_work_object_t *)work; +#if MALI_LICENSE_IS_GPL + queue_work(mali_wq_normal, &work_object->work_handle); +#else + schedule_work(&work_object->work_handle); +#endif +} + +void _mali_osk_wq_schedule_work_high_pri(_mali_osk_wq_work_t *work) +{ + mali_osk_wq_work_object_t *work_object = (mali_osk_wq_work_object_t *)work; +#if MALI_LICENSE_IS_GPL + queue_work(mali_wq_high, &work_object->work_handle); +#else + schedule_work(&work_object->work_handle); +#endif +} + +static void _mali_osk_wq_work_func(struct work_struct *work) +{ + mali_osk_wq_work_object_t *work_object; + + work_object = _MALI_OSK_CONTAINER_OF(work, mali_osk_wq_work_object_t, work_handle); + +#if MALI_LICENSE_IS_GPL +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,36) + /* We want highest Dynamic priority of the thread so that the Jobs depending + ** on this thread could be scheduled in time. Without this, this thread might + ** sometimes need to wait for some threads in user mode to finish its round-robin + ** time, causing *bubble* in the Mali pipeline. Thanks to the new implementation + ** of high-priority workqueue in new kernel, this only happens in older kernel. + */ + if (MALI_TRUE == work_object->high_pri) { + set_user_nice(current, -19); + } +#endif +#endif /* MALI_LICENSE_IS_GPL */ + + work_object->handler(work_object->data); +} + +static void _mali_osk_wq_delayed_work_func(struct work_struct *work) +{ + mali_osk_wq_delayed_work_object_t *work_object; + + work_object = _MALI_OSK_CONTAINER_OF(work, mali_osk_wq_delayed_work_object_t, work.work); + work_object->handler(work_object->data); +} + +mali_osk_wq_delayed_work_object_t *_mali_osk_wq_delayed_create_work(_mali_osk_wq_work_handler_t handler, void *data) +{ + mali_osk_wq_delayed_work_object_t *work = kmalloc(sizeof(mali_osk_wq_delayed_work_object_t), GFP_KERNEL); + + if (NULL == work) return NULL; + + work->handler = handler; + work->data = data; + + INIT_DELAYED_WORK(&work->work, _mali_osk_wq_delayed_work_func); + + return work; +} + +void _mali_osk_wq_delayed_delete_work_nonflush(_mali_osk_wq_delayed_work_t *work) +{ + mali_osk_wq_delayed_work_object_t *work_object = (mali_osk_wq_delayed_work_object_t *)work; + kfree(work_object); +} + +void _mali_osk_wq_delayed_cancel_work_async(_mali_osk_wq_delayed_work_t *work) +{ + mali_osk_wq_delayed_work_object_t *work_object = (mali_osk_wq_delayed_work_object_t *)work; + cancel_delayed_work(&work_object->work); +} + +void _mali_osk_wq_delayed_cancel_work_sync(_mali_osk_wq_delayed_work_t *work) +{ + mali_osk_wq_delayed_work_object_t *work_object = (mali_osk_wq_delayed_work_object_t *)work; + cancel_delayed_work_sync(&work_object->work); +} + +void _mali_osk_wq_delayed_schedule_work(_mali_osk_wq_delayed_work_t *work, u32 delay) +{ + mali_osk_wq_delayed_work_object_t *work_object = (mali_osk_wq_delayed_work_object_t *)work; + +#if MALI_LICENSE_IS_GPL + queue_delayed_work(mali_wq_normal, &work_object->work, delay); +#else + schedule_delayed_work(&work_object->work, delay); +#endif + +} diff --git a/drivers/gpu/arm/utgard/linux/mali_pmu_power_up_down.c b/drivers/gpu/arm/utgard/linux/mali_pmu_power_up_down.c new file mode 100644 index 00000000000000..61ff5c8fdca85b --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_pmu_power_up_down.c @@ -0,0 +1,23 @@ +/** + * Copyright (C) 2010, 2012-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_pmu_power_up_down.c + */ + +#include +#include "mali_executor.h" + +int mali_perf_set_num_pp_cores(unsigned int num_cores) +{ + return mali_executor_set_perf_level(num_cores, MALI_FALSE); +} + +EXPORT_SYMBOL(mali_perf_set_num_pp_cores); diff --git a/drivers/gpu/arm/utgard/linux/mali_profiling_events.h b/drivers/gpu/arm/utgard/linux/mali_profiling_events.h new file mode 100644 index 00000000000000..0b90e8c5cf26f3 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_profiling_events.h @@ -0,0 +1,17 @@ +/* + * Copyright (C) 2012, 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_PROFILING_EVENTS_H__ +#define __MALI_PROFILING_EVENTS_H__ + +/* Simple wrapper in order to find the OS specific location of this file */ +#include + +#endif /* __MALI_PROFILING_EVENTS_H__ */ diff --git a/drivers/gpu/arm/utgard/linux/mali_profiling_gator_api.h b/drivers/gpu/arm/utgard/linux/mali_profiling_gator_api.h new file mode 100644 index 00000000000000..c98d127366babd --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_profiling_gator_api.h @@ -0,0 +1,17 @@ +/* + * Copyright (C) 2012-2013, 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_PROFILING_GATOR_API_H__ +#define __MALI_PROFILING_GATOR_API_H__ + +/* Simple wrapper in order to find the OS specific location of this file */ +#include + +#endif /* __MALI_PROFILING_GATOR_API_H__ */ diff --git a/drivers/gpu/arm/utgard/linux/mali_profiling_internal.c b/drivers/gpu/arm/utgard/linux/mali_profiling_internal.c new file mode 100644 index 00000000000000..12aef4194ff530 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_profiling_internal.c @@ -0,0 +1,275 @@ +/* + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_kernel_common.h" +#include "mali_osk.h" +#include "mali_osk_mali.h" +#include "mali_ukk.h" +#include "mali_timestamp.h" +#include "mali_osk_profiling.h" +#include "mali_user_settings_db.h" +#include "mali_profiling_internal.h" + +typedef struct mali_profiling_entry { + u64 timestamp; + u32 event_id; + u32 data[5]; +} mali_profiling_entry; + +typedef enum mali_profiling_state { + MALI_PROFILING_STATE_UNINITIALIZED, + MALI_PROFILING_STATE_IDLE, + MALI_PROFILING_STATE_RUNNING, + MALI_PROFILING_STATE_RETURN, +} mali_profiling_state; + +static _mali_osk_mutex_t *lock = NULL; +static mali_profiling_state prof_state = MALI_PROFILING_STATE_UNINITIALIZED; +static mali_profiling_entry *profile_entries = NULL; +static _mali_osk_atomic_t profile_insert_index; +static u32 profile_mask = 0; + +static inline void add_event(u32 event_id, u32 data0, u32 data1, u32 data2, u32 data3, u32 data4); + +void probe_mali_timeline_event(void *data, TP_PROTO(unsigned int event_id, unsigned int d0, unsigned int d1, unsigned + int d2, unsigned int d3, unsigned int d4)) +{ + add_event(event_id, d0, d1, d2, d3, d4); +} + +_mali_osk_errcode_t _mali_internal_profiling_init(mali_bool auto_start) +{ + profile_entries = NULL; + profile_mask = 0; + _mali_osk_atomic_init(&profile_insert_index, 0); + + lock = _mali_osk_mutex_init(_MALI_OSK_LOCKFLAG_ORDERED, _MALI_OSK_LOCK_ORDER_PROFILING); + if (NULL == lock) { + return _MALI_OSK_ERR_FAULT; + } + + prof_state = MALI_PROFILING_STATE_IDLE; + + if (MALI_TRUE == auto_start) { + u32 limit = MALI_PROFILING_MAX_BUFFER_ENTRIES; /* Use maximum buffer size */ + + mali_set_user_setting(_MALI_UK_USER_SETTING_SW_EVENTS_ENABLE, MALI_TRUE); + if (_MALI_OSK_ERR_OK != _mali_internal_profiling_start(&limit)) { + return _MALI_OSK_ERR_FAULT; + } + } + + return _MALI_OSK_ERR_OK; +} + +void _mali_internal_profiling_term(void) +{ + u32 count; + + /* Ensure profiling is stopped */ + _mali_internal_profiling_stop(&count); + + prof_state = MALI_PROFILING_STATE_UNINITIALIZED; + + if (NULL != profile_entries) { + _mali_osk_vfree(profile_entries); + profile_entries = NULL; + } + + if (NULL != lock) { + _mali_osk_mutex_term(lock); + lock = NULL; + } +} + +_mali_osk_errcode_t _mali_internal_profiling_start(u32 *limit) +{ + _mali_osk_errcode_t ret; + mali_profiling_entry *new_profile_entries; + + _mali_osk_mutex_wait(lock); + + if (MALI_PROFILING_STATE_RUNNING == prof_state) { + _mali_osk_mutex_signal(lock); + return _MALI_OSK_ERR_BUSY; + } + + new_profile_entries = _mali_osk_valloc(*limit * sizeof(mali_profiling_entry)); + + if (NULL == new_profile_entries) { + _mali_osk_mutex_signal(lock); + _mali_osk_vfree(new_profile_entries); + return _MALI_OSK_ERR_NOMEM; + } + + if (MALI_PROFILING_MAX_BUFFER_ENTRIES < *limit) { + *limit = MALI_PROFILING_MAX_BUFFER_ENTRIES; + } + + profile_mask = 1; + while (profile_mask <= *limit) { + profile_mask <<= 1; + } + profile_mask >>= 1; + + *limit = profile_mask; + + profile_mask--; /* turns the power of two into a mask of one less */ + + if (MALI_PROFILING_STATE_IDLE != prof_state) { + _mali_osk_mutex_signal(lock); + _mali_osk_vfree(new_profile_entries); + return _MALI_OSK_ERR_INVALID_ARGS; /* invalid to call this function in this state */ + } + + profile_entries = new_profile_entries; + + ret = _mali_timestamp_reset(); + + if (_MALI_OSK_ERR_OK == ret) { + prof_state = MALI_PROFILING_STATE_RUNNING; + } else { + _mali_osk_vfree(profile_entries); + profile_entries = NULL; + } + + register_trace_mali_timeline_event(probe_mali_timeline_event, NULL); + + _mali_osk_mutex_signal(lock); + return ret; +} + +static inline void add_event(u32 event_id, u32 data0, u32 data1, u32 data2, u32 data3, u32 data4) +{ + u32 cur_index = (_mali_osk_atomic_inc_return(&profile_insert_index) - 1) & profile_mask; + + profile_entries[cur_index].timestamp = _mali_timestamp_get(); + profile_entries[cur_index].event_id = event_id; + profile_entries[cur_index].data[0] = data0; + profile_entries[cur_index].data[1] = data1; + profile_entries[cur_index].data[2] = data2; + profile_entries[cur_index].data[3] = data3; + profile_entries[cur_index].data[4] = data4; + + /* If event is "leave API function", add current memory usage to the event + * as data point 4. This is used in timeline profiling to indicate how + * much memory was used when leaving a function. */ + if (event_id == (MALI_PROFILING_EVENT_TYPE_SINGLE | MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | MALI_PROFILING_EVENT_REASON_SINGLE_SW_LEAVE_API_FUNC)) { + profile_entries[cur_index].data[4] = _mali_ukk_report_memory_usage(); + } +} + +_mali_osk_errcode_t _mali_internal_profiling_stop(u32 *count) +{ + _mali_osk_mutex_wait(lock); + + if (MALI_PROFILING_STATE_RUNNING != prof_state) { + _mali_osk_mutex_signal(lock); + return _MALI_OSK_ERR_INVALID_ARGS; /* invalid to call this function in this state */ + } + + /* go into return state (user to retreive events), no more events will be added after this */ + prof_state = MALI_PROFILING_STATE_RETURN; + + unregister_trace_mali_timeline_event(probe_mali_timeline_event, NULL); + + _mali_osk_mutex_signal(lock); + + tracepoint_synchronize_unregister(); + + *count = _mali_osk_atomic_read(&profile_insert_index); + if (*count > profile_mask) *count = profile_mask; + + return _MALI_OSK_ERR_OK; +} + +u32 _mali_internal_profiling_get_count(void) +{ + u32 retval = 0; + + _mali_osk_mutex_wait(lock); + if (MALI_PROFILING_STATE_RETURN == prof_state) { + retval = _mali_osk_atomic_read(&profile_insert_index); + if (retval > profile_mask) retval = profile_mask; + } + _mali_osk_mutex_signal(lock); + + return retval; +} + +_mali_osk_errcode_t _mali_internal_profiling_get_event(u32 index, u64 *timestamp, u32 *event_id, u32 data[5]) +{ + u32 raw_index = _mali_osk_atomic_read(&profile_insert_index); + + _mali_osk_mutex_wait(lock); + + if (index < profile_mask) { + if ((raw_index & ~profile_mask) != 0) { + index += raw_index; + index &= profile_mask; + } + + if (prof_state != MALI_PROFILING_STATE_RETURN) { + _mali_osk_mutex_signal(lock); + return _MALI_OSK_ERR_INVALID_ARGS; /* invalid to call this function in this state */ + } + + if (index >= raw_index) { + _mali_osk_mutex_signal(lock); + return _MALI_OSK_ERR_FAULT; + } + + *timestamp = profile_entries[index].timestamp; + *event_id = profile_entries[index].event_id; + data[0] = profile_entries[index].data[0]; + data[1] = profile_entries[index].data[1]; + data[2] = profile_entries[index].data[2]; + data[3] = profile_entries[index].data[3]; + data[4] = profile_entries[index].data[4]; + } else { + _mali_osk_mutex_signal(lock); + return _MALI_OSK_ERR_FAULT; + } + + _mali_osk_mutex_signal(lock); + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t _mali_internal_profiling_clear(void) +{ + _mali_osk_mutex_wait(lock); + + if (MALI_PROFILING_STATE_RETURN != prof_state) { + _mali_osk_mutex_signal(lock); + return _MALI_OSK_ERR_INVALID_ARGS; /* invalid to call this function in this state */ + } + + prof_state = MALI_PROFILING_STATE_IDLE; + profile_mask = 0; + _mali_osk_atomic_init(&profile_insert_index, 0); + + if (NULL != profile_entries) { + _mali_osk_vfree(profile_entries); + profile_entries = NULL; + } + + _mali_osk_mutex_signal(lock); + return _MALI_OSK_ERR_OK; +} + +mali_bool _mali_internal_profiling_is_recording(void) +{ + return prof_state == MALI_PROFILING_STATE_RUNNING ? MALI_TRUE : MALI_FALSE; +} + +mali_bool _mali_internal_profiling_have_recording(void) +{ + return prof_state == MALI_PROFILING_STATE_RETURN ? MALI_TRUE : MALI_FALSE; +} diff --git a/drivers/gpu/arm/utgard/linux/mali_profiling_internal.h b/drivers/gpu/arm/utgard/linux/mali_profiling_internal.h new file mode 100644 index 00000000000000..1c6f4da691d24c --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_profiling_internal.h @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2012-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_PROFILING_INTERNAL_H__ +#define __MALI_PROFILING_INTERNAL_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "mali_osk.h" + +int _mali_internal_profiling_init(mali_bool auto_start); +void _mali_internal_profiling_term(void); + +mali_bool _mali_internal_profiling_is_recording(void); +mali_bool _mali_internal_profiling_have_recording(void); +_mali_osk_errcode_t _mali_internal_profiling_clear(void); +_mali_osk_errcode_t _mali_internal_profiling_get_event(u32 index, u64 *timestamp, u32 *event_id, u32 data[5]); +u32 _mali_internal_profiling_get_count(void); +int _mali_internal_profiling_stop(u32 *count); +int _mali_internal_profiling_start(u32 *limit); + +#ifdef __cplusplus +} +#endif + +#endif /* __MALI_PROFILING_INTERNAL_H__ */ diff --git a/drivers/gpu/arm/utgard/linux/mali_sync.c b/drivers/gpu/arm/utgard/linux/mali_sync.c new file mode 100644 index 00000000000000..dc1e3a2a4d7359 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_sync.c @@ -0,0 +1,447 @@ +/* + * Copyright (C) 2012-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_sync.h" + +#include "mali_osk.h" +#include "mali_kernel_common.h" +#include "mali_timeline.h" +#include "mali_executor.h" + +#include +#include +#include +#include + +struct mali_sync_pt { + struct sync_pt sync_pt; + struct mali_sync_flag *flag; + struct sync_timeline *sync_tl; /**< Sync timeline this pt is connected to. */ +}; + +/** + * The sync flag is used to connect sync fences to the Mali Timeline system. Sync fences can be + * created from a sync flag, and when the flag is signaled, the sync fences will also be signaled. + */ +struct mali_sync_flag { + struct sync_timeline *sync_tl; /**< Sync timeline this flag is connected to. */ + u32 point; /**< Point on timeline. */ + int status; /**< 0 if unsignaled, 1 if signaled without error or negative if signaled with error. */ + struct kref refcount; /**< Reference count. */ +}; + +/** + * Mali sync timeline is used to connect mali timeline to sync_timeline. + * When fence timeout can print more detailed mali timeline system info. + */ +struct mali_sync_timeline_container { + struct sync_timeline sync_timeline; + struct mali_timeline *timeline; +}; + +MALI_STATIC_INLINE struct mali_sync_pt *to_mali_sync_pt(struct sync_pt *pt) +{ + return container_of(pt, struct mali_sync_pt, sync_pt); +} + +MALI_STATIC_INLINE struct mali_sync_timeline_container *to_mali_sync_tl_container(struct sync_timeline *sync_tl) +{ + return container_of(sync_tl, struct mali_sync_timeline_container, sync_timeline); +} + +static struct sync_pt *timeline_dup(struct sync_pt *pt) +{ + struct mali_sync_pt *mpt, *new_mpt; + struct sync_pt *new_pt; + + MALI_DEBUG_ASSERT_POINTER(pt); + mpt = to_mali_sync_pt(pt); + + new_pt = sync_pt_create(mpt->sync_tl, sizeof(struct mali_sync_pt)); + if (NULL == new_pt) return NULL; + + new_mpt = to_mali_sync_pt(new_pt); + + mali_sync_flag_get(mpt->flag); + new_mpt->flag = mpt->flag; + new_mpt->sync_tl = mpt->sync_tl; + + return new_pt; +} + +static int timeline_has_signaled(struct sync_pt *pt) +{ + struct mali_sync_pt *mpt; + + MALI_DEBUG_ASSERT_POINTER(pt); + mpt = to_mali_sync_pt(pt); + + MALI_DEBUG_ASSERT_POINTER(mpt->flag); + + return mpt->flag->status; +} + +static int timeline_compare(struct sync_pt *pta, struct sync_pt *ptb) +{ + struct mali_sync_pt *mpta; + struct mali_sync_pt *mptb; + u32 a, b; + + MALI_DEBUG_ASSERT_POINTER(pta); + MALI_DEBUG_ASSERT_POINTER(ptb); + mpta = to_mali_sync_pt(pta); + mptb = to_mali_sync_pt(ptb); + + MALI_DEBUG_ASSERT_POINTER(mpta->flag); + MALI_DEBUG_ASSERT_POINTER(mptb->flag); + + a = mpta->flag->point; + b = mptb->flag->point; + + if (a == b) return 0; + + return ((b - a) < (a - b) ? -1 : 1); +} + +static void timeline_free_pt(struct sync_pt *pt) +{ + struct mali_sync_pt *mpt; + + MALI_DEBUG_ASSERT_POINTER(pt); + mpt = to_mali_sync_pt(pt); + + mali_sync_flag_put(mpt->flag); +} + +static void timeline_release(struct sync_timeline *sync_timeline) +{ + struct mali_sync_timeline_container *mali_sync_tl = NULL; + struct mali_timeline *mali_tl = NULL; + + MALI_DEBUG_ASSERT_POINTER(sync_timeline); + + mali_sync_tl = to_mali_sync_tl_container(sync_timeline); + MALI_DEBUG_ASSERT_POINTER(mali_sync_tl); + + mali_tl = mali_sync_tl->timeline; + + /* always signaled timeline didn't have mali container */ + if (mali_tl) { + if (NULL != mali_tl->spinlock) { + mali_spinlock_reentrant_term(mali_tl->spinlock); + } + _mali_osk_free(mali_tl); + } + + module_put(THIS_MODULE); +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) +static void timeline_print_pt(struct seq_file *s, struct sync_pt *sync_pt) +{ + struct mali_sync_pt *mpt; + + MALI_DEBUG_ASSERT_POINTER(s); + MALI_DEBUG_ASSERT_POINTER(sync_pt); + + mpt = to_mali_sync_pt(sync_pt); + + /* It is possible this sync point is just under construct, + * make sure the flag is valid before accessing it + */ + if (mpt->flag) { + seq_printf(s, "%u", mpt->flag->point); + } else { + seq_printf(s, "uninitialized"); + } +} + +static void timeline_print_obj(struct seq_file *s, struct sync_timeline *sync_tl) +{ + struct mali_sync_timeline_container *mali_sync_tl = NULL; + struct mali_timeline *mali_tl = NULL; + + MALI_DEBUG_ASSERT_POINTER(sync_tl); + + mali_sync_tl = to_mali_sync_tl_container(sync_tl); + MALI_DEBUG_ASSERT_POINTER(mali_sync_tl); + + mali_tl = mali_sync_tl->timeline; + + if (NULL != mali_tl) { + seq_printf(s, "oldest (%u) ", mali_tl->point_oldest); + seq_printf(s, "next (%u)", mali_tl->point_next); + seq_printf(s, "\n"); + +#if defined(MALI_TIMELINE_DEBUG_FUNCTIONS) + { + u32 tid = _mali_osk_get_tid(); + struct mali_timeline_system *system = mali_tl->system; + + mali_spinlock_reentrant_wait(mali_tl->spinlock, tid); + if (!mali_tl->destroyed) { + mali_spinlock_reentrant_wait(system->spinlock, tid); + mali_timeline_debug_print_timeline(mali_tl, s); + mali_spinlock_reentrant_signal(system->spinlock, tid); + } + mali_spinlock_reentrant_signal(mali_tl->spinlock, tid); + + /* dump job queue status and group running status */ + mali_executor_status_dump(); + } +#endif + } +} +#else +static void timeline_pt_value_str(struct sync_pt *pt, char *str, int size) +{ + struct mali_sync_pt *mpt; + + MALI_DEBUG_ASSERT_POINTER(str); + MALI_DEBUG_ASSERT_POINTER(pt); + + mpt = to_mali_sync_pt(pt); + + /* It is possible this sync point is just under construct, + * make sure the flag is valid before accessing it + */ + if (mpt->flag) { + _mali_osk_snprintf(str, size, "%u", mpt->flag->point); + } else { + _mali_osk_snprintf(str, size, "uninitialized"); + } +} + +static void timeline_value_str(struct sync_timeline *timeline, char *str, int size) +{ + struct mali_sync_timeline_container *mali_sync_tl = NULL; + struct mali_timeline *mali_tl = NULL; + + MALI_DEBUG_ASSERT_POINTER(timeline); + + mali_sync_tl = to_mali_sync_tl_container(timeline); + MALI_DEBUG_ASSERT_POINTER(mali_sync_tl); + + mali_tl = mali_sync_tl->timeline; + + if (NULL != mali_tl) { + _mali_osk_snprintf(str, size, "oldest (%u) ", mali_tl->point_oldest); + _mali_osk_snprintf(str, size, "next (%u)", mali_tl->point_next); + _mali_osk_snprintf(str, size, "\n"); + +#if defined(MALI_TIMELINE_DEBUG_FUNCTIONS) + { + u32 tid = _mali_osk_get_tid(); + struct mali_timeline_system *system = mali_tl->system; + + mali_spinlock_reentrant_wait(mali_tl->spinlock, tid); + if (!mali_tl->destroyed) { + mali_spinlock_reentrant_wait(system->spinlock, tid); + mali_timeline_debug_direct_print_timeline(mali_tl); + mali_spinlock_reentrant_signal(system->spinlock, tid); + } + mali_spinlock_reentrant_signal(mali_tl->spinlock, tid); + + /* dump job queue status and group running status */ + mali_executor_status_dump(); + } +#endif + } +} +#endif + + +static struct sync_timeline_ops mali_timeline_ops = { + .driver_name = "Mali", + .dup = timeline_dup, + .has_signaled = timeline_has_signaled, + .compare = timeline_compare, + .free_pt = timeline_free_pt, + .release_obj = timeline_release, +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) + .print_pt = timeline_print_pt, + .print_obj = timeline_print_obj, +#else + .pt_value_str = timeline_pt_value_str, + .timeline_value_str = timeline_value_str, +#endif +}; + +struct sync_timeline *mali_sync_timeline_create(struct mali_timeline *timeline, const char *name) +{ + struct sync_timeline *sync_tl; + struct mali_sync_timeline_container *mali_sync_tl; + + sync_tl = sync_timeline_create(&mali_timeline_ops, sizeof(struct mali_sync_timeline_container), name); + if (NULL == sync_tl) return NULL; + + mali_sync_tl = to_mali_sync_tl_container(sync_tl); + mali_sync_tl->timeline = timeline; + + /* Grab a reference on the module to ensure the callbacks are present + * as long some timeline exists. The reference is released when the + * timeline is freed. + * Since this function is called from a ioctl on an open file we know + * we already have a reference, so using __module_get is safe. */ + __module_get(THIS_MODULE); + + return sync_tl; +} + +s32 mali_sync_fence_fd_alloc(struct sync_fence *sync_fence) +{ + s32 fd = -1; + + fd = get_unused_fd_flags(O_CLOEXEC); + if (fd < 0) { + sync_fence_put(sync_fence); + return -1; + } + sync_fence_install(sync_fence, fd); + + return fd; +} + +struct sync_fence *mali_sync_fence_merge(struct sync_fence *sync_fence1, struct sync_fence *sync_fence2) +{ + struct sync_fence *sync_fence; + + MALI_DEBUG_ASSERT_POINTER(sync_fence1); + MALI_DEBUG_ASSERT_POINTER(sync_fence1); + + sync_fence = sync_fence_merge("mali_merge_fence", sync_fence1, sync_fence2); + sync_fence_put(sync_fence1); + sync_fence_put(sync_fence2); + + return sync_fence; +} + +struct sync_fence *mali_sync_timeline_create_signaled_fence(struct sync_timeline *sync_tl) +{ + struct mali_sync_flag *flag; + struct sync_fence *sync_fence; + + MALI_DEBUG_ASSERT_POINTER(sync_tl); + + flag = mali_sync_flag_create(sync_tl, 0); + if (NULL == flag) return NULL; + + sync_fence = mali_sync_flag_create_fence(flag); + + mali_sync_flag_signal(flag, 0); + mali_sync_flag_put(flag); + + return sync_fence; +} + +struct mali_sync_flag *mali_sync_flag_create(struct sync_timeline *sync_tl, mali_timeline_point point) +{ + struct mali_sync_flag *flag; + + if (NULL == sync_tl) return NULL; + + flag = _mali_osk_calloc(1, sizeof(*flag)); + if (NULL == flag) return NULL; + + flag->sync_tl = sync_tl; + flag->point = point; + + flag->status = 0; + kref_init(&flag->refcount); + + return flag; +} + +void mali_sync_flag_get(struct mali_sync_flag *flag) +{ + MALI_DEBUG_ASSERT_POINTER(flag); + kref_get(&flag->refcount); +} + +/** + * Free sync flag. + * + * @param ref kref object embedded in sync flag that should be freed. + */ +static void mali_sync_flag_free(struct kref *ref) +{ + struct mali_sync_flag *flag; + + MALI_DEBUG_ASSERT_POINTER(ref); + flag = container_of(ref, struct mali_sync_flag, refcount); + + _mali_osk_free(flag); +} + +void mali_sync_flag_put(struct mali_sync_flag *flag) +{ + MALI_DEBUG_ASSERT_POINTER(flag); + kref_put(&flag->refcount, mali_sync_flag_free); +} + +void mali_sync_flag_signal(struct mali_sync_flag *flag, int error) +{ + MALI_DEBUG_ASSERT_POINTER(flag); + + MALI_DEBUG_ASSERT(0 == flag->status); + flag->status = (0 > error) ? error : 1; + + _mali_osk_write_mem_barrier(); + + sync_timeline_signal(flag->sync_tl); +} + +/** + * Create a sync point attached to given sync flag. + * + * @note Sync points must be triggered in *exactly* the same order as they are created. + * + * @param flag Sync flag. + * @return New sync point if successful, NULL if not. + */ +static struct sync_pt *mali_sync_flag_create_pt(struct mali_sync_flag *flag) +{ + struct sync_pt *pt; + struct mali_sync_pt *mpt; + + MALI_DEBUG_ASSERT_POINTER(flag); + MALI_DEBUG_ASSERT_POINTER(flag->sync_tl); + + pt = sync_pt_create(flag->sync_tl, sizeof(struct mali_sync_pt)); + if (NULL == pt) return NULL; + + mali_sync_flag_get(flag); + + mpt = to_mali_sync_pt(pt); + mpt->flag = flag; + mpt->sync_tl = flag->sync_tl; + + return pt; +} + +struct sync_fence *mali_sync_flag_create_fence(struct mali_sync_flag *flag) +{ + struct sync_pt *sync_pt; + struct sync_fence *sync_fence; + + MALI_DEBUG_ASSERT_POINTER(flag); + MALI_DEBUG_ASSERT_POINTER(flag->sync_tl); + + sync_pt = mali_sync_flag_create_pt(flag); + if (NULL == sync_pt) return NULL; + + sync_fence = sync_fence_create("mali_flag_fence", sync_pt); + if (NULL == sync_fence) { + sync_pt_free(sync_pt); + return NULL; + } + + return sync_fence; +} diff --git a/drivers/gpu/arm/utgard/linux/mali_sync.h b/drivers/gpu/arm/utgard/linux/mali_sync.h new file mode 100644 index 00000000000000..0c541ff9a2e044 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_sync.h @@ -0,0 +1,111 @@ +/* + * Copyright (C) 2012-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_sync.h + * + * Mali interface for Linux sync objects. + */ + +#ifndef _MALI_SYNC_H_ +#define _MALI_SYNC_H_ + +#if defined(CONFIG_SYNC) + +#include +#include +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 10, 0) +#include +#else +#include +#endif + + +#include "mali_osk.h" + +struct mali_sync_flag; +struct mali_timeline; + +/** + * Create a sync timeline. + * + * @param name Name of the sync timeline. + * @return The new sync timeline if successful, NULL if not. + */ +struct sync_timeline *mali_sync_timeline_create(struct mali_timeline *timeline, const char *name); + +/** + * Creates a file descriptor representing the sync fence. Will release sync fence if allocation of + * file descriptor fails. + * + * @param sync_fence Sync fence. + * @return File descriptor representing sync fence if successful, or -1 if not. + */ +s32 mali_sync_fence_fd_alloc(struct sync_fence *sync_fence); + +/** + * Merges two sync fences. Both input sync fences will be released. + * + * @param sync_fence1 First sync fence. + * @param sync_fence2 Second sync fence. + * @return New sync fence that is the result of the merger if successful, or NULL if not. + */ +struct sync_fence *mali_sync_fence_merge(struct sync_fence *sync_fence1, struct sync_fence *sync_fence2); + +/** + * Create a sync fence that is already signaled. + * + * @param tl Sync timeline. + * @return New signaled sync fence if successful, NULL if not. + */ +struct sync_fence *mali_sync_timeline_create_signaled_fence(struct sync_timeline *sync_tl); + +/** + * Create a sync flag. + * + * @param sync_tl Sync timeline. + * @param point Point on Mali timeline. + * @return New sync flag if successful, NULL if not. + */ +struct mali_sync_flag *mali_sync_flag_create(struct sync_timeline *sync_tl, u32 point); + +/** + * Grab sync flag reference. + * + * @param flag Sync flag. + */ +void mali_sync_flag_get(struct mali_sync_flag *flag); + +/** + * Release sync flag reference. If this was the last reference, the sync flag will be freed. + * + * @param flag Sync flag. + */ +void mali_sync_flag_put(struct mali_sync_flag *flag); + +/** + * Signal sync flag. All sync fences created from this flag will be signaled. + * + * @param flag Sync flag to signal. + * @param error Negative error code, or 0 if no error. + */ +void mali_sync_flag_signal(struct mali_sync_flag *flag, int error); + +/** + * Create a sync fence attached to given sync flag. + * + * @param flag Sync flag. + * @return New sync fence if successful, NULL if not. + */ +struct sync_fence *mali_sync_flag_create_fence(struct mali_sync_flag *flag); + +#endif /* defined(CONFIG_SYNC) */ + +#endif /* _MALI_SYNC_H_ */ diff --git a/drivers/gpu/arm/utgard/linux/mali_uk_types.h b/drivers/gpu/arm/utgard/linux/mali_uk_types.h new file mode 100644 index 00000000000000..1884cdbba42483 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_uk_types.h @@ -0,0 +1,17 @@ +/* + * Copyright (C) 2012, 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_UK_TYPES_H__ +#define __MALI_UK_TYPES_H__ + +/* Simple wrapper in order to find the OS specific location of this file */ +#include + +#endif /* __MALI_UK_TYPES_H__ */ diff --git a/drivers/gpu/arm/utgard/linux/mali_ukk_core.c b/drivers/gpu/arm/utgard/linux/mali_ukk_core.c new file mode 100644 index 00000000000000..ba48763c6a4fe1 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_ukk_core.c @@ -0,0 +1,152 @@ +/* + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#include /* file system operations */ +#include /* memort allocation functions */ +#include /* user space access */ + +#include "mali_ukk.h" +#include "mali_osk.h" +#include "mali_kernel_common.h" +#include "mali_session.h" +#include "mali_ukk_wrappers.h" + +int get_api_version_wrapper(struct mali_session_data *session_data, _mali_uk_get_api_version_s __user *uargs) +{ + _mali_uk_get_api_version_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + + if (0 != get_user(kargs.version, &uargs->version)) return -EFAULT; + + kargs.ctx = (uintptr_t)session_data; + err = _mali_ukk_get_api_version(&kargs); + if (_MALI_OSK_ERR_OK != err) return map_errcode(err); + + if (0 != put_user(kargs.version, &uargs->version)) return -EFAULT; + if (0 != put_user(kargs.compatible, &uargs->compatible)) return -EFAULT; + + return 0; +} + +int get_api_version_v2_wrapper(struct mali_session_data *session_data, _mali_uk_get_api_version_v2_s __user *uargs) +{ + _mali_uk_get_api_version_v2_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + + if (0 != get_user(kargs.version, &uargs->version)) return -EFAULT; + + kargs.ctx = (uintptr_t)session_data; + err = _mali_ukk_get_api_version_v2(&kargs); + if (_MALI_OSK_ERR_OK != err) return map_errcode(err); + + if (0 != put_user(kargs.version, &uargs->version)) return -EFAULT; + if (0 != put_user(kargs.compatible, &uargs->compatible)) return -EFAULT; + + return 0; +} + +int wait_for_notification_wrapper(struct mali_session_data *session_data, _mali_uk_wait_for_notification_s __user *uargs) +{ + _mali_uk_wait_for_notification_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + + /* zero kargs to make sure we don't leak stack info to userspace */ + memset(&kargs, 0, sizeof(_mali_uk_wait_for_notification_s)); + + kargs.ctx = (uintptr_t)session_data; + err = _mali_ukk_wait_for_notification(&kargs); + if (_MALI_OSK_ERR_OK != err) return map_errcode(err); + + if (_MALI_NOTIFICATION_CORE_SHUTDOWN_IN_PROGRESS != kargs.type) { + kargs.ctx = (uintptr_t)NULL; /* prevent kernel address to be returned to user space */ + if (0 != copy_to_user(uargs, &kargs, sizeof(_mali_uk_wait_for_notification_s))) return -EFAULT; + } else { + if (0 != put_user(kargs.type, &uargs->type)) return -EFAULT; + } + + return 0; +} + +int post_notification_wrapper(struct mali_session_data *session_data, _mali_uk_post_notification_s __user *uargs) +{ + _mali_uk_post_notification_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + + kargs.ctx = (uintptr_t)session_data; + + if (0 != get_user(kargs.type, &uargs->type)) { + return -EFAULT; + } + + err = _mali_ukk_post_notification(&kargs); + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + return 0; +} + +int get_user_settings_wrapper(struct mali_session_data *session_data, _mali_uk_get_user_settings_s __user *uargs) +{ + _mali_uk_get_user_settings_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + + /* zero kargs to make sure we don't leak stack info to userspace */ + memset(&kargs, 0, sizeof(_mali_uk_get_user_settings_s)); + + kargs.ctx = (uintptr_t)session_data; + err = _mali_ukk_get_user_settings(&kargs); + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + kargs.ctx = 0; /* prevent kernel address to be returned to user space */ + if (0 != copy_to_user(uargs, &kargs, sizeof(_mali_uk_get_user_settings_s))) return -EFAULT; + + return 0; +} + +int request_high_priority_wrapper(struct mali_session_data *session_data, _mali_uk_request_high_priority_s __user *uargs) +{ + _mali_uk_request_high_priority_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + + kargs.ctx = (uintptr_t)session_data; + err = _mali_ukk_request_high_priority(&kargs); + + kargs.ctx = 0; + + return map_errcode(err); +} + +int pending_submit_wrapper(struct mali_session_data *session_data, _mali_uk_pending_submit_s __user *uargs) +{ + _mali_uk_pending_submit_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + + kargs.ctx = (uintptr_t)session_data; + err = _mali_ukk_pending_submit(&kargs); + if (_MALI_OSK_ERR_OK != err) return map_errcode(err); + + return 0; +} diff --git a/drivers/gpu/arm/utgard/linux/mali_ukk_gp.c b/drivers/gpu/arm/utgard/linux/mali_ukk_gp.c new file mode 100644 index 00000000000000..d4144c0f5e4816 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_ukk_gp.c @@ -0,0 +1,91 @@ +/* + * Copyright (C) 2010, 2012-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#include /* file system operations */ +#include /* user space access */ + +#include "mali_ukk.h" +#include "mali_osk.h" +#include "mali_kernel_common.h" +#include "mali_session.h" +#include "mali_ukk_wrappers.h" + +int gp_start_job_wrapper(struct mali_session_data *session_data, _mali_uk_gp_start_job_s __user *uargs) +{ + _mali_osk_errcode_t err; + + /* If the job was started successfully, 0 is returned. If there was an error, but the job + * was started, we return -ENOENT. For anything else returned, the job was not started. */ + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + err = _mali_ukk_gp_start_job(session_data, uargs); + if (_MALI_OSK_ERR_OK != err) return map_errcode(err); + + return 0; +} + +int gp_get_core_version_wrapper(struct mali_session_data *session_data, _mali_uk_get_gp_core_version_s __user *uargs) +{ + _mali_uk_get_gp_core_version_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + kargs.ctx = (uintptr_t)session_data; + err = _mali_ukk_get_gp_core_version(&kargs); + if (_MALI_OSK_ERR_OK != err) return map_errcode(err); + + /* no known transactions to roll-back */ + + if (0 != put_user(kargs.version, &uargs->version)) return -EFAULT; + + return 0; +} + +int gp_suspend_response_wrapper(struct mali_session_data *session_data, _mali_uk_gp_suspend_response_s __user *uargs) +{ + _mali_uk_gp_suspend_response_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_gp_suspend_response_s))) return -EFAULT; + + kargs.ctx = (uintptr_t)session_data; + err = _mali_ukk_gp_suspend_response(&kargs); + if (_MALI_OSK_ERR_OK != err) return map_errcode(err); + + if (0 != put_user(kargs.cookie, &uargs->cookie)) return -EFAULT; + + /* no known transactions to roll-back */ + return 0; +} + +int gp_get_number_of_cores_wrapper(struct mali_session_data *session_data, _mali_uk_get_gp_number_of_cores_s __user *uargs) +{ + _mali_uk_get_gp_number_of_cores_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + kargs.ctx = (uintptr_t)session_data; + err = _mali_ukk_get_gp_number_of_cores(&kargs); + if (_MALI_OSK_ERR_OK != err) return map_errcode(err); + + /* no known transactions to roll-back */ + + if (0 != put_user(kargs.number_of_cores, &uargs->number_of_cores)) return -EFAULT; + + return 0; +} diff --git a/drivers/gpu/arm/utgard/linux/mali_ukk_mem.c b/drivers/gpu/arm/utgard/linux/mali_ukk_mem.c new file mode 100644 index 00000000000000..ca1cba0585b393 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_ukk_mem.c @@ -0,0 +1,333 @@ +/* + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#include /* file system operations */ +#include /* user space access */ + +#include "mali_ukk.h" +#include "mali_osk.h" +#include "mali_kernel_common.h" +#include "mali_session.h" +#include "mali_ukk_wrappers.h" + +int mem_alloc_wrapper(struct mali_session_data *session_data, _mali_uk_alloc_mem_s __user *uargs) +{ + _mali_uk_alloc_mem_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_alloc_mem_s))) { + return -EFAULT; + } + kargs.ctx = (uintptr_t)session_data; + + err = _mali_ukk_mem_allocate(&kargs); + + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + if (0 != put_user(kargs.backend_handle, &uargs->backend_handle)) { + return -EFAULT; + } + + return 0; +} + +int mem_free_wrapper(struct mali_session_data *session_data, _mali_uk_free_mem_s __user *uargs) +{ + _mali_uk_free_mem_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_free_mem_s))) { + return -EFAULT; + } + kargs.ctx = (uintptr_t)session_data; + + err = _mali_ukk_mem_free(&kargs); + + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + if (0 != put_user(kargs.free_pages_nr, &uargs->free_pages_nr)) { + return -EFAULT; + } + + return 0; +} + +int mem_bind_wrapper(struct mali_session_data *session_data, _mali_uk_bind_mem_s __user *uargs) +{ + _mali_uk_bind_mem_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_bind_mem_s))) { + return -EFAULT; + } + kargs.ctx = (uintptr_t)session_data; + + err = _mali_ukk_mem_bind(&kargs); + + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + return 0; +} + +int mem_unbind_wrapper(struct mali_session_data *session_data, _mali_uk_unbind_mem_s __user *uargs) +{ + _mali_uk_unbind_mem_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_unbind_mem_s))) { + return -EFAULT; + } + kargs.ctx = (uintptr_t)session_data; + + err = _mali_ukk_mem_unbind(&kargs); + + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + return 0; +} + + +int mem_cow_wrapper(struct mali_session_data *session_data, _mali_uk_cow_mem_s __user *uargs) +{ + _mali_uk_cow_mem_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_cow_mem_s))) { + return -EFAULT; + } + kargs.ctx = (uintptr_t)session_data; + + err = _mali_ukk_mem_cow(&kargs); + + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + if (0 != put_user(kargs.backend_handle, &uargs->backend_handle)) { + return -EFAULT; + } + + return 0; +} + +int mem_cow_modify_range_wrapper(struct mali_session_data *session_data, _mali_uk_cow_modify_range_s __user *uargs) +{ + _mali_uk_cow_modify_range_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_cow_modify_range_s))) { + return -EFAULT; + } + kargs.ctx = (uintptr_t)session_data; + + err = _mali_ukk_mem_cow_modify_range(&kargs); + + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + if (0 != put_user(kargs.change_pages_nr, &uargs->change_pages_nr)) { + return -EFAULT; + } + return 0; +} + + +int mem_resize_mem_wrapper(struct mali_session_data *session_data, _mali_uk_mem_resize_s __user *uargs) +{ + _mali_uk_mem_resize_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_mem_resize_s))) { + return -EFAULT; + } + kargs.ctx = (uintptr_t)session_data; + + err = _mali_ukk_mem_resize(&kargs); + + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + return 0; +} + +int mem_write_safe_wrapper(struct mali_session_data *session_data, _mali_uk_mem_write_safe_s __user *uargs) +{ + _mali_uk_mem_write_safe_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_mem_write_safe_s))) { + return -EFAULT; + } + + kargs.ctx = (uintptr_t)session_data; + + /* Check if we can access the buffers */ + if (!access_ok(VERIFY_WRITE, kargs.dest, kargs.size) + || !access_ok(VERIFY_READ, kargs.src, kargs.size)) { + return -EINVAL; + } + + /* Check if size wraps */ + if ((kargs.size + kargs.dest) <= kargs.dest + || (kargs.size + kargs.src) <= kargs.src) { + return -EINVAL; + } + + err = _mali_ukk_mem_write_safe(&kargs); + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + if (0 != put_user(kargs.size, &uargs->size)) { + return -EFAULT; + } + + return 0; +} + + + +int mem_query_mmu_page_table_dump_size_wrapper(struct mali_session_data *session_data, _mali_uk_query_mmu_page_table_dump_size_s __user *uargs) +{ + _mali_uk_query_mmu_page_table_dump_size_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + kargs.ctx = (uintptr_t)session_data; + + err = _mali_ukk_query_mmu_page_table_dump_size(&kargs); + if (_MALI_OSK_ERR_OK != err) return map_errcode(err); + + if (0 != put_user(kargs.size, &uargs->size)) return -EFAULT; + + return 0; +} + +int mem_dump_mmu_page_table_wrapper(struct mali_session_data *session_data, _mali_uk_dump_mmu_page_table_s __user *uargs) +{ + _mali_uk_dump_mmu_page_table_s kargs; + _mali_osk_errcode_t err; + void __user *user_buffer; + void *buffer = NULL; + int rc = -EFAULT; + + /* validate input */ + MALI_CHECK_NON_NULL(uargs, -EINVAL); + /* the session_data pointer was validated by caller */ + + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_dump_mmu_page_table_s))) + goto err_exit; + + user_buffer = (void __user *)(uintptr_t)kargs.buffer; + if (!access_ok(VERIFY_WRITE, user_buffer, kargs.size)) + goto err_exit; + + /* allocate temporary buffer (kernel side) to store mmu page table info */ + if (kargs.size <= 0) + return -EINVAL; + /* Allow at most 8MiB buffers, this is more than enough to dump a fully + * populated page table. */ + if (kargs.size > SZ_8M) + return -EINVAL; + + buffer = (void *)(uintptr_t)_mali_osk_valloc(kargs.size); + if (NULL == buffer) { + rc = -ENOMEM; + goto err_exit; + } + + kargs.ctx = (uintptr_t)session_data; + kargs.buffer = (uintptr_t)buffer; + err = _mali_ukk_dump_mmu_page_table(&kargs); + if (_MALI_OSK_ERR_OK != err) { + rc = map_errcode(err); + goto err_exit; + } + + /* copy mmu page table info back to user space and update pointers */ + if (0 != copy_to_user(user_buffer, buffer, kargs.size)) + goto err_exit; + + kargs.register_writes = kargs.register_writes - + (uintptr_t)buffer + (uintptr_t)user_buffer; + kargs.page_table_dump = kargs.page_table_dump - + (uintptr_t)buffer + (uintptr_t)user_buffer; + + if (0 != copy_to_user(uargs, &kargs, sizeof(kargs))) + goto err_exit; + + rc = 0; + +err_exit: + if (buffer) _mali_osk_vfree(buffer); + return rc; +} + +int mem_usage_get_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_memory_usage_get_s __user *uargs) +{ + _mali_osk_errcode_t err; + _mali_uk_profiling_memory_usage_get_s kargs; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_profiling_memory_usage_get_s))) { + return -EFAULT; + } + + kargs.ctx = (uintptr_t)session_data; + err = _mali_ukk_mem_usage_get(&kargs); + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + kargs.ctx = (uintptr_t)NULL; /* prevent kernel address to be returned to user space */ + if (0 != copy_to_user(uargs, &kargs, sizeof(_mali_uk_profiling_memory_usage_get_s))) { + return -EFAULT; + } + + return 0; +} + diff --git a/drivers/gpu/arm/utgard/linux/mali_ukk_pp.c b/drivers/gpu/arm/utgard/linux/mali_ukk_pp.c new file mode 100644 index 00000000000000..4c1c381cb90f9d --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_ukk_pp.c @@ -0,0 +1,105 @@ +/* + * Copyright (C) 2010, 2012-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#include /* file system operations */ +#include /* user space access */ + +#include "mali_ukk.h" +#include "mali_osk.h" +#include "mali_kernel_common.h" +#include "mali_session.h" +#include "mali_ukk_wrappers.h" + +int pp_start_job_wrapper(struct mali_session_data *session_data, _mali_uk_pp_start_job_s __user *uargs) +{ + _mali_osk_errcode_t err; + + /* If the job was started successfully, 0 is returned. If there was an error, but the job + * was started, we return -ENOENT. For anything else returned, the job was not started. */ + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + err = _mali_ukk_pp_start_job(session_data, uargs); + if (_MALI_OSK_ERR_OK != err) return map_errcode(err); + + return 0; +} + +int pp_and_gp_start_job_wrapper(struct mali_session_data *session_data, _mali_uk_pp_and_gp_start_job_s __user *uargs) +{ + _mali_osk_errcode_t err; + + /* If the jobs were started successfully, 0 is returned. If there was an error, but the + * jobs were started, we return -ENOENT. For anything else returned, the jobs were not + * started. */ + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + err = _mali_ukk_pp_and_gp_start_job(session_data, uargs); + if (_MALI_OSK_ERR_OK != err) return map_errcode(err); + + return 0; +} + +int pp_get_number_of_cores_wrapper(struct mali_session_data *session_data, _mali_uk_get_pp_number_of_cores_s __user *uargs) +{ + _mali_uk_get_pp_number_of_cores_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + kargs.ctx = (uintptr_t)session_data; + + err = _mali_ukk_get_pp_number_of_cores(&kargs); + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + kargs.ctx = (uintptr_t)NULL; /* prevent kernel address to be returned to user space */ + if (0 != copy_to_user(uargs, &kargs, sizeof(_mali_uk_get_pp_number_of_cores_s))) { + return -EFAULT; + } + + return 0; +} + +int pp_get_core_version_wrapper(struct mali_session_data *session_data, _mali_uk_get_pp_core_version_s __user *uargs) +{ + _mali_uk_get_pp_core_version_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + kargs.ctx = (uintptr_t)session_data; + err = _mali_ukk_get_pp_core_version(&kargs); + if (_MALI_OSK_ERR_OK != err) return map_errcode(err); + + if (0 != put_user(kargs.version, &uargs->version)) return -EFAULT; + + return 0; +} + +int pp_disable_wb_wrapper(struct mali_session_data *session_data, _mali_uk_pp_disable_wb_s __user *uargs) +{ + _mali_uk_pp_disable_wb_s kargs; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_pp_disable_wb_s))) return -EFAULT; + + kargs.ctx = (uintptr_t)session_data; + _mali_ukk_pp_job_disable_wb(&kargs); + + return 0; +} diff --git a/drivers/gpu/arm/utgard/linux/mali_ukk_profiling.c b/drivers/gpu/arm/utgard/linux/mali_ukk_profiling.c new file mode 100644 index 00000000000000..e84544dee07d9c --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_ukk_profiling.c @@ -0,0 +1,177 @@ +/* + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#include /* file system operations */ +#include /* user space access */ +#include + +#include "mali_ukk.h" +#include "mali_osk.h" +#include "mali_kernel_common.h" +#include "mali_session.h" +#include "mali_ukk_wrappers.h" + +int profiling_add_event_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_add_event_s __user *uargs) +{ + _mali_uk_profiling_add_event_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_profiling_add_event_s))) { + return -EFAULT; + } + + kargs.ctx = (uintptr_t)session_data; + err = _mali_ukk_profiling_add_event(&kargs); + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + return 0; +} + +int profiling_report_sw_counters_wrapper(struct mali_session_data *session_data, _mali_uk_sw_counters_report_s __user *uargs) +{ + _mali_uk_sw_counters_report_s kargs; + _mali_osk_errcode_t err; + u32 *counter_buffer; + u32 __user *counters; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_sw_counters_report_s))) { + return -EFAULT; + } + + /* make sure that kargs.num_counters is [at least somewhat] sane */ + if (kargs.num_counters > 10000) { + MALI_DEBUG_PRINT(1, ("User space attempted to allocate too many counters.\n")); + return -EINVAL; + } + + counter_buffer = (u32 *)kmalloc(sizeof(u32) * kargs.num_counters, GFP_KERNEL); + if (NULL == counter_buffer) { + return -ENOMEM; + } + + counters = (u32 *)(uintptr_t)kargs.counters; + + if (0 != copy_from_user(counter_buffer, counters, sizeof(u32) * kargs.num_counters)) { + kfree(counter_buffer); + return -EFAULT; + } + + kargs.ctx = (uintptr_t)session_data; + kargs.counters = (uintptr_t)counter_buffer; + + err = _mali_ukk_sw_counters_report(&kargs); + + kfree(counter_buffer); + + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + return 0; +} + +int profiling_get_stream_fd_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_stream_fd_get_s __user *uargs) +{ + _mali_uk_profiling_stream_fd_get_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_profiling_stream_fd_get_s))) { + return -EFAULT; + } + + kargs.ctx = (uintptr_t)session_data; + err = _mali_ukk_profiling_stream_fd_get(&kargs); + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + if (0 != copy_to_user(uargs, &kargs, sizeof(_mali_uk_profiling_stream_fd_get_s))) { + return -EFAULT; + } + + return 0; +} + +int profiling_control_set_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_control_set_s __user *uargs) +{ + _mali_uk_profiling_control_set_s kargs; + _mali_osk_errcode_t err; + u8 *kernel_control_data = NULL; + u8 *kernel_response_data = NULL; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + + if (0 != get_user(kargs.control_packet_size, &uargs->control_packet_size)) return -EFAULT; + if (0 != get_user(kargs.response_packet_size, &uargs->response_packet_size)) return -EFAULT; + + kargs.ctx = (uintptr_t)session_data; + + if (0 != kargs.control_packet_size) { + + kernel_control_data = _mali_osk_calloc(1, kargs.control_packet_size); + if (NULL == kernel_control_data) { + return -ENOMEM; + } + + MALI_DEBUG_ASSERT(0 != kargs.response_packet_size); + + kernel_response_data = _mali_osk_calloc(1, kargs.response_packet_size); + if (NULL == kernel_response_data) { + _mali_osk_free(kernel_control_data); + return -ENOMEM; + } + + kargs.control_packet_data = (uintptr_t)kernel_control_data; + kargs.response_packet_data = (uintptr_t)kernel_response_data; + + if (0 != copy_from_user((void *)(uintptr_t)kernel_control_data, (void *)(uintptr_t)uargs->control_packet_data, kargs.control_packet_size)) { + _mali_osk_free(kernel_control_data); + _mali_osk_free(kernel_response_data); + return -EFAULT; + } + + err = _mali_ukk_profiling_control_set(&kargs); + if (_MALI_OSK_ERR_OK != err) { + _mali_osk_free(kernel_control_data); + _mali_osk_free(kernel_response_data); + return map_errcode(err); + } + + if (0 != kargs.response_packet_size && 0 != copy_to_user(((void *)(uintptr_t)uargs->response_packet_data), ((void *)(uintptr_t)kargs.response_packet_data), kargs.response_packet_size)) { + _mali_osk_free(kernel_control_data); + _mali_osk_free(kernel_response_data); + return -EFAULT; + } + + if (0 != put_user(kargs.response_packet_size, &uargs->response_packet_size)) { + _mali_osk_free(kernel_control_data); + _mali_osk_free(kernel_response_data); + return -EFAULT; + } + + _mali_osk_free(kernel_control_data); + _mali_osk_free(kernel_response_data); + } else { + + err = _mali_ukk_profiling_control_set(&kargs); + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + } + return 0; +} diff --git a/drivers/gpu/arm/utgard/linux/mali_ukk_soft_job.c b/drivers/gpu/arm/utgard/linux/mali_ukk_soft_job.c new file mode 100644 index 00000000000000..11c70060e489d4 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_ukk_soft_job.c @@ -0,0 +1,90 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#include /* file system operations */ +#include /* user space access */ + +#include "mali_ukk.h" +#include "mali_osk.h" +#include "mali_kernel_common.h" +#include "mali_session.h" +#include "mali_ukk_wrappers.h" + +#include "mali_soft_job.h" +#include "mali_timeline.h" + +int soft_job_start_wrapper(struct mali_session_data *session, _mali_uk_soft_job_start_s __user *uargs) +{ + _mali_uk_soft_job_start_s kargs; + u32 type, point; + u64 user_job; + struct mali_timeline_fence fence; + struct mali_soft_job *job = NULL; + u32 __user *job_id_ptr = NULL; + + /* If the job was started successfully, 0 is returned. If there was an error, but the job + * was started, we return -ENOENT. For anything else returned, the job was not started. */ + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session, -EINVAL); + + MALI_DEBUG_ASSERT_POINTER(session->soft_job_system); + + if (0 != copy_from_user(&kargs, uargs, sizeof(kargs))) { + return -EFAULT; + } + + type = kargs.type; + user_job = kargs.user_job; + job_id_ptr = (u32 __user *)(uintptr_t)kargs.job_id_ptr; + + mali_timeline_fence_copy_uk_fence(&fence, &kargs.fence); + + if ((MALI_SOFT_JOB_TYPE_USER_SIGNALED != type) && (MALI_SOFT_JOB_TYPE_SELF_SIGNALED != type)) { + MALI_DEBUG_PRINT_ERROR(("Invalid soft job type specified\n")); + return -EINVAL; + } + + /* Create soft job. */ + job = mali_soft_job_create(session->soft_job_system, (enum mali_soft_job_type)type, user_job); + if (unlikely(NULL == job)) { + return map_errcode(_MALI_OSK_ERR_NOMEM); + } + + /* Write job id back to user space. */ + if (0 != put_user(job->id, job_id_ptr)) { + MALI_PRINT_ERROR(("Mali Soft Job: failed to put job id")); + mali_soft_job_destroy(job); + return map_errcode(_MALI_OSK_ERR_NOMEM); + } + + /* Start soft job. */ + point = mali_soft_job_start(job, &fence); + + if (0 != put_user(point, &uargs->point)) { + /* Let user space know that something failed after the job was started. */ + return -ENOENT; + } + + return 0; +} + +int soft_job_signal_wrapper(struct mali_session_data *session, _mali_uk_soft_job_signal_s __user *uargs) +{ + u32 job_id; + _mali_osk_errcode_t err; + + MALI_DEBUG_ASSERT_POINTER(session); + + if (0 != get_user(job_id, &uargs->job_id)) return -EFAULT; + + err = mali_soft_job_system_signal_job(session->soft_job_system, job_id); + + return map_errcode(err); +} diff --git a/drivers/gpu/arm/utgard/linux/mali_ukk_timeline.c b/drivers/gpu/arm/utgard/linux/mali_ukk_timeline.c new file mode 100644 index 00000000000000..484d4041c8694b --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_ukk_timeline.c @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2013, 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#include /* file system operations */ +#include /* user space access */ + +#include "mali_ukk.h" +#include "mali_osk.h" +#include "mali_kernel_common.h" +#include "mali_session.h" +#include "mali_ukk_wrappers.h" + +#include "mali_timeline.h" +#include "mali_timeline_fence_wait.h" +#include "mali_timeline_sync_fence.h" + +int timeline_get_latest_point_wrapper(struct mali_session_data *session, _mali_uk_timeline_get_latest_point_s __user *uargs) +{ + u32 val; + mali_timeline_id timeline; + mali_timeline_point point; + + MALI_DEBUG_ASSERT_POINTER(session); + + if (0 != get_user(val, &uargs->timeline)) return -EFAULT; + + if (MALI_UK_TIMELINE_MAX <= val) { + return -EINVAL; + } + + timeline = (mali_timeline_id)val; + + point = mali_timeline_system_get_latest_point(session->timeline_system, timeline); + + if (0 != put_user(point, &uargs->point)) return -EFAULT; + + return 0; +} + +int timeline_wait_wrapper(struct mali_session_data *session, _mali_uk_timeline_wait_s __user *uargs) +{ + u32 timeout, status; + mali_bool ret; + _mali_uk_fence_t uk_fence; + struct mali_timeline_fence fence; + + MALI_DEBUG_ASSERT_POINTER(session); + + if (0 != copy_from_user(&uk_fence, &uargs->fence, sizeof(_mali_uk_fence_t))) return -EFAULT; + if (0 != get_user(timeout, &uargs->timeout)) return -EFAULT; + + mali_timeline_fence_copy_uk_fence(&fence, &uk_fence); + + ret = mali_timeline_fence_wait(session->timeline_system, &fence, timeout); + status = (MALI_TRUE == ret ? 1 : 0); + + if (0 != put_user(status, &uargs->status)) return -EFAULT; + + return 0; +} + +int timeline_create_sync_fence_wrapper(struct mali_session_data *session, _mali_uk_timeline_create_sync_fence_s __user *uargs) +{ + s32 sync_fd = -1; + _mali_uk_fence_t uk_fence; + struct mali_timeline_fence fence; + + MALI_DEBUG_ASSERT_POINTER(session); + + if (0 != copy_from_user(&uk_fence, &uargs->fence, sizeof(_mali_uk_fence_t))) return -EFAULT; + mali_timeline_fence_copy_uk_fence(&fence, &uk_fence); + +#if defined(CONFIG_SYNC) + sync_fd = mali_timeline_sync_fence_create(session->timeline_system, &fence); +#else + sync_fd = -1; +#endif /* defined(CONFIG_SYNC) */ + + if (0 != put_user(sync_fd, &uargs->sync_fd)) return -EFAULT; + + return 0; +} diff --git a/drivers/gpu/arm/utgard/linux/mali_ukk_vsync.c b/drivers/gpu/arm/utgard/linux/mali_ukk_vsync.c new file mode 100644 index 00000000000000..487c2478df4d67 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_ukk_vsync.c @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2011-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#include /* file system operations */ +#include /* user space access */ + +#include "mali_ukk.h" +#include "mali_osk.h" +#include "mali_kernel_common.h" +#include "mali_session.h" +#include "mali_ukk_wrappers.h" + + +int vsync_event_report_wrapper(struct mali_session_data *session_data, _mali_uk_vsync_event_report_s __user *uargs) +{ + _mali_uk_vsync_event_report_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_vsync_event_report_s))) { + return -EFAULT; + } + + kargs.ctx = (uintptr_t)session_data; + err = _mali_ukk_vsync_event_report(&kargs); + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + return 0; +} + diff --git a/drivers/gpu/arm/utgard/linux/mali_ukk_wrappers.h b/drivers/gpu/arm/utgard/linux/mali_ukk_wrappers.h new file mode 100644 index 00000000000000..ac84e446bd1239 --- /dev/null +++ b/drivers/gpu/arm/utgard/linux/mali_ukk_wrappers.h @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_ukk_wrappers.h + * Defines the wrapper functions for each user-kernel function + */ + +#ifndef __MALI_UKK_WRAPPERS_H__ +#define __MALI_UKK_WRAPPERS_H__ + +#include "mali_uk_types.h" +#include "mali_osk.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int wait_for_notification_wrapper(struct mali_session_data *session_data, _mali_uk_wait_for_notification_s __user *uargs); +int get_api_version_wrapper(struct mali_session_data *session_data, _mali_uk_get_api_version_s __user *uargs); +int get_api_version_v2_wrapper(struct mali_session_data *session_data, _mali_uk_get_api_version_v2_s __user *uargs); +int get_user_settings_wrapper(struct mali_session_data *session_data, _mali_uk_get_user_settings_s __user *uargs); +int post_notification_wrapper(struct mali_session_data *session_data, _mali_uk_post_notification_s __user *uargs); +int request_high_priority_wrapper(struct mali_session_data *session_data, _mali_uk_request_high_priority_s __user *uargs); +int pending_submit_wrapper(struct mali_session_data *session_data, _mali_uk_pending_submit_s __user *uargs); + +int mem_alloc_wrapper(struct mali_session_data *session_data, _mali_uk_alloc_mem_s __user *uargs); +int mem_free_wrapper(struct mali_session_data *session_data, _mali_uk_free_mem_s __user *uargs); +int mem_bind_wrapper(struct mali_session_data *session_data, _mali_uk_bind_mem_s __user *uargs); +int mem_unbind_wrapper(struct mali_session_data *session_data, _mali_uk_unbind_mem_s __user *uargs); +int mem_cow_wrapper(struct mali_session_data *session_data, _mali_uk_cow_mem_s __user *uargs); +int mem_cow_modify_range_wrapper(struct mali_session_data *session_data, _mali_uk_cow_modify_range_s __user *uargs); +int mem_resize_mem_wrapper(struct mali_session_data *session_data, _mali_uk_mem_resize_s __user *uargs); +int mem_write_safe_wrapper(struct mali_session_data *session_data, _mali_uk_mem_write_safe_s __user *uargs); +int mem_query_mmu_page_table_dump_size_wrapper(struct mali_session_data *session_data, _mali_uk_query_mmu_page_table_dump_size_s __user *uargs); +int mem_dump_mmu_page_table_wrapper(struct mali_session_data *session_data, _mali_uk_dump_mmu_page_table_s __user *uargs); +int mem_usage_get_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_memory_usage_get_s __user *uargs); + +int timeline_get_latest_point_wrapper(struct mali_session_data *session, _mali_uk_timeline_get_latest_point_s __user *uargs); +int timeline_wait_wrapper(struct mali_session_data *session, _mali_uk_timeline_wait_s __user *uargs); +int timeline_create_sync_fence_wrapper(struct mali_session_data *session, _mali_uk_timeline_create_sync_fence_s __user *uargs); +int soft_job_start_wrapper(struct mali_session_data *session, _mali_uk_soft_job_start_s __user *uargs); +int soft_job_signal_wrapper(struct mali_session_data *session, _mali_uk_soft_job_signal_s __user *uargs); +int pp_start_job_wrapper(struct mali_session_data *session_data, _mali_uk_pp_start_job_s __user *uargs); +int pp_and_gp_start_job_wrapper(struct mali_session_data *session_data, _mali_uk_pp_and_gp_start_job_s __user *uargs); +int pp_get_number_of_cores_wrapper(struct mali_session_data *session_data, _mali_uk_get_pp_number_of_cores_s __user *uargs); +int pp_get_core_version_wrapper(struct mali_session_data *session_data, _mali_uk_get_pp_core_version_s __user *uargs); +int pp_disable_wb_wrapper(struct mali_session_data *session_data, _mali_uk_pp_disable_wb_s __user *uargs); +int gp_start_job_wrapper(struct mali_session_data *session_data, _mali_uk_gp_start_job_s __user *uargs); +int gp_get_number_of_cores_wrapper(struct mali_session_data *session_data, _mali_uk_get_gp_number_of_cores_s __user *uargs); +int gp_get_core_version_wrapper(struct mali_session_data *session_data, _mali_uk_get_gp_core_version_s __user *uargs); +int gp_suspend_response_wrapper(struct mali_session_data *session_data, _mali_uk_gp_suspend_response_s __user *uargs); + +int profiling_add_event_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_add_event_s __user *uargs); +int profiling_report_sw_counters_wrapper(struct mali_session_data *session_data, _mali_uk_sw_counters_report_s __user *uargs); +int profiling_get_stream_fd_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_stream_fd_get_s __user *uargs); +int profiling_control_set_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_control_set_s __user *uargs); + +int vsync_event_report_wrapper(struct mali_session_data *session_data, _mali_uk_vsync_event_report_s __user *uargs); + + +int map_errcode(_mali_osk_errcode_t err); + +#ifdef __cplusplus +} +#endif + +#endif /* __MALI_UKK_WRAPPERS_H__ */ diff --git a/drivers/gpu/arm/utgard/platform/arm/arm.c b/drivers/gpu/arm/utgard/platform/arm/arm.c new file mode 100644 index 00000000000000..41ad63c0793bf9 --- /dev/null +++ b/drivers/gpu/arm/utgard/platform/arm/arm.c @@ -0,0 +1,439 @@ +/* + * Copyright (C) 2010, 2012-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_platform.c + * Platform specific Mali driver functions for: + * - Realview Versatile platforms with ARM11 Mpcore and virtex 5. + * - Versatile Express platforms with ARM Cortex-A9 and virtex 6. + */ +#include +#include +#include +#ifdef CONFIG_PM_RUNTIME +#include +#endif +#include +#include +#include "mali_kernel_common.h" +#include +#include + +#include "arm_core_scaling.h" +#include "mali_executor.h" + + +static int mali_core_scaling_enable = 0; + +void mali_gpu_utilization_callback(struct mali_gpu_utilization_data *data); +static u32 mali_read_phys(u32 phys_addr); +#if defined(CONFIG_ARCH_REALVIEW) +static void mali_write_phys(u32 phys_addr, u32 value); +#endif + +#ifndef CONFIG_MALI_DT +static void mali_platform_device_release(struct device *device); + +#if defined(CONFIG_ARCH_VEXPRESS) + +#if defined(CONFIG_ARM64) +/* Juno + Mali-450 MP6 in V7 FPGA */ +static struct resource mali_gpu_resources_m450_mp6[] = { + MALI_GPU_RESOURCES_MALI450_MP6_PMU(0x6F040000, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200) +}; + +static struct resource mali_gpu_resources_m470_mp4[] = { + MALI_GPU_RESOURCES_MALI470_MP4_PMU(0x6F040000, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200) +}; + +static struct resource mali_gpu_resources_m470_mp3[] = { + MALI_GPU_RESOURCES_MALI470_MP3_PMU(0x6F040000, 200, 200, 200, 200, 200, 200, 200, 200, 200) +}; + +static struct resource mali_gpu_resources_m470_mp2[] = { + MALI_GPU_RESOURCES_MALI470_MP2_PMU(0x6F040000, 200, 200, 200, 200, 200, 200, 200) +}; + +static struct resource mali_gpu_resources_m470_mp1[] = { + MALI_GPU_RESOURCES_MALI470_MP1_PMU(0x6F040000, 200, 200, 200, 200, 200) +}; + +#else +static struct resource mali_gpu_resources_m450_mp8[] = { + MALI_GPU_RESOURCES_MALI450_MP8_PMU(0xFC040000, -1, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 68) +}; + +static struct resource mali_gpu_resources_m450_mp6[] = { + MALI_GPU_RESOURCES_MALI450_MP6_PMU(0xFC040000, -1, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 68) +}; + +static struct resource mali_gpu_resources_m450_mp4[] = { + MALI_GPU_RESOURCES_MALI450_MP4_PMU(0xFC040000, -1, 70, 70, 70, 70, 70, 70, 70, 70, 70, 68) +}; + +static struct resource mali_gpu_resources_m470_mp4[] = { + MALI_GPU_RESOURCES_MALI470_MP4_PMU(0xFC040000, -1, 70, 70, 70, 70, 70, 70, 70, 70, 70, 68) +}; +#endif /* CONFIG_ARM64 */ + +#elif defined(CONFIG_ARCH_REALVIEW) + +static struct resource mali_gpu_resources_m300[] = { + MALI_GPU_RESOURCES_MALI300_PMU(0xC0000000, -1, -1, -1, -1) +}; + +static struct resource mali_gpu_resources_m400_mp1[] = { + MALI_GPU_RESOURCES_MALI400_MP1_PMU(0xC0000000, -1, -1, -1, -1) +}; + +static struct resource mali_gpu_resources_m400_mp2[] = { + MALI_GPU_RESOURCES_MALI400_MP2_PMU(0xC0000000, -1, -1, -1, -1, -1, -1) +}; + +#endif +#endif + +static struct mali_gpu_device_data mali_gpu_data = { +#ifndef CONFIG_MALI_DT + .pmu_switch_delay = 0xFF, /* do not have to be this high on FPGA, but it is good for testing to have a delay */ + .max_job_runtime = 60000, /* 60 seconds */ +#if defined(CONFIG_ARCH_VEXPRESS) + .shared_mem_size = 256 * 1024 * 1024, /* 256MB */ +#endif +#endif + +#if defined(CONFIG_ARCH_REALVIEW) + .dedicated_mem_start = 0x80000000, /* Physical start address (use 0xD0000000 for old indirect setup) */ + .dedicated_mem_size = 0x10000000, /* 256MB */ +#endif +#if defined(CONFIG_ARM64) + /* Some framebuffer drivers get the framebuffer dynamically, such as through GEM, + * in which the memory resource can't be predicted in advance. + */ + .fb_start = 0x0, + .fb_size = 0xFFFFF000, +#else + .fb_start = 0xe0000000, + .fb_size = 0x01000000, +#endif + .control_interval = 1000, /* 1000ms */ + .utilization_callback = mali_gpu_utilization_callback, + .get_clock_info = NULL, + .get_freq = NULL, + .set_freq = NULL, +}; + +#ifndef CONFIG_MALI_DT +static struct platform_device mali_gpu_device = { + .name = MALI_GPU_NAME_UTGARD, + .id = 0, + .dev.release = mali_platform_device_release, + .dev.dma_mask = &mali_gpu_device.dev.coherent_dma_mask, + .dev.coherent_dma_mask = DMA_BIT_MASK(32), + + .dev.platform_data = &mali_gpu_data, +}; + +int mali_platform_device_register(void) +{ + int err = -1; + int num_pp_cores = 0; +#if defined(CONFIG_ARCH_REALVIEW) + u32 m400_gp_version; +#endif + + MALI_DEBUG_PRINT(4, ("mali_platform_device_register() called\n")); + + /* Detect present Mali GPU and connect the correct resources to the device */ +#if defined(CONFIG_ARCH_VEXPRESS) + +#if defined(CONFIG_ARM64) + mali_gpu_device.dev.archdata.dma_ops = dma_ops; + if ((mali_read_phys(0x6F000000) & 0x00600450) == 0x00600450) { + MALI_DEBUG_PRINT(4, ("Registering Mali-450 MP6 device\n")); + num_pp_cores = 6; + mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m450_mp6); + mali_gpu_device.resource = mali_gpu_resources_m450_mp6; + } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00400430) { + MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP4 device\n")); + num_pp_cores = 4; + mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m470_mp4); + mali_gpu_device.resource = mali_gpu_resources_m470_mp4; + } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00300430) { + MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP3 device\n")); + num_pp_cores = 3; + mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m470_mp3); + mali_gpu_device.resource = mali_gpu_resources_m470_mp3; + } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00200430) { + MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP2 device\n")); + num_pp_cores = 2; + mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m470_mp2); + mali_gpu_device.resource = mali_gpu_resources_m470_mp2; + } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00100430) { + MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP1 device\n")); + num_pp_cores = 1; + mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m470_mp1); + mali_gpu_device.resource = mali_gpu_resources_m470_mp1; + } +#else + if (mali_read_phys(0xFC000000) == 0x00000450) { + MALI_DEBUG_PRINT(4, ("Registering Mali-450 MP8 device\n")); + num_pp_cores = 8; + mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m450_mp8); + mali_gpu_device.resource = mali_gpu_resources_m450_mp8; + } else if (mali_read_phys(0xFC000000) == 0x40600450) { + MALI_DEBUG_PRINT(4, ("Registering Mali-450 MP6 device\n")); + num_pp_cores = 6; + mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m450_mp6); + mali_gpu_device.resource = mali_gpu_resources_m450_mp6; + } else if (mali_read_phys(0xFC000000) == 0x40400450) { + MALI_DEBUG_PRINT(4, ("Registering Mali-450 MP4 device\n")); + num_pp_cores = 4; + mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m450_mp4); + mali_gpu_device.resource = mali_gpu_resources_m450_mp4; + } else if (mali_read_phys(0xFC000000) == 0xFFFFFFFF) { + MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP4 device\n")); + num_pp_cores = 4; + mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m470_mp4); + mali_gpu_device.resource = mali_gpu_resources_m470_mp4; + } +#endif /* CONFIG_ARM64 */ + +#elif defined(CONFIG_ARCH_REALVIEW) + + m400_gp_version = mali_read_phys(0xC000006C); + if ((m400_gp_version & 0xFFFF0000) == 0x0C070000) { + MALI_DEBUG_PRINT(4, ("Registering Mali-300 device\n")); + num_pp_cores = 1; + mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m300); + mali_gpu_device.resource = mali_gpu_resources_m300; + mali_write_phys(0xC0010020, 0xA); /* Enable direct memory mapping for FPGA */ + } else if ((m400_gp_version & 0xFFFF0000) == 0x0B070000) { + u32 fpga_fw_version = mali_read_phys(0xC0010000); + if (fpga_fw_version == 0x130C008F || fpga_fw_version == 0x110C008F) { + /* Mali-400 MP1 r1p0 or r1p1 */ + MALI_DEBUG_PRINT(4, ("Registering Mali-400 MP1 device\n")); + num_pp_cores = 1; + mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m400_mp1); + mali_gpu_device.resource = mali_gpu_resources_m400_mp1; + mali_write_phys(0xC0010020, 0xA); /* Enable direct memory mapping for FPGA */ + } else if (fpga_fw_version == 0x130C000F) { + /* Mali-400 MP2 r1p1 */ + MALI_DEBUG_PRINT(4, ("Registering Mali-400 MP2 device\n")); + num_pp_cores = 2; + mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m400_mp2); + mali_gpu_device.resource = mali_gpu_resources_m400_mp2; + mali_write_phys(0xC0010020, 0xA); /* Enable direct memory mapping for FPGA */ + } + } + +#endif + /* Register the platform device */ + err = platform_device_register(&mali_gpu_device); + if (0 == err) { +#ifdef CONFIG_PM_RUNTIME +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) + pm_runtime_set_autosuspend_delay(&(mali_gpu_device.dev), 1000); + pm_runtime_use_autosuspend(&(mali_gpu_device.dev)); +#endif + pm_runtime_enable(&(mali_gpu_device.dev)); +#endif + MALI_DEBUG_ASSERT(0 < num_pp_cores); + mali_core_scaling_init(num_pp_cores); + + return 0; + } + + return err; +} + +void mali_platform_device_unregister(void) +{ + MALI_DEBUG_PRINT(4, ("mali_platform_device_unregister() called\n")); + + mali_core_scaling_term(); + platform_device_unregister(&mali_gpu_device); + + platform_device_put(&mali_gpu_device); + +#if defined(CONFIG_ARCH_REALVIEW) + mali_write_phys(0xC0010020, 0x9); /* Restore default (legacy) memory mapping */ +#endif +} + +static void mali_platform_device_release(struct device *device) +{ + MALI_DEBUG_PRINT(4, ("mali_platform_device_release() called\n")); +} + +#else /* CONFIG_MALI_DT */ +int mali_platform_device_init(struct platform_device *device) +{ + int num_pp_cores = 0; + int err = -1; +#if defined(CONFIG_ARCH_REALVIEW) + u32 m400_gp_version; +#endif + + /* Detect present Mali GPU and connect the correct resources to the device */ +#if defined(CONFIG_ARCH_VEXPRESS) + +#if defined(CONFIG_ARM64) + if ((mali_read_phys(0x6F000000) & 0x00600450) == 0x00600450) { + MALI_DEBUG_PRINT(4, ("Registering Mali-450 MP6 device\n")); + num_pp_cores = 6; + } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00400430) { + MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP4 device\n")); + num_pp_cores = 4; + } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00300430) { + MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP3 device\n")); + num_pp_cores = 3; + } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00200430) { + MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP2 device\n")); + num_pp_cores = 2; + } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00100430) { + MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP1 device\n")); + num_pp_cores = 1; + } +#else + if (mali_read_phys(0xFC000000) == 0x00000450) { + MALI_DEBUG_PRINT(4, ("Registering Mali-450 MP8 device\n")); + num_pp_cores = 8; + } else if (mali_read_phys(0xFC000000) == 0x40400450) { + MALI_DEBUG_PRINT(4, ("Registering Mali-450 MP4 device\n")); + num_pp_cores = 4; + } else if (mali_read_phys(0xFC000000) == 0xFFFFFFFF) { + MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP4 device\n")); + num_pp_cores = 4; + } +#endif + +#elif defined(CONFIG_ARCH_REALVIEW) + + m400_gp_version = mali_read_phys(0xC000006C); + if ((m400_gp_version & 0xFFFF0000) == 0x0C070000) { + MALI_DEBUG_PRINT(4, ("Registering Mali-300 device\n")); + num_pp_cores = 1; + mali_write_phys(0xC0010020, 0xA); /* Enable direct memory mapping for FPGA */ + } else if ((m400_gp_version & 0xFFFF0000) == 0x0B070000) { + u32 fpga_fw_version = mali_read_phys(0xC0010000); + if (fpga_fw_version == 0x130C008F || fpga_fw_version == 0x110C008F) { + /* Mali-400 MP1 r1p0 or r1p1 */ + MALI_DEBUG_PRINT(4, ("Registering Mali-400 MP1 device\n")); + num_pp_cores = 1; + mali_write_phys(0xC0010020, 0xA); /* Enable direct memory mapping for FPGA */ + } else if (fpga_fw_version == 0x130C000F) { + /* Mali-400 MP2 r1p1 */ + MALI_DEBUG_PRINT(4, ("Registering Mali-400 MP2 device\n")); + num_pp_cores = 2; + mali_write_phys(0xC0010020, 0xA); /* Enable direct memory mapping for FPGA */ + } + } +#endif + + /* After kernel 3.15 device tree will default set dev + * related parameters in of_platform_device_create_pdata. + * But kernel changes from version to version, + * For example 3.10 didn't include device->dev.dma_mask parameter setting, + * if we didn't include here will cause dma_mapping error, + * but in kernel 3.15 it include device->dev.dma_mask parameter setting, + * so it's better to set must need paramter by DDK itself. + */ + if (!device->dev.dma_mask) + device->dev.dma_mask = &device->dev.coherent_dma_mask; + device->dev.archdata.dma_ops = dma_ops; + + err = platform_device_add_data(device, &mali_gpu_data, sizeof(mali_gpu_data)); + + if (0 == err) { +#ifdef CONFIG_PM_RUNTIME +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) + pm_runtime_set_autosuspend_delay(&(device->dev), 1000); + pm_runtime_use_autosuspend(&(device->dev)); +#endif + pm_runtime_enable(&(device->dev)); +#endif + MALI_DEBUG_ASSERT(0 < num_pp_cores); + mali_core_scaling_init(num_pp_cores); + } + + return err; +} + +int mali_platform_device_deinit(struct platform_device *device) +{ + MALI_IGNORE(device); + + MALI_DEBUG_PRINT(4, ("mali_platform_device_deinit() called\n")); + + mali_core_scaling_term(); + +#if defined(CONFIG_ARCH_REALVIEW) + mali_write_phys(0xC0010020, 0x9); /* Restore default (legacy) memory mapping */ +#endif + + return 0; +} + +#endif /* CONFIG_MALI_DT */ + +static u32 mali_read_phys(u32 phys_addr) +{ + u32 phys_addr_page = phys_addr & 0xFFFFE000; + u32 phys_offset = phys_addr & 0x00001FFF; + u32 map_size = phys_offset + sizeof(u32); + u32 ret = 0xDEADBEEF; + void *mem_mapped = ioremap_nocache(phys_addr_page, map_size); + if (NULL != mem_mapped) { + ret = (u32)ioread32(((u8 *)mem_mapped) + phys_offset); + iounmap(mem_mapped); + } + + return ret; +} + +#if defined(CONFIG_ARCH_REALVIEW) +static void mali_write_phys(u32 phys_addr, u32 value) +{ + u32 phys_addr_page = phys_addr & 0xFFFFE000; + u32 phys_offset = phys_addr & 0x00001FFF; + u32 map_size = phys_offset + sizeof(u32); + void *mem_mapped = ioremap_nocache(phys_addr_page, map_size); + if (NULL != mem_mapped) { + iowrite32(value, ((u8 *)mem_mapped) + phys_offset); + iounmap(mem_mapped); + } +} +#endif + +static int param_set_core_scaling(const char *val, const struct kernel_param *kp) +{ + int ret = param_set_int(val, kp); + + if (1 == mali_core_scaling_enable) { + mali_core_scaling_sync(mali_executor_get_num_cores_enabled()); + } + return ret; +} + +static struct kernel_param_ops param_ops_core_scaling = { + .set = param_set_core_scaling, + .get = param_get_int, +}; + +module_param_cb(mali_core_scaling_enable, ¶m_ops_core_scaling, &mali_core_scaling_enable, 0644); +MODULE_PARM_DESC(mali_core_scaling_enable, "1 means to enable core scaling policy, 0 means to disable core scaling policy"); + +void mali_gpu_utilization_callback(struct mali_gpu_utilization_data *data) +{ + if (1 == mali_core_scaling_enable) { + mali_core_scaling_update(data); + } +} diff --git a/drivers/gpu/arm/utgard/platform/arm/arm_core_scaling.c b/drivers/gpu/arm/utgard/platform/arm/arm_core_scaling.c new file mode 100644 index 00000000000000..2c24742eb4de4b --- /dev/null +++ b/drivers/gpu/arm/utgard/platform/arm/arm_core_scaling.c @@ -0,0 +1,122 @@ +/* + * Copyright (C) 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file arm_core_scaling.c + * Example core scaling policy. + */ + +#include "arm_core_scaling.h" + +#include +#include "mali_kernel_common.h" + +#include + +static int num_cores_total; +static int num_cores_enabled; + +static struct work_struct wq_work; + +static void set_num_cores(struct work_struct *work) +{ + int err = mali_perf_set_num_pp_cores(num_cores_enabled); + MALI_DEBUG_ASSERT(0 == err); + MALI_IGNORE(err); +} + +static void enable_one_core(void) +{ + if (num_cores_enabled < num_cores_total) { + ++num_cores_enabled; + schedule_work(&wq_work); + MALI_DEBUG_PRINT(3, ("Core scaling: Enabling one more core\n")); + } + + MALI_DEBUG_ASSERT(1 <= num_cores_enabled); + MALI_DEBUG_ASSERT(num_cores_total >= num_cores_enabled); +} + +static void disable_one_core(void) +{ + if (1 < num_cores_enabled) { + --num_cores_enabled; + schedule_work(&wq_work); + MALI_DEBUG_PRINT(3, ("Core scaling: Disabling one core\n")); + } + + MALI_DEBUG_ASSERT(1 <= num_cores_enabled); + MALI_DEBUG_ASSERT(num_cores_total >= num_cores_enabled); +} + +static void enable_max_num_cores(void) +{ + if (num_cores_enabled < num_cores_total) { + num_cores_enabled = num_cores_total; + schedule_work(&wq_work); + MALI_DEBUG_PRINT(3, ("Core scaling: Enabling maximum number of cores\n")); + } + + MALI_DEBUG_ASSERT(num_cores_total == num_cores_enabled); +} + +void mali_core_scaling_init(int num_pp_cores) +{ + INIT_WORK(&wq_work, set_num_cores); + + num_cores_total = num_pp_cores; + num_cores_enabled = num_pp_cores; + + /* NOTE: Mali is not fully initialized at this point. */ +} + +void mali_core_scaling_sync(int num_cores) +{ + num_cores_enabled = num_cores; +} + +void mali_core_scaling_term(void) +{ + flush_scheduled_work(); +} + +#define PERCENT_OF(percent, max) ((int) ((percent)*(max)/100.0 + 0.5)) + +void mali_core_scaling_update(struct mali_gpu_utilization_data *data) +{ + /* + * This function implements a very trivial PP core scaling algorithm. + * + * It is _NOT_ of production quality. + * The only intention behind this algorithm is to exercise and test the + * core scaling functionality of the driver. + * It is _NOT_ tuned for neither power saving nor performance! + * + * Other metrics than PP utilization need to be considered as well + * in order to make a good core scaling algorithm. + */ + + MALI_DEBUG_PRINT(3, ("Utilization: (%3d, %3d, %3d), cores enabled: %d/%d\n", data->utilization_gpu, data->utilization_gp, data->utilization_pp, num_cores_enabled, num_cores_total)); + + /* NOTE: this function is normally called directly from the utilization callback which is in + * timer context. */ + + if (PERCENT_OF(90, 256) < data->utilization_pp) { + enable_max_num_cores(); + } else if (PERCENT_OF(50, 256) < data->utilization_pp) { + enable_one_core(); + } else if (PERCENT_OF(40, 256) < data->utilization_pp) { + /* do nothing */ + } else if (PERCENT_OF(0, 256) < data->utilization_pp) { + disable_one_core(); + } else { + /* do nothing */ + } +} diff --git a/drivers/gpu/arm/utgard/platform/arm/arm_core_scaling.h b/drivers/gpu/arm/utgard/platform/arm/arm_core_scaling.h new file mode 100644 index 00000000000000..325b5b1c6894c0 --- /dev/null +++ b/drivers/gpu/arm/utgard/platform/arm/arm_core_scaling.h @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2013, 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file arm_core_scaling.h + * Example core scaling policy. + */ + +#ifndef __ARM_CORE_SCALING_H__ +#define __ARM_CORE_SCALING_H__ + +struct mali_gpu_utilization_data; + +/** + * Initialize core scaling policy. + * + * @note The core scaling policy will assume that all PP cores are on initially. + * + * @param num_pp_cores Total number of PP cores. + */ +void mali_core_scaling_init(int num_pp_cores); + +/** + * Terminate core scaling policy. + */ +void mali_core_scaling_term(void); + +/** + * Update core scaling policy with new utilization data. + * + * @param data Utilization data. + */ +void mali_core_scaling_update(struct mali_gpu_utilization_data *data); + +void mali_core_scaling_sync(int num_cores); + +#endif /* __ARM_CORE_SCALING_H__ */ diff --git a/drivers/gpu/arm/utgard/platform/hikey/mali_hikey.c b/drivers/gpu/arm/utgard/platform/hikey/mali_hikey.c new file mode 100644 index 00000000000000..b53672fca45546 --- /dev/null +++ b/drivers/gpu/arm/utgard/platform/hikey/mali_hikey.c @@ -0,0 +1,638 @@ +/* + * Copyright (C) 2014 Hisilicon Co. Ltd. + * Copyright (C) 2015 ARM Limited + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + * + */ + +/** + * @file mali_hikey.c + * HiKey platform specific Mali driver functions. + */ + +/* Set to 1 to enable ION (not tested yet). */ +#define HISI6220_USE_ION 0 + +#define pr_fmt(fmt) "Mali: HiKey: " fmt + +#include +#include +#include +#include +#include +#include + +#ifdef CONFIG_PM_RUNTIME +#include +#endif +#include +#include +#include +#include +#if HISI6220_USE_ION +#include +#endif +#include + +#include + +#include "mali_kernel_common.h" +#include "mali_osk.h" +#include "mali_hikey_hi6220_registers_gpu.h" + +#define MALI_GPU_MHZ 1000000 +#define MALI_IRQ_ID 142 +#define MALI_FRAME_BUFFER_ADDR 0x3F100000 +#define MALI_FRAME_BUFFER_SIZE 0x00708000 + +#define MALI_CALC_REG_MASK(bit_start, bit_end) \ + (((0x1 << (bit_end - bit_start + 1)) - 1) << bit_start) + +enum mali_core_type { + MALI_CORE_400_MP1 = 0, + MALI_CORE_400_MP2 = 1, + MALI_CORE_450_MP4 = 2, + MALI_CORE_TYPE_MAX +}; + +enum mali_power_mode { + MALI_POWER_MODE_ON, /**< Power on */ + MALI_POWER_MODE_LIGHT_SLEEP, /**< Idle for a short or PM suspend */ + MALI_POWER_MODE_DEEP_SLEEP, /**< Idle for a long or OS suspend */ +}; + +struct mali_soc_remap_addr_table { + u8 *soc_media_sctrl_base_addr; + u8 *soc_ao_sctrl_base_addr; + u8 *soc_peri_sctrl_base_addr; + u8 *soc_pmctl_base_addr; +}; + +static struct clk *mali_clk_g3d; +static struct clk *mali_pclk_g3d; +static struct device_node *mali_np; +static bool mali_gpu_power_status; + +static struct resource mali_gpu_resources_m450_mp4[] = { + MALI_GPU_RESOURCES_MALI450_MP4( + SOC_G3D_S_BASE_ADDR, MALI_IRQ_ID, MALI_IRQ_ID, MALI_IRQ_ID, + MALI_IRQ_ID, MALI_IRQ_ID, MALI_IRQ_ID, MALI_IRQ_ID, + MALI_IRQ_ID, MALI_IRQ_ID, MALI_IRQ_ID, MALI_IRQ_ID) +}; + +static struct mali_soc_remap_addr_table *mali_soc_addr_table; + +static void mali_reg_writel(u8 *base_addr, unsigned int reg_offset, + unsigned char start_bit, unsigned char end_bit, + unsigned int val) +{ + int read_val; + unsigned long flags; + static DEFINE_SPINLOCK(reg_lock); + void __iomem *addr; + + WARN_ON(!base_addr); + + addr = base_addr + reg_offset; + spin_lock_irqsave(®_lock, flags); + read_val = readl(addr) & ~(MALI_CALC_REG_MASK(start_bit, end_bit)); + read_val |= (MALI_CALC_REG_MASK(start_bit, end_bit) + & (val << start_bit)); + writel(read_val, addr); + spin_unlock_irqrestore(®_lock, flags); +} + +static unsigned int mali_reg_readl(u8 *base_addr, unsigned int reg_offset, + unsigned char start_bit, + unsigned char end_bit) +{ + unsigned int val; + + WARN_ON(!base_addr); + + val = readl((void __iomem *)(base_addr + reg_offset)); + val &= MALI_CALC_REG_MASK(start_bit, end_bit); + + return val >> start_bit; +} + +static int mali_clock_on(void) +{ + u32 core_freq = 0; + u32 pclk_freq = 0; + int stat; + + stat = clk_prepare_enable(mali_pclk_g3d); + if (stat) + return stat; + + stat = of_property_read_u32(mali_np, "pclk_freq", &pclk_freq); + if (stat) + return stat; + + stat = clk_set_rate(mali_pclk_g3d, pclk_freq * MALI_GPU_MHZ); + if (stat) + return stat; + + stat = of_property_read_u32(mali_np, "mali_def_freq", &core_freq); + if (stat) + return stat; + + stat = clk_set_rate(mali_clk_g3d, core_freq * MALI_GPU_MHZ); + if (stat) + return stat; + + stat = clk_prepare_enable(mali_clk_g3d); + if (stat) + return stat; + + mali_reg_writel(mali_soc_addr_table->soc_media_sctrl_base_addr, + SOC_MEDIA_SCTRL_SC_MEDIA_CLKDIS_ADDR(0), 17, 17, 1); + + return 0; +} + +static void mali_clock_off(void) +{ + clk_disable_unprepare(mali_clk_g3d); + clk_disable_unprepare(mali_pclk_g3d); +} + +static int mali_domain_powerup_finish(void) +{ + unsigned int ret; + + mali_reg_writel(mali_soc_addr_table->soc_ao_sctrl_base_addr, + SOC_AO_SCTRL_SC_PW_RSTDIS0_ADDR(0), 1, 1, 1); + ret = mali_reg_readl(mali_soc_addr_table->soc_ao_sctrl_base_addr, + SOC_AO_SCTRL_SC_PW_RST_STAT0_ADDR(0), 1, 1); + if (ret != 0) { + pr_err("SET SC_PW_RSTDIS0 failed!\n"); + return -EFAULT; + } + + mali_reg_writel(mali_soc_addr_table->soc_ao_sctrl_base_addr, + SOC_AO_SCTRL_SC_PW_ISODIS0_ADDR(0), 1, 1, 1); + ret = mali_reg_readl(mali_soc_addr_table->soc_ao_sctrl_base_addr, + SOC_AO_SCTRL_SC_PW_ISO_STAT0_ADDR(0), 1, 1); + if (ret != 0) { + pr_err("SET SC_PW_ISODIS0 failed!\n"); + return -EFAULT; + } + + mali_reg_writel(mali_soc_addr_table->soc_ao_sctrl_base_addr, + SOC_AO_SCTRL_SC_PW_CLKEN0_ADDR(0), 1, 1, 1); + ret = mali_reg_readl(mali_soc_addr_table->soc_ao_sctrl_base_addr, + SOC_AO_SCTRL_SC_PW_CLK_STAT0_ADDR(0), 1, 1); + if (ret != 1) { + pr_err("SET SC_PW_CLKEN0 failed!\n"); + return -EFAULT; + } + + mali_reg_writel(mali_soc_addr_table->soc_media_sctrl_base_addr, + SOC_MEDIA_SCTRL_SC_MEDIA_RSTDIS_ADDR(0), 0, 0, 1); + ret = mali_reg_readl(mali_soc_addr_table->soc_media_sctrl_base_addr, + SOC_MEDIA_SCTRL_SC_MEDIA_RST_STAT_ADDR(0), 0, 0); + if (ret != 0) { + pr_err("SET SC_MEDIA_RSTDIS failed!\n"); + return -EFAULT; + } + + return 0; +} + +static int mali_platform_powerup(void) +{ + int stat; + + if (mali_gpu_power_status) + return 0; + + stat = mali_clock_on(); + if (stat) + return stat; + + stat = mali_domain_powerup_finish(); + if (stat) + return stat; + + mali_gpu_power_status = true; + + return 0; +} + +static int mali_regulator_disable(void) +{ + mali_reg_writel(mali_soc_addr_table->soc_media_sctrl_base_addr, + SOC_MEDIA_SCTRL_SC_MEDIA_RSTEN_ADDR(0), 0, 0, 1); + mali_reg_writel(mali_soc_addr_table->soc_ao_sctrl_base_addr, + SOC_AO_SCTRL_SC_PW_CLKDIS0_ADDR(0), 1, 1, 1); + mali_reg_writel(mali_soc_addr_table->soc_ao_sctrl_base_addr, + SOC_AO_SCTRL_SC_PW_ISOEN0_ADDR(0), 1, 1, 1); + mali_reg_writel(mali_soc_addr_table->soc_ao_sctrl_base_addr, + SOC_AO_SCTRL_SC_PW_RSTEN0_ADDR(0), 1, 1, 1); + + return 0; +} + +static int mali_platform_powerdown(void) +{ + int stat; + + if (!mali_gpu_power_status) + return 0; + + stat = mali_regulator_disable(); + if (stat) + return stat; + + mali_clock_off(); + mali_gpu_power_status = false; + + return 0; +} + +static int mali_platform_power_mode_change(enum mali_power_mode power_mode) +{ + int stat; + + switch (power_mode) { + case MALI_POWER_MODE_ON: + stat = mali_platform_powerup(); + break; + case MALI_POWER_MODE_LIGHT_SLEEP: + case MALI_POWER_MODE_DEEP_SLEEP: + stat = mali_platform_powerdown(); + break; + default: + pr_err("Invalid power mode\n"); + stat = -EINVAL; + break; + } + + return stat; +} + +static int mali_os_suspend(struct device *device) +{ + int stat; + + if (device->driver && + device->driver->pm && + device->driver->pm->suspend) { + stat = device->driver->pm->suspend(device); + } else { + stat = 0; + } + + if (stat) + return stat; + + return mali_platform_power_mode_change(MALI_POWER_MODE_DEEP_SLEEP); +} + +static int mali_os_resume(struct device *device) +{ + int stat; + + stat = mali_platform_power_mode_change(MALI_POWER_MODE_ON); + if (stat) + return stat; + + if (device->driver && + device->driver->pm && + device->driver->pm->resume) { + stat = device->driver->pm->resume(device); + } + + return stat; +} + +static int mali_os_freeze(struct device *device) +{ + int stat; + + if (device->driver && + device->driver->pm && + device->driver->pm->freeze) { + stat = device->driver->pm->freeze(device); + } else { + stat = 0; + } + + return stat; +} + +static int mali_os_thaw(struct device *device) +{ + int stat; + + if (device->driver && + device->driver->pm && + device->driver->pm->thaw) { + stat = device->driver->pm->thaw(device); + } else { + stat = 0; + } + + return stat; +} + +#ifdef CONFIG_PM_RUNTIME +static int mali_runtime_suspend(struct device *device) +{ + int stat; + + if (device->driver && + device->driver->pm && + device->driver->pm->runtime_suspend) { + stat = device->driver->pm->runtime_suspend(device); + } else { + stat = 0; + } + + if (stat) + return stat; + + return mali_platform_power_mode_change(MALI_POWER_MODE_LIGHT_SLEEP); +} + +static int mali_runtime_resume(struct device *device) +{ + int stat; + + stat = mali_platform_power_mode_change(MALI_POWER_MODE_ON); + if (stat) + return stat; + + if (device->driver && + device->driver->pm && + device->driver->pm->runtime_resume) { + stat = device->driver->pm->runtime_resume(device); + } + + return stat; +} + +static int mali_runtime_idle(struct device *device) +{ + int stat; + + if (device->driver && + device->driver->pm && + device->driver->pm->runtime_idle) { + stat = device->driver->pm->runtime_idle(device); + } else { + stat = 0; + } + + if (stat) + return stat; + + return pm_runtime_suspend(device); +} +#endif + +static int init_mali_clock_regulator(struct platform_device *pdev) +{ + int stat, ret; + + BUG_ON(mali_clk_g3d || mali_pclk_g3d); + + mali_gpu_power_status = true; + + /* clk init */ + mali_clk_g3d = clk_get(&pdev->dev, "clk_g3d"); + if (IS_ERR(mali_clk_g3d)) { + pr_err("failed to get source CLK_G3D\n"); + return -ENODEV; + } + + mali_pclk_g3d = clk_get(&pdev->dev, "pclk_g3d"); + if (IS_ERR(mali_pclk_g3d)) { + pr_err("failed to get source PCLK_G3D\n"); + return -ENODEV; + } + + ret = mali_reg_readl(mali_soc_addr_table->soc_peri_sctrl_base_addr, + SOC_PERI_SCTRL_SC_PERIPH_CLKSTAT12_ADDR(0), + 10, 10); + if (ret != 1) { + mali_reg_writel(mali_soc_addr_table->soc_peri_sctrl_base_addr, + SOC_PERI_SCTRL_SC_PERIPH_CLKEN12_ADDR(0), + 10, 10, 1); + ret = mali_reg_readl( + mali_soc_addr_table->soc_peri_sctrl_base_addr, + SOC_PERI_SCTRL_SC_PERIPH_CLKSTAT12_ADDR(0), 10, 10); + if (ret != 1) { + pr_err("SET SC_PERIPH_CLKEN12 failed!\n"); + return -EFAULT; + } + } + + stat = mali_clock_on(); + if (stat) + return stat; + + mali_reg_writel(mali_soc_addr_table->soc_media_sctrl_base_addr, + SOC_MEDIA_SCTRL_SC_MEDIA_CLKCFG2_ADDR(0), 15, 15, 1); + ret = mali_reg_readl(mali_soc_addr_table->soc_media_sctrl_base_addr, + SOC_MEDIA_SCTRL_SC_MEDIA_CLKCFG2_ADDR(0), 15, 15); + if (ret != 1) { + pr_err("SET SC_MEDIA_CLKCFG2 failed!\n"); + return -EFAULT; + } + + return mali_domain_powerup_finish(); +} + +static int deinit_mali_clock_regulator(void) +{ + int stat; + + BUG_ON(!mali_clk_g3d || !mali_pclk_g3d); + + stat = mali_platform_powerdown(); + if (stat) + return stat; + + clk_put(mali_clk_g3d); + mali_clk_g3d = NULL; + clk_put(mali_pclk_g3d); + mali_pclk_g3d = NULL; + + return 0; +} + +static struct mali_gpu_device_data mali_gpu_data = { + .shared_mem_size = 1024 * 1024 * 1024, /* 1024MB */ + .fb_start = MALI_FRAME_BUFFER_ADDR, + .fb_size = MALI_FRAME_BUFFER_SIZE, + .max_job_runtime = 2000, /* 2 seconds time out */ + .control_interval = 50, /* 50ms */ +#ifdef CONFIG_MALI_DVFS + .utilization_callback = mali_gpu_utilization_proc, +#endif +}; + +static const struct dev_pm_ops mali_gpu_device_type_pm_ops = { + .suspend = mali_os_suspend, + .resume = mali_os_resume, + .freeze = mali_os_freeze, + .thaw = mali_os_thaw, +#ifdef CONFIG_PM_RUNTIME + .runtime_suspend = mali_runtime_suspend, + .runtime_resume = mali_runtime_resume, + .runtime_idle = mali_runtime_idle, +#endif +}; + +static struct device_type mali_gpu_device_device_type = { + .pm = &mali_gpu_device_type_pm_ops, +}; + +static enum mali_core_type mali_get_gpu_type(void) +{ + u32 gpu_type = MALI_CORE_TYPE_MAX; + int err = of_property_read_u32(mali_np, "mali_type", &gpu_type); + + if (err) { + pr_err("failed to read mali_type from device tree\n"); + return -EFAULT; + } + + return gpu_type; +} + +#if HISI6220_USE_ION +static int mali_ion_mem_init(void) +{ + struct ion_heap_info_data mem_data; + + if (hisi_ion_get_heap_info(ION_FB_HEAP_ID, &mem_data)) { + pr_err("Failed to get ION_FB_HEAP_ID\n"); + return -EFAULT; + } + + if (mem_data.heap_size == 0) { + pr_err("fb size is 0\n"); + return -EINVAL; + } + + mali_gpu_data.fb_size = mem_data.heap_size; + mali_gpu_data.fb_start = (unsigned long)(mem_data.heap_phy); + pr_debug("fb_size=0x%x, fb_start=0x%x\n", + mali_gpu_data.fb_size, mali_gpu_data.fb_start); + + return 0; +} +#endif + +static int mali_remap_soc_addr(void) +{ + BUG_ON(mali_soc_addr_table); + + mali_soc_addr_table = kmalloc(sizeof(struct mali_soc_remap_addr_table), + GFP_KERNEL); + if (!mali_soc_addr_table) + return -ENOMEM; + + mali_soc_addr_table->soc_media_sctrl_base_addr = + ioremap(SOC_MEDIA_SCTRL_BASE_ADDR, REG_MEDIA_SC_IOSIZE); + mali_soc_addr_table->soc_ao_sctrl_base_addr = + ioremap(SOC_AO_SCTRL_BASE_ADDR, REG_SC_ON_IOSIZE); + mali_soc_addr_table->soc_peri_sctrl_base_addr = + ioremap(SOC_PERI_SCTRL_BASE_ADDR, REG_SC_OFF_IOSIZE); + mali_soc_addr_table->soc_pmctl_base_addr = + ioremap(SOC_PMCTRL_BASE_ADDR, REG_PMCTRL_IOSIZE); + + if (!mali_soc_addr_table->soc_media_sctrl_base_addr + || !mali_soc_addr_table->soc_ao_sctrl_base_addr + || !mali_soc_addr_table->soc_peri_sctrl_base_addr + || !mali_soc_addr_table->soc_pmctl_base_addr) { + pr_err("Failed to remap SoC addresses\n"); + return -ENOMEM; + } + + return 0; +} + +static void mali_unmap_soc_addr(void) +{ + iounmap((void __iomem *)mali_soc_addr_table->soc_media_sctrl_base_addr); + iounmap((void __iomem *)mali_soc_addr_table->soc_ao_sctrl_base_addr); + iounmap((void __iomem *)mali_soc_addr_table->soc_peri_sctrl_base_addr); + iounmap((void __iomem *)mali_soc_addr_table->soc_pmctl_base_addr); + kfree(mali_soc_addr_table); + mali_soc_addr_table = NULL; +} + +int mali_platform_device_init(struct platform_device *pdev) +{ + int stat; + int irq, i; + +#if HISI6220_USE_ION + stat = mali_ion_mem_init(); + if (stat) + return stat; +#endif + + stat = mali_remap_soc_addr(); + if (stat) + return stat; + + pdev->dev.coherent_dma_mask = DMA_BIT_MASK(32); + pdev->dev.type = &mali_gpu_device_device_type; + pdev->dev.platform_data = &mali_gpu_data; + pdev->dev.dma_mask = &pdev->dev.coherent_dma_mask; + mali_np = pdev->dev.of_node; + + if (mali_get_gpu_type() != MALI_CORE_450_MP4) { + pr_err("Unexpected GPU type\n"); + return -EINVAL; + } + + /* + * We need to use DT to get the irq domain, so rewrite the static + * table with the irq given from platform_get_irq(). + */ + irq = platform_get_irq(pdev, 0); + for (i = 0; i < ARRAY_SIZE(mali_gpu_resources_m450_mp4); i++) { + if (IORESOURCE_IRQ & mali_gpu_resources_m450_mp4[i].flags) { + mali_gpu_resources_m450_mp4[i].start = irq; + mali_gpu_resources_m450_mp4[i].end = irq; + } + } + pdev->num_resources = ARRAY_SIZE(mali_gpu_resources_m450_mp4); + pdev->resource = mali_gpu_resources_m450_mp4; + + stat = init_mali_clock_regulator(pdev); + if (stat) + return stat; + +#ifdef CONFIG_PM_RUNTIME + pm_runtime_set_autosuspend_delay(&(pdev->dev), 1); + pm_runtime_use_autosuspend(&(pdev->dev)); + pm_runtime_enable(&pdev->dev); +#endif + + return 0; +} + +int mali_platform_device_deinit(void) +{ + int stat; + + stat = deinit_mali_clock_regulator(); + if (stat) + return stat; + + mali_unmap_soc_addr(); + + return 0; +} diff --git a/drivers/gpu/arm/utgard/platform/hikey/mali_hikey_hi6220_registers_gpu.h b/drivers/gpu/arm/utgard/platform/hikey/mali_hikey_hi6220_registers_gpu.h new file mode 100644 index 00000000000000..0bdf4a0482fdcf --- /dev/null +++ b/drivers/gpu/arm/utgard/platform/hikey/mali_hikey_hi6220_registers_gpu.h @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2014 Hisilicon Co. Ltd. + * Copyright (C) 2015 ARM Ltd. + * + * Author: Xuzixin + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + * + */ + +#ifndef MALI_HIKEY_HI6220_REGISTERS_GPU_H +#define MALI_HIKEY_HI6220_REGISTERS_GPU_H 1 + +#include + +#define SOC_G3D_S_BASE_ADDR 0xF4080000 /* G3D ctrl base addr */ +#define SOC_MEDIA_SCTRL_BASE_ADDR 0xF4410000 /* media ctrl base addr */ +#define REG_MEDIA_SC_IOSIZE PAGE_ALIGN(SZ_4K) +#define SOC_PMCTRL_BASE_ADDR 0xF7032000 /* pm ctrl base addr */ +#define REG_PMCTRL_IOSIZE PAGE_ALIGN(SZ_4K) +#define SOC_AO_SCTRL_BASE_ADDR 0xF7800000 /* ao ctrl base addr */ +#define SOC_PERI_SCTRL_BASE_ADDR 0xF7030000 /* peri ctrl base addr */ +#define REG_SC_ON_IOSIZE PAGE_ALIGN(SZ_8K) +#define REG_SC_OFF_IOSIZE PAGE_ALIGN(SZ_4K) + +/* ---------------------------------------------------------------------------- + * MEDIA SCTRL + */ + +#define SOC_MEDIA_SCTRL_SC_MEDIA_SUBSYS_CTRL5_ADDR(base) ((base) + (0x51C)) +#define SOC_MEDIA_SCTRL_SC_MEDIA_CLKCFG0_ADDR(base) ((base) + (0xCBC)) +#define SOC_MEDIA_SCTRL_SC_MEDIA_CLKCFG2_ADDR(base) ((base) + (0xCC4)) +#define SOC_MEDIA_SCTRL_SC_MEDIA_CLKEN_ADDR(base) ((base) + (0x520)) +#define SOC_MEDIA_SCTRL_SC_MEDIA_CLKDIS_ADDR(base) ((base) + (0x524)) +#define SOC_MEDIA_SCTRL_SC_MEDIA_RSTEN_ADDR(base) ((base) + (0x52C)) +#define SOC_MEDIA_SCTRL_SC_MEDIA_RSTDIS_ADDR(base) ((base) + (0x530)) +#define SOC_MEDIA_SCTRL_SC_MEDIA_RST_STAT_ADDR(base) ((base) + (0x534)) + +/* ---------------------------------------------------------------------------- + * AO SCTRL,only bit 1 is necessary for GPU. + */ + +#define SOC_AO_SCTRL_SC_PW_CLKEN0_ADDR(base) ((base) + (0x800)) +#define SOC_AO_SCTRL_SC_PW_CLKDIS0_ADDR(base) ((base) + (0x804)) +#define SOC_AO_SCTRL_SC_PW_CLK_STAT0_ADDR(base) ((base) + (0x808)) +#define SOC_AO_SCTRL_SC_PW_RSTEN0_ADDR(base) ((base) + (0x810)) +#define SOC_AO_SCTRL_SC_PW_RSTDIS0_ADDR(base) ((base) + (0x814)) +#define SOC_AO_SCTRL_SC_PW_RST_STAT0_ADDR(base) ((base) + (0x818)) +#define SOC_AO_SCTRL_SC_PW_ISOEN0_ADDR(base) ((base) + (0x820)) +#define SOC_AO_SCTRL_SC_PW_ISODIS0_ADDR(base) ((base) + (0x824)) +#define SOC_AO_SCTRL_SC_PW_ISO_STAT0_ADDR(base) ((base) + (0x828)) +#define SOC_AO_SCTRL_SC_PW_MTCMOS_EN0_ADDR(base) ((base) + (0x830)) +#define SOC_AO_SCTRL_SC_PW_MTCMOS_DIS0_ADDR(base) ((base) + (0x834)) +#define SOC_AO_SCTRL_SC_PW_MTCMOS_STAT0_ADDR(base) ((base) + (0x838)) + +/* ---------------------------------------------------------------------------- + * PERI SCTRL,only bit 10 is necessary for GPU. + */ + +#define SOC_PERI_SCTRL_SC_PERIPH_CLKEN12_ADDR(base) ((base) + (0x270)) +#define SOC_PERI_SCTRL_SC_PERIPH_CLKSTAT12_ADDR(base) ((base) + (0x278)) + +#endif /* MALI_HIKEY_HI6220_REGISTERS_GPU_H */ diff --git a/drivers/gpu/arm/utgard/readme.txt b/drivers/gpu/arm/utgard/readme.txt new file mode 100644 index 00000000000000..6785ac933b38b1 --- /dev/null +++ b/drivers/gpu/arm/utgard/readme.txt @@ -0,0 +1,28 @@ +Building the Mali Device Driver for Linux +----------------------------------------- + +Build the Mali Device Driver for Linux by running the following make command: + +KDIR= USING_UMP= BUILD= make + +where + kdir_path: Path to your Linux Kernel directory + ump_option: 1 = Enable UMP support(*) + 0 = disable UMP support + build_option: debug = debug build of driver + release = release build of driver + +(*) For newer Linux Kernels, the Module.symvers file for the UMP device driver + must be available. The UMP_SYMVERS_FILE variable in the Makefile should + point to this file. This file is generated when the UMP driver is built. + +The result will be a mali.ko file, which can be loaded into the Linux kernel +by using the insmod command. + +Use of UMP is not recommended. The dma-buf API in the Linux kernel has +replaced UMP. The Mali Device Driver will be built with dma-buf support if the +kernel config includes enabled dma-buf. + +The kernel needs to be provided with a platform_device struct for the Mali GPU +device. See the mali_utgard.h header file for how to set up the Mali GPU +resources. diff --git a/drivers/gpu/arm/utgard/regs/mali_200_regs.h b/drivers/gpu/arm/utgard/regs/mali_200_regs.h new file mode 100644 index 00000000000000..e76f9926f5c0a8 --- /dev/null +++ b/drivers/gpu/arm/utgard/regs/mali_200_regs.h @@ -0,0 +1,131 @@ +/* + * Copyright (C) 2010, 2012-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef _MALI200_REGS_H_ +#define _MALI200_REGS_H_ + +/** + * Enum for management register addresses. + */ +enum mali200_mgmt_reg { + MALI200_REG_ADDR_MGMT_VERSION = 0x1000, + MALI200_REG_ADDR_MGMT_CURRENT_REND_LIST_ADDR = 0x1004, + MALI200_REG_ADDR_MGMT_STATUS = 0x1008, + MALI200_REG_ADDR_MGMT_CTRL_MGMT = 0x100c, + + MALI200_REG_ADDR_MGMT_INT_RAWSTAT = 0x1020, + MALI200_REG_ADDR_MGMT_INT_CLEAR = 0x1024, + MALI200_REG_ADDR_MGMT_INT_MASK = 0x1028, + MALI200_REG_ADDR_MGMT_INT_STATUS = 0x102c, + + MALI200_REG_ADDR_MGMT_BUS_ERROR_STATUS = 0x1050, + + MALI200_REG_ADDR_MGMT_PERF_CNT_0_ENABLE = 0x1080, + MALI200_REG_ADDR_MGMT_PERF_CNT_0_SRC = 0x1084, + MALI200_REG_ADDR_MGMT_PERF_CNT_0_LIMIT = 0x1088, + MALI200_REG_ADDR_MGMT_PERF_CNT_0_VALUE = 0x108c, + + MALI200_REG_ADDR_MGMT_PERF_CNT_1_ENABLE = 0x10a0, + MALI200_REG_ADDR_MGMT_PERF_CNT_1_SRC = 0x10a4, + MALI200_REG_ADDR_MGMT_PERF_CNT_1_VALUE = 0x10ac, + + MALI200_REG_ADDR_MGMT_PERFMON_CONTR = 0x10b0, + MALI200_REG_ADDR_MGMT_PERFMON_BASE = 0x10b4, + + MALI200_REG_SIZEOF_REGISTER_BANK = 0x10f0 + +}; + +#define MALI200_REG_VAL_PERF_CNT_ENABLE 1 + +enum mali200_mgmt_ctrl_mgmt { + MALI200_REG_VAL_CTRL_MGMT_STOP_BUS = (1 << 0), + MALI200_REG_VAL_CTRL_MGMT_FLUSH_CACHES = (1 << 3), + MALI200_REG_VAL_CTRL_MGMT_FORCE_RESET = (1 << 5), + MALI200_REG_VAL_CTRL_MGMT_START_RENDERING = (1 << 6), + MALI400PP_REG_VAL_CTRL_MGMT_SOFT_RESET = (1 << 7), /* Only valid for Mali-300 and later */ +}; + +enum mali200_mgmt_irq { + MALI200_REG_VAL_IRQ_END_OF_FRAME = (1 << 0), + MALI200_REG_VAL_IRQ_END_OF_TILE = (1 << 1), + MALI200_REG_VAL_IRQ_HANG = (1 << 2), + MALI200_REG_VAL_IRQ_FORCE_HANG = (1 << 3), + MALI200_REG_VAL_IRQ_BUS_ERROR = (1 << 4), + MALI200_REG_VAL_IRQ_BUS_STOP = (1 << 5), + MALI200_REG_VAL_IRQ_CNT_0_LIMIT = (1 << 6), + MALI200_REG_VAL_IRQ_CNT_1_LIMIT = (1 << 7), + MALI200_REG_VAL_IRQ_WRITE_BOUNDARY_ERROR = (1 << 8), + MALI400PP_REG_VAL_IRQ_INVALID_PLIST_COMMAND = (1 << 9), + MALI400PP_REG_VAL_IRQ_CALL_STACK_UNDERFLOW = (1 << 10), + MALI400PP_REG_VAL_IRQ_CALL_STACK_OVERFLOW = (1 << 11), + MALI400PP_REG_VAL_IRQ_RESET_COMPLETED = (1 << 12), +}; + +#define MALI200_REG_VAL_IRQ_MASK_ALL ((enum mali200_mgmt_irq) (\ + MALI200_REG_VAL_IRQ_END_OF_FRAME |\ + MALI200_REG_VAL_IRQ_END_OF_TILE |\ + MALI200_REG_VAL_IRQ_HANG |\ + MALI200_REG_VAL_IRQ_FORCE_HANG |\ + MALI200_REG_VAL_IRQ_BUS_ERROR |\ + MALI200_REG_VAL_IRQ_BUS_STOP |\ + MALI200_REG_VAL_IRQ_CNT_0_LIMIT |\ + MALI200_REG_VAL_IRQ_CNT_1_LIMIT |\ + MALI200_REG_VAL_IRQ_WRITE_BOUNDARY_ERROR |\ + MALI400PP_REG_VAL_IRQ_INVALID_PLIST_COMMAND |\ + MALI400PP_REG_VAL_IRQ_CALL_STACK_UNDERFLOW |\ + MALI400PP_REG_VAL_IRQ_CALL_STACK_OVERFLOW |\ + MALI400PP_REG_VAL_IRQ_RESET_COMPLETED)) + +#define MALI200_REG_VAL_IRQ_MASK_USED ((enum mali200_mgmt_irq) (\ + MALI200_REG_VAL_IRQ_END_OF_FRAME |\ + MALI200_REG_VAL_IRQ_FORCE_HANG |\ + MALI200_REG_VAL_IRQ_BUS_ERROR |\ + MALI200_REG_VAL_IRQ_WRITE_BOUNDARY_ERROR |\ + MALI400PP_REG_VAL_IRQ_INVALID_PLIST_COMMAND |\ + MALI400PP_REG_VAL_IRQ_CALL_STACK_UNDERFLOW |\ + MALI400PP_REG_VAL_IRQ_CALL_STACK_OVERFLOW)) + +#define MALI200_REG_VAL_IRQ_MASK_NONE ((enum mali200_mgmt_irq)(0)) + +enum mali200_mgmt_status { + MALI200_REG_VAL_STATUS_RENDERING_ACTIVE = (1 << 0), + MALI200_REG_VAL_STATUS_BUS_STOPPED = (1 << 4), +}; + +enum mali200_render_unit { + MALI200_REG_ADDR_FRAME = 0x0000, + MALI200_REG_ADDR_RSW = 0x0004, + MALI200_REG_ADDR_STACK = 0x0030, + MALI200_REG_ADDR_STACK_SIZE = 0x0034, + MALI200_REG_ADDR_ORIGIN_OFFSET_X = 0x0040 +}; + +enum mali200_wb_unit { + MALI200_REG_ADDR_WB0 = 0x0100, + MALI200_REG_ADDR_WB1 = 0x0200, + MALI200_REG_ADDR_WB2 = 0x0300 +}; + +enum mali200_wb_unit_regs { + MALI200_REG_ADDR_WB_SOURCE_SELECT = 0x0000, + MALI200_REG_ADDR_WB_SOURCE_ADDR = 0x0004, +}; + +/* This should be in the top 16 bit of the version register of Mali PP */ +#define MALI200_PP_PRODUCT_ID 0xC807 +#define MALI300_PP_PRODUCT_ID 0xCE07 +#define MALI400_PP_PRODUCT_ID 0xCD07 +#define MALI450_PP_PRODUCT_ID 0xCF07 +#define MALI470_PP_PRODUCT_ID 0xCF08 + + + +#endif /* _MALI200_REGS_H_ */ diff --git a/drivers/gpu/arm/utgard/regs/mali_gp_regs.h b/drivers/gpu/arm/utgard/regs/mali_gp_regs.h new file mode 100644 index 00000000000000..9c101f9ddd2232 --- /dev/null +++ b/drivers/gpu/arm/utgard/regs/mali_gp_regs.h @@ -0,0 +1,172 @@ +/* + * Copyright (C) 2010, 2012-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef _MALIGP2_CONROL_REGS_H_ +#define _MALIGP2_CONROL_REGS_H_ + +/** + * These are the different geometry processor control registers. + * Their usage is to control and monitor the operation of the + * Vertex Shader and the Polygon List Builder in the geometry processor. + * Addresses are in 32-bit word relative sizes. + * @see [P0081] "Geometry Processor Data Structures" for details + */ + +typedef enum { + MALIGP2_REG_ADDR_MGMT_VSCL_START_ADDR = 0x00, + MALIGP2_REG_ADDR_MGMT_VSCL_END_ADDR = 0x04, + MALIGP2_REG_ADDR_MGMT_PLBUCL_START_ADDR = 0x08, + MALIGP2_REG_ADDR_MGMT_PLBUCL_END_ADDR = 0x0c, + MALIGP2_REG_ADDR_MGMT_PLBU_ALLOC_START_ADDR = 0x10, + MALIGP2_REG_ADDR_MGMT_PLBU_ALLOC_END_ADDR = 0x14, + MALIGP2_REG_ADDR_MGMT_CMD = 0x20, + MALIGP2_REG_ADDR_MGMT_INT_RAWSTAT = 0x24, + MALIGP2_REG_ADDR_MGMT_INT_CLEAR = 0x28, + MALIGP2_REG_ADDR_MGMT_INT_MASK = 0x2C, + MALIGP2_REG_ADDR_MGMT_INT_STAT = 0x30, + MALIGP2_REG_ADDR_MGMT_PERF_CNT_0_ENABLE = 0x3C, + MALIGP2_REG_ADDR_MGMT_PERF_CNT_1_ENABLE = 0x40, + MALIGP2_REG_ADDR_MGMT_PERF_CNT_0_SRC = 0x44, + MALIGP2_REG_ADDR_MGMT_PERF_CNT_1_SRC = 0x48, + MALIGP2_REG_ADDR_MGMT_PERF_CNT_0_VALUE = 0x4C, + MALIGP2_REG_ADDR_MGMT_PERF_CNT_1_VALUE = 0x50, + MALIGP2_REG_ADDR_MGMT_PERF_CNT_0_LIMIT = 0x54, + MALIGP2_REG_ADDR_MGMT_STATUS = 0x68, + MALIGP2_REG_ADDR_MGMT_VERSION = 0x6C, + MALIGP2_REG_ADDR_MGMT_VSCL_START_ADDR_READ = 0x80, + MALIGP2_REG_ADDR_MGMT_PLBCL_START_ADDR_READ = 0x84, + MALIGP2_CONTR_AXI_BUS_ERROR_STAT = 0x94, + MALIGP2_REGISTER_ADDRESS_SPACE_SIZE = 0x98, +} maligp_reg_addr_mgmt_addr; + +#define MALIGP2_REG_VAL_PERF_CNT_ENABLE 1 + +/** + * Commands to geometry processor. + * @see MALIGP2_CTRL_REG_CMD + */ +typedef enum { + MALIGP2_REG_VAL_CMD_START_VS = (1 << 0), + MALIGP2_REG_VAL_CMD_START_PLBU = (1 << 1), + MALIGP2_REG_VAL_CMD_UPDATE_PLBU_ALLOC = (1 << 4), + MALIGP2_REG_VAL_CMD_RESET = (1 << 5), + MALIGP2_REG_VAL_CMD_FORCE_HANG = (1 << 6), + MALIGP2_REG_VAL_CMD_STOP_BUS = (1 << 9), + MALI400GP_REG_VAL_CMD_SOFT_RESET = (1 << 10), /* only valid for Mali-300 and later */ +} mgp_contr_reg_val_cmd; + + +/** @defgroup MALIGP2_IRQ + * Interrupt status of geometry processor. + * @see MALIGP2_CTRL_REG_INT_RAWSTAT, MALIGP2_REG_ADDR_MGMT_INT_CLEAR, + * MALIGP2_REG_ADDR_MGMT_INT_MASK, MALIGP2_REG_ADDR_MGMT_INT_STAT + * @{ + */ +#define MALIGP2_REG_VAL_IRQ_VS_END_CMD_LST (1 << 0) +#define MALIGP2_REG_VAL_IRQ_PLBU_END_CMD_LST (1 << 1) +#define MALIGP2_REG_VAL_IRQ_PLBU_OUT_OF_MEM (1 << 2) +#define MALIGP2_REG_VAL_IRQ_VS_SEM_IRQ (1 << 3) +#define MALIGP2_REG_VAL_IRQ_PLBU_SEM_IRQ (1 << 4) +#define MALIGP2_REG_VAL_IRQ_HANG (1 << 5) +#define MALIGP2_REG_VAL_IRQ_FORCE_HANG (1 << 6) +#define MALIGP2_REG_VAL_IRQ_PERF_CNT_0_LIMIT (1 << 7) +#define MALIGP2_REG_VAL_IRQ_PERF_CNT_1_LIMIT (1 << 8) +#define MALIGP2_REG_VAL_IRQ_WRITE_BOUND_ERR (1 << 9) +#define MALIGP2_REG_VAL_IRQ_SYNC_ERROR (1 << 10) +#define MALIGP2_REG_VAL_IRQ_AXI_BUS_ERROR (1 << 11) +#define MALI400GP_REG_VAL_IRQ_AXI_BUS_STOPPED (1 << 12) +#define MALI400GP_REG_VAL_IRQ_VS_INVALID_CMD (1 << 13) +#define MALI400GP_REG_VAL_IRQ_PLB_INVALID_CMD (1 << 14) +#define MALI400GP_REG_VAL_IRQ_RESET_COMPLETED (1 << 19) +#define MALI400GP_REG_VAL_IRQ_SEMAPHORE_UNDERFLOW (1 << 20) +#define MALI400GP_REG_VAL_IRQ_SEMAPHORE_OVERFLOW (1 << 21) +#define MALI400GP_REG_VAL_IRQ_PTR_ARRAY_OUT_OF_BOUNDS (1 << 22) + +/* Mask defining all IRQs in Mali GP */ +#define MALIGP2_REG_VAL_IRQ_MASK_ALL \ + (\ + MALIGP2_REG_VAL_IRQ_VS_END_CMD_LST | \ + MALIGP2_REG_VAL_IRQ_PLBU_END_CMD_LST | \ + MALIGP2_REG_VAL_IRQ_PLBU_OUT_OF_MEM | \ + MALIGP2_REG_VAL_IRQ_VS_SEM_IRQ | \ + MALIGP2_REG_VAL_IRQ_PLBU_SEM_IRQ | \ + MALIGP2_REG_VAL_IRQ_HANG | \ + MALIGP2_REG_VAL_IRQ_FORCE_HANG | \ + MALIGP2_REG_VAL_IRQ_PERF_CNT_0_LIMIT | \ + MALIGP2_REG_VAL_IRQ_PERF_CNT_1_LIMIT | \ + MALIGP2_REG_VAL_IRQ_WRITE_BOUND_ERR | \ + MALIGP2_REG_VAL_IRQ_SYNC_ERROR | \ + MALIGP2_REG_VAL_IRQ_AXI_BUS_ERROR | \ + MALI400GP_REG_VAL_IRQ_AXI_BUS_STOPPED | \ + MALI400GP_REG_VAL_IRQ_VS_INVALID_CMD | \ + MALI400GP_REG_VAL_IRQ_PLB_INVALID_CMD | \ + MALI400GP_REG_VAL_IRQ_RESET_COMPLETED | \ + MALI400GP_REG_VAL_IRQ_SEMAPHORE_UNDERFLOW | \ + MALI400GP_REG_VAL_IRQ_SEMAPHORE_OVERFLOW | \ + MALI400GP_REG_VAL_IRQ_PTR_ARRAY_OUT_OF_BOUNDS) + +/* Mask defining the IRQs in Mali GP which we use */ +#define MALIGP2_REG_VAL_IRQ_MASK_USED \ + (\ + MALIGP2_REG_VAL_IRQ_VS_END_CMD_LST | \ + MALIGP2_REG_VAL_IRQ_PLBU_END_CMD_LST | \ + MALIGP2_REG_VAL_IRQ_PLBU_OUT_OF_MEM | \ + MALIGP2_REG_VAL_IRQ_FORCE_HANG | \ + MALIGP2_REG_VAL_IRQ_WRITE_BOUND_ERR | \ + MALIGP2_REG_VAL_IRQ_SYNC_ERROR | \ + MALIGP2_REG_VAL_IRQ_AXI_BUS_ERROR | \ + MALI400GP_REG_VAL_IRQ_VS_INVALID_CMD | \ + MALI400GP_REG_VAL_IRQ_PLB_INVALID_CMD | \ + MALI400GP_REG_VAL_IRQ_SEMAPHORE_UNDERFLOW | \ + MALI400GP_REG_VAL_IRQ_SEMAPHORE_OVERFLOW | \ + MALI400GP_REG_VAL_IRQ_PTR_ARRAY_OUT_OF_BOUNDS) + +/* Mask defining non IRQs on MaliGP2*/ +#define MALIGP2_REG_VAL_IRQ_MASK_NONE 0 + +/** }@ defgroup MALIGP2_IRQ*/ + +/** @defgroup MALIGP2_STATUS + * The different Status values to the geometry processor. + * @see MALIGP2_CTRL_REG_STATUS + * @{ + */ +#define MALIGP2_REG_VAL_STATUS_VS_ACTIVE 0x0002 +#define MALIGP2_REG_VAL_STATUS_BUS_STOPPED 0x0004 +#define MALIGP2_REG_VAL_STATUS_PLBU_ACTIVE 0x0008 +#define MALIGP2_REG_VAL_STATUS_BUS_ERROR 0x0040 +#define MALIGP2_REG_VAL_STATUS_WRITE_BOUND_ERR 0x0100 +/** }@ defgroup MALIGP2_STATUS*/ + +#define MALIGP2_REG_VAL_STATUS_MASK_ACTIVE (\ + MALIGP2_REG_VAL_STATUS_VS_ACTIVE|\ + MALIGP2_REG_VAL_STATUS_PLBU_ACTIVE) + + +#define MALIGP2_REG_VAL_STATUS_MASK_ERROR (\ + MALIGP2_REG_VAL_STATUS_BUS_ERROR |\ + MALIGP2_REG_VAL_STATUS_WRITE_BOUND_ERR ) + +/* This should be in the top 16 bit of the version register of gp.*/ +#define MALI200_GP_PRODUCT_ID 0xA07 +#define MALI300_GP_PRODUCT_ID 0xC07 +#define MALI400_GP_PRODUCT_ID 0xB07 +#define MALI450_GP_PRODUCT_ID 0xD07 + +/** + * The different sources for instrumented on the geometry processor. + * @see MALIGP2_REG_ADDR_MGMT_PERF_CNT_0_SRC + */ + +enum MALIGP2_cont_reg_perf_cnt_src { + MALIGP2_REG_VAL_PERF_CNT1_SRC_NUMBER_OF_VERTICES_PROCESSED = 0x0a, +}; + +#endif diff --git a/drivers/gpu/arm/utgard/timestamp-arm11-cc/mali_timestamp.c b/drivers/gpu/arm/utgard/timestamp-arm11-cc/mali_timestamp.c new file mode 100644 index 00000000000000..a486e2f7684f74 --- /dev/null +++ b/drivers/gpu/arm/utgard/timestamp-arm11-cc/mali_timestamp.c @@ -0,0 +1,13 @@ +/* + * Copyright (C) 2010-2011, 2013, 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_timestamp.h" + +/* This file is intentionally left empty, as all functions are inlined in mali_profiling_sampler.h */ diff --git a/drivers/gpu/arm/utgard/timestamp-arm11-cc/mali_timestamp.h b/drivers/gpu/arm/utgard/timestamp-arm11-cc/mali_timestamp.h new file mode 100644 index 00000000000000..65f3ab274c09ba --- /dev/null +++ b/drivers/gpu/arm/utgard/timestamp-arm11-cc/mali_timestamp.h @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2010-2011, 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_TIMESTAMP_H__ +#define __MALI_TIMESTAMP_H__ + +#include "mali_osk.h" + +MALI_STATIC_INLINE _mali_osk_errcode_t _mali_timestamp_reset(void) +{ + /* + * reset counters and overflow flags + */ + + u32 mask = (1 << 0) | /* enable all three counters */ + (0 << 1) | /* reset both Count Registers to 0x0 */ + (1 << 2) | /* reset the Cycle Counter Register to 0x0 */ + (0 << 3) | /* 1 = Cycle Counter Register counts every 64th processor clock cycle */ + (0 << 4) | /* Count Register 0 interrupt enable */ + (0 << 5) | /* Count Register 1 interrupt enable */ + (0 << 6) | /* Cycle Counter interrupt enable */ + (0 << 8) | /* Count Register 0 overflow flag (clear or write, flag on read) */ + (0 << 9) | /* Count Register 1 overflow flag (clear or write, flag on read) */ + (1 << 10); /* Cycle Counter Register overflow flag (clear or write, flag on read) */ + + __asm__ __volatile__("MCR p15, 0, %0, c15, c12, 0" : : "r"(mask)); + + return _MALI_OSK_ERR_OK; +} + +MALI_STATIC_INLINE u64 _mali_timestamp_get(void) +{ + u32 result; + + /* this is for the clock cycles */ + __asm__ __volatile__("MRC p15, 0, %0, c15, c12, 1" : "=r"(result)); + + return (u64)result; +} + +#endif /* __MALI_TIMESTAMP_H__ */ diff --git a/drivers/gpu/arm/utgard/timestamp-default/mali_timestamp.c b/drivers/gpu/arm/utgard/timestamp-default/mali_timestamp.c new file mode 100644 index 00000000000000..a486e2f7684f74 --- /dev/null +++ b/drivers/gpu/arm/utgard/timestamp-default/mali_timestamp.c @@ -0,0 +1,13 @@ +/* + * Copyright (C) 2010-2011, 2013, 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_timestamp.h" + +/* This file is intentionally left empty, as all functions are inlined in mali_profiling_sampler.h */ diff --git a/drivers/gpu/arm/utgard/timestamp-default/mali_timestamp.h b/drivers/gpu/arm/utgard/timestamp-default/mali_timestamp.h new file mode 100644 index 00000000000000..8ba47060828f03 --- /dev/null +++ b/drivers/gpu/arm/utgard/timestamp-default/mali_timestamp.h @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2010-2011, 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_TIMESTAMP_H__ +#define __MALI_TIMESTAMP_H__ + +#include "mali_osk.h" + +MALI_STATIC_INLINE _mali_osk_errcode_t _mali_timestamp_reset(void) +{ + return _MALI_OSK_ERR_OK; +} + +MALI_STATIC_INLINE u64 _mali_timestamp_get(void) +{ + return _mali_osk_boot_time_get_ns(); +} + +#endif /* __MALI_TIMESTAMP_H__ */ diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig index 483059a22b1b71..cb5b1383a26233 100644 --- a/drivers/gpu/drm/Kconfig +++ b/drivers/gpu/drm/Kconfig @@ -106,6 +106,14 @@ config DRM_KMS_CMA_HELPER help Choose this if you need the KMS CMA helper functions +config DRM_CMA_FBDEV_BUFFER_NUM + int "Cma Fbdev Buffer Number" + depends on DRM_KMS_CMA_HELPER + default 1 + help + Defines the buffer number of cma fbdev. Default is one buffer. + For double buffer please set to 2 and 3 for triple buffer. + source "drivers/gpu/drm/i2c/Kconfig" source "drivers/gpu/drm/arm/Kconfig" diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c index 9260caef74fa07..882404cefbc238 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c @@ -2577,6 +2577,9 @@ static int dce_v10_0_cursor_move_locked(struct drm_crtc *crtc, struct amdgpu_device *adev = crtc->dev->dev_private; int xorigin = 0, yorigin = 0; + amdgpu_crtc->cursor_x = x; + amdgpu_crtc->cursor_y = y; + /* avivo cursor are offset into the total surface */ x += crtc->x; y += crtc->y; @@ -2596,9 +2599,6 @@ static int dce_v10_0_cursor_move_locked(struct drm_crtc *crtc, WREG32(mmCUR_SIZE + amdgpu_crtc->crtc_offset, ((amdgpu_crtc->cursor_width - 1) << 16) | (amdgpu_crtc->cursor_height - 1)); - amdgpu_crtc->cursor_x = x; - amdgpu_crtc->cursor_y = y; - return 0; } @@ -2661,12 +2661,11 @@ static int dce_v10_0_crtc_cursor_set2(struct drm_crtc *crtc, return ret; } - amdgpu_crtc->cursor_width = width; - amdgpu_crtc->cursor_height = height; - dce_v10_0_lock_cursor(crtc, true); - if (hot_x != amdgpu_crtc->cursor_hot_x || + if (width != amdgpu_crtc->cursor_width || + height != amdgpu_crtc->cursor_height || + hot_x != amdgpu_crtc->cursor_hot_x || hot_y != amdgpu_crtc->cursor_hot_y) { int x, y; @@ -2675,6 +2674,8 @@ static int dce_v10_0_crtc_cursor_set2(struct drm_crtc *crtc, dce_v10_0_cursor_move_locked(crtc, x, y); + amdgpu_crtc->cursor_width = width; + amdgpu_crtc->cursor_height = height; amdgpu_crtc->cursor_hot_x = hot_x; amdgpu_crtc->cursor_hot_y = hot_y; } diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c index 367739bd19279f..7ddc32127d88f0 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c @@ -2593,6 +2593,9 @@ static int dce_v11_0_cursor_move_locked(struct drm_crtc *crtc, struct amdgpu_device *adev = crtc->dev->dev_private; int xorigin = 0, yorigin = 0; + amdgpu_crtc->cursor_x = x; + amdgpu_crtc->cursor_y = y; + /* avivo cursor are offset into the total surface */ x += crtc->x; y += crtc->y; @@ -2612,9 +2615,6 @@ static int dce_v11_0_cursor_move_locked(struct drm_crtc *crtc, WREG32(mmCUR_SIZE + amdgpu_crtc->crtc_offset, ((amdgpu_crtc->cursor_width - 1) << 16) | (amdgpu_crtc->cursor_height - 1)); - amdgpu_crtc->cursor_x = x; - amdgpu_crtc->cursor_y = y; - return 0; } @@ -2677,12 +2677,11 @@ static int dce_v11_0_crtc_cursor_set2(struct drm_crtc *crtc, return ret; } - amdgpu_crtc->cursor_width = width; - amdgpu_crtc->cursor_height = height; - dce_v11_0_lock_cursor(crtc, true); - if (hot_x != amdgpu_crtc->cursor_hot_x || + if (width != amdgpu_crtc->cursor_width || + height != amdgpu_crtc->cursor_height || + hot_x != amdgpu_crtc->cursor_hot_x || hot_y != amdgpu_crtc->cursor_hot_y) { int x, y; @@ -2691,6 +2690,8 @@ static int dce_v11_0_crtc_cursor_set2(struct drm_crtc *crtc, dce_v11_0_cursor_move_locked(crtc, x, y); + amdgpu_crtc->cursor_width = width; + amdgpu_crtc->cursor_height = height; amdgpu_crtc->cursor_hot_x = hot_x; amdgpu_crtc->cursor_hot_y = hot_y; } diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c index 15f9fc0514b29b..fde6ee1f6f2b3b 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c @@ -1933,6 +1933,9 @@ static int dce_v6_0_cursor_move_locked(struct drm_crtc *crtc, int w = amdgpu_crtc->cursor_width; + amdgpu_crtc->cursor_x = x; + amdgpu_crtc->cursor_y = y; + /* avivo cursor are offset into the total surface */ x += crtc->x; y += crtc->y; @@ -1952,8 +1955,6 @@ static int dce_v6_0_cursor_move_locked(struct drm_crtc *crtc, WREG32(EVERGREEN_CUR_SIZE + amdgpu_crtc->crtc_offset, ((w - 1) << 16) | (amdgpu_crtc->cursor_height - 1)); - amdgpu_crtc->cursor_x = x; - amdgpu_crtc->cursor_y = y; return 0; } @@ -2016,12 +2017,11 @@ static int dce_v6_0_crtc_cursor_set2(struct drm_crtc *crtc, return ret; } - amdgpu_crtc->cursor_width = width; - amdgpu_crtc->cursor_height = height; - dce_v6_0_lock_cursor(crtc, true); - if (hot_x != amdgpu_crtc->cursor_hot_x || + if (width != amdgpu_crtc->cursor_width || + height != amdgpu_crtc->cursor_height || + hot_x != amdgpu_crtc->cursor_hot_x || hot_y != amdgpu_crtc->cursor_hot_y) { int x, y; @@ -2030,6 +2030,8 @@ static int dce_v6_0_crtc_cursor_set2(struct drm_crtc *crtc, dce_v6_0_cursor_move_locked(crtc, x, y); + amdgpu_crtc->cursor_width = width; + amdgpu_crtc->cursor_height = height; amdgpu_crtc->cursor_hot_x = hot_x; amdgpu_crtc->cursor_hot_y = hot_y; } diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c index 8c4d808db0f127..7d9ffde0a628db 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c @@ -2465,6 +2465,9 @@ static int dce_v8_0_cursor_move_locked(struct drm_crtc *crtc, struct amdgpu_device *adev = crtc->dev->dev_private; int xorigin = 0, yorigin = 0; + amdgpu_crtc->cursor_x = x; + amdgpu_crtc->cursor_y = y; + /* avivo cursor are offset into the total surface */ x += crtc->x; y += crtc->y; @@ -2484,9 +2487,6 @@ static int dce_v8_0_cursor_move_locked(struct drm_crtc *crtc, WREG32(mmCUR_SIZE + amdgpu_crtc->crtc_offset, ((amdgpu_crtc->cursor_width - 1) << 16) | (amdgpu_crtc->cursor_height - 1)); - amdgpu_crtc->cursor_x = x; - amdgpu_crtc->cursor_y = y; - return 0; } @@ -2549,12 +2549,11 @@ static int dce_v8_0_crtc_cursor_set2(struct drm_crtc *crtc, return ret; } - amdgpu_crtc->cursor_width = width; - amdgpu_crtc->cursor_height = height; - dce_v8_0_lock_cursor(crtc, true); - if (hot_x != amdgpu_crtc->cursor_hot_x || + if (width != amdgpu_crtc->cursor_width || + height != amdgpu_crtc->cursor_height || + hot_x != amdgpu_crtc->cursor_hot_x || hot_y != amdgpu_crtc->cursor_hot_y) { int x, y; @@ -2563,6 +2562,8 @@ static int dce_v8_0_crtc_cursor_set2(struct drm_crtc *crtc, dce_v8_0_cursor_move_locked(crtc, x, y); + amdgpu_crtc->cursor_width = width; + amdgpu_crtc->cursor_height = height; amdgpu_crtc->cursor_hot_x = hot_x; amdgpu_crtc->cursor_hot_y = hot_y; } diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c index bb97182dc74991..a88d365be4c566 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c @@ -3947,8 +3947,12 @@ static int gfx_v8_0_init_save_restore_list(struct amdgpu_device *adev) temp = mmRLC_SRM_INDEX_CNTL_ADDR_0; data = mmRLC_SRM_INDEX_CNTL_DATA_0; for (i = 0; i < sizeof(unique_indices) / sizeof(int); i++) { - amdgpu_mm_wreg(adev, temp + i, unique_indices[i] & 0x3FFFF, false); - amdgpu_mm_wreg(adev, data + i, unique_indices[i] >> 20, false); + if (unique_indices[i] != 0) { + amdgpu_mm_wreg(adev, temp + i, + unique_indices[i] & 0x3FFFF, false); + amdgpu_mm_wreg(adev, data + i, + unique_indices[i] >> 20, false); + } } kfree(register_list_format); @@ -3994,7 +3998,7 @@ static void cz_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev, static void cz_enable_cp_power_gating(struct amdgpu_device *adev, bool enable) { - WREG32_FIELD(RLC_PG_CNTL, CP_PG_DISABLE, enable ? 1 : 0); + WREG32_FIELD(RLC_PG_CNTL, CP_PG_DISABLE, enable ? 0 : 1); } static void gfx_v8_0_init_pg(struct amdgpu_device *adev) @@ -5891,29 +5895,24 @@ static void gfx_v8_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev adev->gfx.rlc.funcs->enter_safe_mode(adev); if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) { - /* 1 enable cntx_empty_int_enable/cntx_busy_int_enable/ - * Cmp_busy/GFX_Idle interrupts - */ - gfx_v8_0_enable_gui_idle_interrupt(adev, true); - temp1 = data1 = RREG32(mmRLC_CGTT_MGCG_OVERRIDE); data1 &= ~RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK; if (temp1 != data1) WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1); - /* 2 wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ + /* : wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ gfx_v8_0_wait_for_rlc_serdes(adev); - /* 3 - clear cgcg override */ + /* 2 - clear cgcg override */ gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGCG_OVERRIDE, CLE_BPM_SERDES_CMD); /* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ gfx_v8_0_wait_for_rlc_serdes(adev); - /* 4 - write cmd to set CGLS */ + /* 3 - write cmd to set CGLS */ gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGLS_EN, SET_BPM_SERDES_CMD); - /* 5 - enable cgcg */ + /* 4 - enable cgcg */ data |= RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) { @@ -5931,6 +5930,11 @@ static void gfx_v8_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev if (temp != data) WREG32(mmRLC_CGCG_CGLS_CTRL, data); + + /* 5 enable cntx_empty_int_enable/cntx_busy_int_enable/ + * Cmp_busy/GFX_Idle interrupts + */ + gfx_v8_0_enable_gui_idle_interrupt(adev, true); } else { /* disable cntx_empty_int_enable & GFX Idle interrupt */ gfx_v8_0_enable_gui_idle_interrupt(adev, false); diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c index b13c8aaec078e6..6df924f72f29cd 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c @@ -227,6 +227,9 @@ static void gmc_v6_0_mc_program(struct amdgpu_device *adev) } WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0); + if (adev->mode_info.num_crtc) + amdgpu_display_set_vga_render_state(adev, false); + gmc_v6_0_mc_stop(adev, &save); if (gmc_v6_0_wait_for_idle((void *)adev)) { @@ -256,7 +259,6 @@ static void gmc_v6_0_mc_program(struct amdgpu_device *adev) dev_warn(adev->dev, "Wait for MC idle timedout !\n"); } gmc_v6_0_mc_resume(adev, &save); - amdgpu_display_set_vga_render_state(adev, false); } static int gmc_v6_0_mc_init(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/si_dpm.c b/drivers/gpu/drm/amd/amdgpu/si_dpm.c index d6f85b1a0b9354..b447a01ab21a6f 100644 --- a/drivers/gpu/drm/amd/amdgpu/si_dpm.c +++ b/drivers/gpu/drm/amd/amdgpu/si_dpm.c @@ -56,7 +56,6 @@ #define BIOS_SCRATCH_4 0x5cd MODULE_FIRMWARE("radeon/tahiti_smc.bin"); -MODULE_FIRMWARE("radeon/tahiti_k_smc.bin"); MODULE_FIRMWARE("radeon/pitcairn_smc.bin"); MODULE_FIRMWARE("radeon/pitcairn_k_smc.bin"); MODULE_FIRMWARE("radeon/verde_smc.bin"); @@ -3486,24 +3485,12 @@ static void si_apply_state_adjust_rules(struct amdgpu_device *adev, (adev->pdev->device == 0x6817) || (adev->pdev->device == 0x6806)) max_mclk = 120000; - } else if (adev->asic_type == CHIP_VERDE) { - if ((adev->pdev->revision == 0x81) || - (adev->pdev->revision == 0x83) || - (adev->pdev->revision == 0x87) || - (adev->pdev->device == 0x6820) || - (adev->pdev->device == 0x6821) || - (adev->pdev->device == 0x6822) || - (adev->pdev->device == 0x6823) || - (adev->pdev->device == 0x682A) || - (adev->pdev->device == 0x682B)) { - max_sclk = 75000; - max_mclk = 80000; - } } else if (adev->asic_type == CHIP_OLAND) { if ((adev->pdev->revision == 0xC7) || (adev->pdev->revision == 0x80) || (adev->pdev->revision == 0x81) || (adev->pdev->revision == 0x83) || + (adev->pdev->revision == 0x87) || (adev->pdev->device == 0x6604) || (adev->pdev->device == 0x6605)) { max_sclk = 75000; @@ -7684,48 +7671,49 @@ static int si_dpm_init_microcode(struct amdgpu_device *adev) chip_name = "tahiti"; break; case CHIP_PITCAIRN: - if ((adev->pdev->revision == 0x81) || - (adev->pdev->device == 0x6810) || - (adev->pdev->device == 0x6811) || - (adev->pdev->device == 0x6816) || - (adev->pdev->device == 0x6817) || - (adev->pdev->device == 0x6806)) + if ((adev->pdev->revision == 0x81) && + ((adev->pdev->device == 0x6810) || + (adev->pdev->device == 0x6811))) chip_name = "pitcairn_k"; else chip_name = "pitcairn"; break; case CHIP_VERDE: - if ((adev->pdev->revision == 0x81) || - (adev->pdev->revision == 0x83) || - (adev->pdev->revision == 0x87) || - (adev->pdev->device == 0x6820) || - (adev->pdev->device == 0x6821) || - (adev->pdev->device == 0x6822) || - (adev->pdev->device == 0x6823) || - (adev->pdev->device == 0x682A) || - (adev->pdev->device == 0x682B)) + if (((adev->pdev->device == 0x6820) && + ((adev->pdev->revision == 0x81) || + (adev->pdev->revision == 0x83))) || + ((adev->pdev->device == 0x6821) && + ((adev->pdev->revision == 0x83) || + (adev->pdev->revision == 0x87))) || + ((adev->pdev->revision == 0x87) && + ((adev->pdev->device == 0x6823) || + (adev->pdev->device == 0x682b)))) chip_name = "verde_k"; else chip_name = "verde"; break; case CHIP_OLAND: - if ((adev->pdev->revision == 0xC7) || - (adev->pdev->revision == 0x80) || - (adev->pdev->revision == 0x81) || - (adev->pdev->revision == 0x83) || - (adev->pdev->device == 0x6604) || - (adev->pdev->device == 0x6605)) + if (((adev->pdev->revision == 0x81) && + ((adev->pdev->device == 0x6600) || + (adev->pdev->device == 0x6604) || + (adev->pdev->device == 0x6605) || + (adev->pdev->device == 0x6610))) || + ((adev->pdev->revision == 0x83) && + (adev->pdev->device == 0x6610))) chip_name = "oland_k"; else chip_name = "oland"; break; case CHIP_HAINAN: - if ((adev->pdev->revision == 0x81) || - (adev->pdev->revision == 0x83) || - (adev->pdev->revision == 0xC3) || - (adev->pdev->device == 0x6664) || - (adev->pdev->device == 0x6665) || - (adev->pdev->device == 0x6667)) + if (((adev->pdev->revision == 0x81) && + (adev->pdev->device == 0x6660)) || + ((adev->pdev->revision == 0x83) && + ((adev->pdev->device == 0x6660) || + (adev->pdev->device == 0x6663) || + (adev->pdev->device == 0x6665) || + (adev->pdev->device == 0x6667))) || + ((adev->pdev->revision == 0xc3) && + (adev->pdev->device == 0x6665))) chip_name = "hainan_k"; else chip_name = "hainan"; diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smc.c b/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smc.c index 76310ac7ef0d12..dca1b13fda2f43 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smc.c +++ b/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smc.c @@ -1958,6 +1958,12 @@ int fiji_thermal_setup_fan_table(struct pp_hwmgr *hwmgr) int res; uint64_t tmp64; + if (hwmgr->thermal_controller.fanInfo.bNoFan) { + phm_cap_unset(hwmgr->platform_descriptor.platformCaps, + PHM_PlatformCaps_MicrocodeFanControl); + return 0; + } + if (smu_data->smu7_data.fan_table_start == 0) { phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_MicrocodeFanControl); diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smc.c b/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smc.c index 8c889caba420dc..6c26b83655d0d4 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smc.c +++ b/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smc.c @@ -2006,6 +2006,12 @@ int iceland_thermal_setup_fan_table(struct pp_hwmgr *hwmgr) if (!phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_MicrocodeFanControl)) return 0; + if (hwmgr->thermal_controller.fanInfo.bNoFan) { + phm_cap_unset(hwmgr->platform_descriptor.platformCaps, + PHM_PlatformCaps_MicrocodeFanControl); + return 0; + } + if (0 == smu7_data->fan_table_start) { phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_MicrocodeFanControl); return 0; diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smc.c b/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smc.c index 71bb2f8dc157ba..8ca1a3341dea7b 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smc.c +++ b/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smc.c @@ -1885,6 +1885,12 @@ int polaris10_thermal_setup_fan_table(struct pp_hwmgr *hwmgr) int res; uint64_t tmp64; + if (hwmgr->thermal_controller.fanInfo.bNoFan) { + phm_cap_unset(hwmgr->platform_descriptor.platformCaps, + PHM_PlatformCaps_MicrocodeFanControl); + return 0; + } + if (smu_data->smu7_data.fan_table_start == 0) { phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_MicrocodeFanControl); diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smc.c b/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smc.c index de2a24d85f48b9..a6619e530fe3a3 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smc.c +++ b/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smc.c @@ -2496,6 +2496,12 @@ int tonga_thermal_setup_fan_table(struct pp_hwmgr *hwmgr) PHM_PlatformCaps_MicrocodeFanControl)) return 0; + if (hwmgr->thermal_controller.fanInfo.bNoFan) { + phm_cap_unset(hwmgr->platform_descriptor.platformCaps, + PHM_PlatformCaps_MicrocodeFanControl); + return 0; + } + if (0 == smu_data->smu7_data.fan_table_start) { phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_MicrocodeFanControl); diff --git a/drivers/gpu/drm/ast/ast_main.c b/drivers/gpu/drm/ast/ast_main.c index 904beaa932d03f..f75c6421db6239 100644 --- a/drivers/gpu/drm/ast/ast_main.c +++ b/drivers/gpu/drm/ast/ast_main.c @@ -223,7 +223,8 @@ static int ast_get_dram_info(struct drm_device *dev) ast_write32(ast, 0x10000, 0xfc600309); do { - ; + if (pci_channel_offline(dev->pdev)) + return -EIO; } while (ast_read32(ast, 0x10000) != 0x01); data = ast_read32(ast, 0x10004); @@ -428,7 +429,9 @@ int ast_driver_load(struct drm_device *dev, unsigned long flags) ast_detect_chip(dev, &need_post); if (ast->chip != AST1180) { - ast_get_dram_info(dev); + ret = ast_get_dram_info(dev); + if (ret) + goto out_free; ast->vram_size = ast_get_vram_info(dev); DRM_INFO("dram %d %d %d %08x\n", ast->mclk, ast->dram_type, ast->dram_bus_width, ast->vram_size); } diff --git a/drivers/gpu/drm/bridge/adv7511/Kconfig b/drivers/gpu/drm/bridge/adv7511/Kconfig index d2b0499ab7d75d..2fed567f994378 100644 --- a/drivers/gpu/drm/bridge/adv7511/Kconfig +++ b/drivers/gpu/drm/bridge/adv7511/Kconfig @@ -6,6 +6,14 @@ config DRM_I2C_ADV7511 help Support for the Analog Device ADV7511(W) and ADV7513 HDMI encoders. +config DRM_I2C_ADV7511_AUDIO + bool "ADV7511 HDMI Audio driver" + depends on DRM_I2C_ADV7511 && SND_SOC + select SND_SOC_HDMI_CODEC + help + Support the ADV7511 HDMI Audio interface. This is used in + conjunction with the AV7511 HDMI driver. + config DRM_I2C_ADV7533 bool "ADV7533 encoder" depends on DRM_I2C_ADV7511 diff --git a/drivers/gpu/drm/bridge/adv7511/Makefile b/drivers/gpu/drm/bridge/adv7511/Makefile index 9019327fff4c0b..5ba675534f6eca 100644 --- a/drivers/gpu/drm/bridge/adv7511/Makefile +++ b/drivers/gpu/drm/bridge/adv7511/Makefile @@ -1,3 +1,4 @@ adv7511-y := adv7511_drv.o +adv7511-$(CONFIG_DRM_I2C_ADV7511_AUDIO) += adv7511_audio.o adv7511-$(CONFIG_DRM_I2C_ADV7533) += adv7533.o obj-$(CONFIG_DRM_I2C_ADV7511) += adv7511.o diff --git a/drivers/gpu/drm/bridge/adv7511/adv7511.h b/drivers/gpu/drm/bridge/adv7511/adv7511.h index 161c923d6162cf..0396791a0cd06f 100644 --- a/drivers/gpu/drm/bridge/adv7511/adv7511.h +++ b/drivers/gpu/drm/bridge/adv7511/adv7511.h @@ -309,12 +309,16 @@ struct adv7511 { struct drm_display_mode curr_mode; unsigned int f_tmds; + unsigned int f_audio; + unsigned int audio_source; unsigned int current_edid_segment; uint8_t edid_buf[256]; bool edid_read; wait_queue_head_t wq; + struct work_struct hpd_work; + struct drm_bridge bridge; struct drm_connector connector; @@ -334,6 +338,7 @@ struct adv7511 { bool use_timing_gen; enum adv7511_type type; + struct platform_device *audio_pdev; }; #ifdef CONFIG_DRM_I2C_ADV7533 @@ -389,4 +394,17 @@ static inline int adv7533_parse_dt(struct device_node *np, struct adv7511 *adv) } #endif +#ifdef CONFIG_DRM_I2C_ADV7511_AUDIO +int adv7511_audio_init(struct device *dev, struct adv7511 *adv7511); +void adv7511_audio_exit(struct adv7511 *adv7511); +#else /*CONFIG_DRM_I2C_ADV7511_AUDIO */ +static inline int adv7511_audio_init(struct device *dev, struct adv7511 *adv7511) +{ + return 0; +} +static inline void adv7511_audio_exit(struct adv7511 *adv7511) +{ +} +#endif /* CONFIG_DRM_I2C_ADV7511_AUDIO */ + #endif /* __DRM_I2C_ADV7511_H__ */ diff --git a/drivers/gpu/drm/bridge/adv7511/adv7511_audio.c b/drivers/gpu/drm/bridge/adv7511/adv7511_audio.c new file mode 100644 index 00000000000000..5ce29a5a8b0942 --- /dev/null +++ b/drivers/gpu/drm/bridge/adv7511/adv7511_audio.c @@ -0,0 +1,213 @@ +/* + * Analog Devices ADV7511 HDMI transmitter driver + * + * Copyright 2012 Analog Devices Inc. + * Copyright (c) 2016, Linaro Limited + * + * Licensed under the GPL-2. + */ + +#include +#include +#include +#include + +#include "adv7511.h" + +static void adv7511_calc_cts_n(unsigned int f_tmds, unsigned int fs, + unsigned int *cts, unsigned int *n) +{ + switch (fs) { + case 32000: + *n = 4096; + break; + case 44100: + *n = 6272; + break; + case 48000: + *n = 6144; + break; + } + + *cts = ((f_tmds * *n) / (128 * fs)) * 1000; +} + +static int adv7511_update_cts_n(struct adv7511 *adv7511) +{ + unsigned int cts = 0; + unsigned int n = 0; + + adv7511_calc_cts_n(adv7511->f_tmds, adv7511->f_audio, &cts, &n); + + regmap_write(adv7511->regmap, ADV7511_REG_N0, (n >> 16) & 0xf); + regmap_write(adv7511->regmap, ADV7511_REG_N1, (n >> 8) & 0xff); + regmap_write(adv7511->regmap, ADV7511_REG_N2, n & 0xff); + + regmap_write(adv7511->regmap, ADV7511_REG_CTS_MANUAL0, + (cts >> 16) & 0xf); + regmap_write(adv7511->regmap, ADV7511_REG_CTS_MANUAL1, + (cts >> 8) & 0xff); + regmap_write(adv7511->regmap, ADV7511_REG_CTS_MANUAL2, + cts & 0xff); + + return 0; +} + +int adv7511_hdmi_hw_params(struct device *dev, void *data, + struct hdmi_codec_daifmt *fmt, + struct hdmi_codec_params *hparms) +{ + struct adv7511 *adv7511 = dev_get_drvdata(dev); + unsigned int audio_source, i2s_format = 0; + unsigned int invert_clock; + unsigned int rate; + unsigned int len; + + switch (hparms->sample_rate) { + case 32000: + rate = ADV7511_SAMPLE_FREQ_32000; + break; + case 44100: + rate = ADV7511_SAMPLE_FREQ_44100; + break; + case 48000: + rate = ADV7511_SAMPLE_FREQ_48000; + break; + case 88200: + rate = ADV7511_SAMPLE_FREQ_88200; + break; + case 96000: + rate = ADV7511_SAMPLE_FREQ_96000; + break; + case 176400: + rate = ADV7511_SAMPLE_FREQ_176400; + break; + case 192000: + rate = ADV7511_SAMPLE_FREQ_192000; + break; + default: + return -EINVAL; + } + + switch (hparms->sample_width) { + case 16: + len = ADV7511_I2S_SAMPLE_LEN_16; + break; + case 18: + len = ADV7511_I2S_SAMPLE_LEN_18; + break; + case 20: + len = ADV7511_I2S_SAMPLE_LEN_20; + break; + case 24: + len = ADV7511_I2S_SAMPLE_LEN_24; + break; + default: + return -EINVAL; + } + + switch (fmt->fmt) { + case HDMI_I2S: + audio_source = ADV7511_AUDIO_SOURCE_I2S; + i2s_format = ADV7511_I2S_FORMAT_I2S; + break; + case HDMI_RIGHT_J: + audio_source = ADV7511_AUDIO_SOURCE_I2S; + i2s_format = ADV7511_I2S_FORMAT_RIGHT_J; + break; + case HDMI_LEFT_J: + audio_source = ADV7511_AUDIO_SOURCE_I2S; + i2s_format = ADV7511_I2S_FORMAT_LEFT_J; + break; + default: + return -EINVAL; + } + + invert_clock = fmt->bit_clk_inv; + + regmap_update_bits(adv7511->regmap, ADV7511_REG_AUDIO_SOURCE, 0x70, + audio_source << 4); + regmap_update_bits(adv7511->regmap, ADV7511_REG_AUDIO_CONFIG, BIT(6), + invert_clock << 6); + regmap_update_bits(adv7511->regmap, ADV7511_REG_I2S_CONFIG, 0x03, + i2s_format); + + adv7511->audio_source = audio_source; + + adv7511->f_audio = hparms->sample_rate; + + adv7511_update_cts_n(adv7511); + + regmap_update_bits(adv7511->regmap, ADV7511_REG_AUDIO_CFG3, + ADV7511_AUDIO_CFG3_LEN_MASK, len); + regmap_update_bits(adv7511->regmap, ADV7511_REG_I2C_FREQ_ID_CFG, + ADV7511_I2C_FREQ_ID_CFG_RATE_MASK, rate << 4); + regmap_write(adv7511->regmap, 0x73, 0x1); + + return 0; +} + +static int audio_startup(struct device *dev, void *data) +{ + struct adv7511 *adv7511 = dev_get_drvdata(dev); + + regmap_update_bits(adv7511->regmap, ADV7511_REG_AUDIO_CONFIG, + BIT(7), 0); + + /* hide Audio infoframe updates */ + regmap_update_bits(adv7511->regmap, ADV7511_REG_INFOFRAME_UPDATE, + BIT(5), BIT(5)); + /* enable N/CTS, enable Audio sample packets */ + regmap_update_bits(adv7511->regmap, ADV7511_REG_PACKET_ENABLE1, + BIT(5), BIT(5)); + /* enable N/CTS */ + regmap_update_bits(adv7511->regmap, ADV7511_REG_PACKET_ENABLE1, + BIT(6), BIT(6)); + /* not copyrighted */ + regmap_update_bits(adv7511->regmap, ADV7511_REG_AUDIO_CFG1, + BIT(5), BIT(5)); + /* enable audio infoframes */ + regmap_update_bits(adv7511->regmap, ADV7511_REG_PACKET_ENABLE1, + BIT(3), BIT(3)); + /* AV mute disable */ + regmap_update_bits(adv7511->regmap, ADV7511_REG_GC(0), + BIT(7) | BIT(6), BIT(7)); + /* use Audio infoframe updated info */ + regmap_update_bits(adv7511->regmap, ADV7511_REG_GC(1), + BIT(5), 0); + return 0; +} + +static void audio_shutdown(struct device *dev, void *data) +{ +} + +static const struct hdmi_codec_ops adv7511_codec_ops = { + .hw_params = adv7511_hdmi_hw_params, + .audio_shutdown = audio_shutdown, + .audio_startup = audio_startup, +}; + +static struct hdmi_codec_pdata codec_data = { + .ops = &adv7511_codec_ops, + .max_i2s_channels = 2, + .i2s = 1, +}; + +int adv7511_audio_init(struct device *dev, struct adv7511 *adv7511) +{ + adv7511->audio_pdev = platform_device_register_data(dev, + HDMI_CODEC_DRV_NAME, + PLATFORM_DEVID_AUTO, + &codec_data, + sizeof(codec_data)); + return PTR_ERR_OR_ZERO(adv7511->audio_pdev); +} + +void adv7511_audio_exit(struct adv7511 *adv7511) +{ + if (adv7511->audio_pdev) { + platform_device_unregister(adv7511->audio_pdev); + adv7511->audio_pdev = NULL; + } +} diff --git a/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c b/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c index 8ed3906dd411ec..4bbfcd03dc51f7 100644 --- a/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c +++ b/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c @@ -325,7 +325,7 @@ static void adv7511_set_link_config(struct adv7511 *adv7511, adv7511->rgb = config->input_colorspace == HDMI_COLORSPACE_RGB; } -static void adv7511_power_on(struct adv7511 *adv7511) +static void __adv7511_power_on(struct adv7511 *adv7511) { adv7511->current_edid_segment = -1; @@ -338,7 +338,7 @@ static void adv7511_power_on(struct adv7511 *adv7511) * Still, let's be safe and stick to the documentation. */ regmap_write(adv7511->regmap, ADV7511_REG_INT_ENABLE(0), - ADV7511_INT0_EDID_READY); + ADV7511_INT0_EDID_READY | ADV7511_INT0_HPD); regmap_write(adv7511->regmap, ADV7511_REG_INT_ENABLE(1), ADV7511_INT1_DDC_ERROR); } @@ -359,24 +359,30 @@ static void adv7511_power_on(struct adv7511 *adv7511) * Most of the registers are reset during power down or when HPD is low. */ regcache_sync(adv7511->regmap); +} +static void adv7511_power_on(struct adv7511 *adv7511) +{ + __adv7511_power_on(adv7511); if (adv7511->type == ADV7533) adv7533_dsi_power_on(adv7511); - adv7511->powered = true; } -static void adv7511_power_off(struct adv7511 *adv7511) +static void __adv7511_power_off(struct adv7511 *adv7511) { /* TODO: setup additional power down modes */ regmap_update_bits(adv7511->regmap, ADV7511_REG_POWER, ADV7511_POWER_POWER_DOWN, ADV7511_POWER_POWER_DOWN); regcache_mark_dirty(adv7511->regmap); +} +static void adv7511_power_off(struct adv7511 *adv7511) +{ + __adv7511_power_off(adv7511); if (adv7511->type == ADV7533) adv7533_dsi_power_off(adv7511); - adv7511->powered = false; } @@ -402,6 +408,27 @@ static bool adv7511_hpd(struct adv7511 *adv7511) return false; } +static void adv7511_hpd_work(struct work_struct *work) +{ + struct adv7511 *adv7511 = container_of(work, struct adv7511, hpd_work); + enum drm_connector_status status; + unsigned int val; + int ret; + + ret = regmap_read(adv7511->regmap, ADV7511_REG_STATUS, &val); + if (ret < 0) + status = connector_status_disconnected; + else if (val & ADV7511_STATUS_HPD) + status = connector_status_connected; + else + status = connector_status_disconnected; + + if (adv7511->connector.status != status) + drm_kms_helper_hotplug_event(adv7511->connector.dev); + + adv7511->connector.status = status; +} + static int adv7511_irq_process(struct adv7511 *adv7511, bool process_hpd) { unsigned int irq0, irq1; @@ -419,7 +446,7 @@ static int adv7511_irq_process(struct adv7511 *adv7511, bool process_hpd) regmap_write(adv7511->regmap, ADV7511_REG_INT(1), irq1); if (process_hpd && irq0 & ADV7511_INT0_HPD && adv7511->bridge.encoder) - drm_helper_hpd_irq_event(adv7511->connector.dev); + schedule_work(&adv7511->hpd_work); if (irq0 & ADV7511_INT0_EDID_READY || irq1 & ADV7511_INT1_DDC_ERROR) { adv7511->edid_read = true; @@ -545,24 +572,13 @@ static int adv7511_get_modes(struct adv7511 *adv7511, unsigned int count; /* Reading the EDID only works if the device is powered */ - if (!adv7511->powered) { - regmap_update_bits(adv7511->regmap, ADV7511_REG_POWER, - ADV7511_POWER_POWER_DOWN, 0); - if (adv7511->i2c_main->irq) { - regmap_write(adv7511->regmap, ADV7511_REG_INT_ENABLE(0), - ADV7511_INT0_EDID_READY); - regmap_write(adv7511->regmap, ADV7511_REG_INT_ENABLE(1), - ADV7511_INT1_DDC_ERROR); - } - adv7511->current_edid_segment = -1; - } + if (!adv7511->powered) + __adv7511_power_on(adv7511); edid = drm_do_get_edid(connector, adv7511_get_edid_block, adv7511); if (!adv7511->powered) - regmap_update_bits(adv7511->regmap, ADV7511_REG_POWER, - ADV7511_POWER_POWER_DOWN, - ADV7511_POWER_POWER_DOWN); + __adv7511_power_off(adv7511); kfree(adv7511->edid); adv7511->edid = edid; @@ -622,8 +638,24 @@ static int adv7511_mode_valid(struct adv7511 *adv7511, { if (mode->clock > 165000) return MODE_CLOCK_HIGH; - - return MODE_OK; + /* + * some work well modes which want to put in the front of the mode list. + */ + DRM_DEBUG("Checking mode %ix%i@%i clock: %i...", + mode->hdisplay, mode->vdisplay, drm_mode_vrefresh(mode), mode->clock); + if ((mode->hdisplay == 1920 && mode->vdisplay == 1080 && mode->clock == 148500) || + (mode->hdisplay == 1280 && mode->vdisplay == 800 && mode->clock == 83496) || + (mode->hdisplay == 1280 && mode->vdisplay == 720 && mode->clock == 74440) || + (mode->hdisplay == 1280 && mode->vdisplay == 720 && mode->clock == 74250) || + (mode->hdisplay == 1024 && mode->vdisplay == 768 && mode->clock == 75000) || + (mode->hdisplay == 1024 && mode->vdisplay == 768 && mode->clock == 81833) || + (mode->hdisplay == 800 && mode->vdisplay == 600 && mode->clock == 40000)) { + mode->type |= DRM_MODE_TYPE_PREFERRED; + DRM_DEBUG("OK\n"); + return MODE_OK; + } + DRM_DEBUG("BAD\n"); + return MODE_BAD; } static void adv7511_mode_set(struct adv7511 *adv7511, @@ -825,6 +857,10 @@ static int adv7511_bridge_attach(struct drm_bridge *bridge) if (adv->type == ADV7533) ret = adv7533_attach_dsi(adv); + if (adv->i2c_main->irq) + regmap_write(adv->regmap, ADV7511_REG_INT_ENABLE(0), + ADV7511_INT0_HPD); + return ret; } @@ -1006,6 +1042,8 @@ static int adv7511_probe(struct i2c_client *i2c, const struct i2c_device_id *id) goto err_i2c_unregister_edid; } + INIT_WORK(&adv7511->hpd_work, adv7511_hpd_work); + if (i2c->irq) { init_waitqueue_head(&adv7511->wq); @@ -1037,6 +1075,8 @@ static int adv7511_probe(struct i2c_client *i2c, const struct i2c_device_id *id) goto err_unregister_cec; } + adv7511_audio_init(dev, adv7511); + return 0; err_unregister_cec: @@ -1058,6 +1098,8 @@ static int adv7511_remove(struct i2c_client *i2c) drm_bridge_remove(&adv7511->bridge); + adv7511_audio_exit(adv7511); + i2c_unregister_device(adv7511->i2c_edid); kfree(adv7511->edid); diff --git a/drivers/gpu/drm/bridge/adv7511/adv7533.c b/drivers/gpu/drm/bridge/adv7511/adv7533.c index d7f7b7ce8ebe27..8b210373cfa272 100644 --- a/drivers/gpu/drm/bridge/adv7511/adv7533.c +++ b/drivers/gpu/drm/bridge/adv7511/adv7533.c @@ -29,6 +29,7 @@ static const struct reg_sequence adv7533_cec_fixed_registers[] = { { 0x17, 0xd0 }, { 0x24, 0x20 }, { 0x57, 0x11 }, + { 0x05, 0xc8 }, }; static const struct regmap_config adv7533_cec_regmap_config = { diff --git a/drivers/gpu/drm/drm_atomic.c b/drivers/gpu/drm/drm_atomic.c index e6862a7442104f..4e19bde4bbffac 100644 --- a/drivers/gpu/drm/drm_atomic.c +++ b/drivers/gpu/drm/drm_atomic.c @@ -1759,16 +1759,16 @@ int drm_mode_atomic_ioctl(struct drm_device *dev, if (ret && arg->flags & DRM_MODE_PAGE_FLIP_EVENT) { /* - * TEST_ONLY and PAGE_FLIP_EVENT are mutually exclusive, - * if they weren't, this code should be called on success - * for TEST_ONLY too. + * Free the allocated event. drm_atomic_helper_setup_commit + * can allocate an event too, so only free it if it's ours + * to prevent a double free in drm_atomic_state_clear. */ - for_each_crtc_in_state(state, crtc, crtc_state, i) { - if (!crtc_state->event) - continue; - - drm_event_cancel_free(dev, &crtc_state->event->base); + struct drm_pending_vblank_event *event = crtc_state->event; + if (event && (event->base.fence || event->base.file_priv)) { + drm_event_cancel_free(dev, &event->base); + crtc_state->event = NULL; + } } } diff --git a/drivers/gpu/drm/drm_atomic_helper.c b/drivers/gpu/drm/drm_atomic_helper.c index 21f9926055415e..a05bb389111910 100644 --- a/drivers/gpu/drm/drm_atomic_helper.c +++ b/drivers/gpu/drm/drm_atomic_helper.c @@ -1253,8 +1253,10 @@ int drm_atomic_helper_commit(struct drm_device *dev, if (!nonblock) { ret = drm_atomic_helper_wait_for_fences(dev, state, true); - if (ret) + if (ret) { + drm_atomic_helper_cleanup_planes(dev, state); return ret; + } } /* @@ -3113,6 +3115,8 @@ void __drm_atomic_helper_plane_duplicate_state(struct drm_plane *plane, if (state->fb) drm_framebuffer_reference(state->fb); + + state->fence = NULL; } EXPORT_SYMBOL(__drm_atomic_helper_plane_duplicate_state); diff --git a/drivers/gpu/drm/drm_fb_cma_helper.c b/drivers/gpu/drm/drm_fb_cma_helper.c index 1fd6eac1400c1c..887b946aec7016 100644 --- a/drivers/gpu/drm/drm_fb_cma_helper.c +++ b/drivers/gpu/drm/drm_fb_cma_helper.c @@ -27,6 +27,11 @@ #include #define DEFAULT_FBDEFIO_DELAY_MS 50 +#ifdef CONFIG_DRM_CMA_FBDEV_BUFFER_NUM +#define FBDEV_BUFFER_NUM CONFIG_DRM_CMA_FBDEV_BUFFER_NUM +#else +#define FBDEV_BUFFER_NUM 1 +#endif struct drm_fb_cma { struct drm_framebuffer fb; @@ -403,7 +408,7 @@ int drm_fbdev_cma_create_with_funcs(struct drm_fb_helper *helper, bytes_per_pixel = DIV_ROUND_UP(sizes->surface_bpp, 8); mode_cmd.width = sizes->surface_width; - mode_cmd.height = sizes->surface_height; + mode_cmd.height = sizes->surface_height * FBDEV_BUFFER_NUM; mode_cmd.pitches[0] = sizes->surface_width * bytes_per_pixel; mode_cmd.pixel_format = drm_mode_legacy_fb_format(sizes->surface_bpp, sizes->surface_depth); diff --git a/drivers/gpu/drm/drm_irq.c b/drivers/gpu/drm/drm_irq.c index b969a64a15140c..48a6167f5e7ba5 100644 --- a/drivers/gpu/drm/drm_irq.c +++ b/drivers/gpu/drm/drm_irq.c @@ -952,8 +952,10 @@ static u32 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe, u32 vblank_count; unsigned int seq; - if (WARN_ON(pipe >= dev->num_crtcs)) + if (WARN_ON(pipe >= dev->num_crtcs)) { + *vblanktime = (struct timeval) { 0 }; return 0; + } do { seq = read_seqbegin(&vblank->seqlock); diff --git a/drivers/gpu/drm/drm_mm.c b/drivers/gpu/drm/drm_mm.c index 11d44a1e0ab355..ee07bb4a57b74e 100644 --- a/drivers/gpu/drm/drm_mm.c +++ b/drivers/gpu/drm/drm_mm.c @@ -839,6 +839,7 @@ void drm_mm_init(struct drm_mm * mm, u64 start, u64 size) /* Clever trick to avoid a special case in the free hole tracking. */ INIT_LIST_HEAD(&mm->head_node.node_list); + mm->head_node.allocated = 0; mm->head_node.hole_follows = 1; mm->head_node.scanned_block = 0; mm->head_node.scanned_prev_free = 0; diff --git a/drivers/gpu/drm/drm_modes.c b/drivers/gpu/drm/drm_modes.c index 53f07ac7c1749d..e14366de0e6eb1 100644 --- a/drivers/gpu/drm/drm_modes.c +++ b/drivers/gpu/drm/drm_modes.c @@ -1462,6 +1462,13 @@ drm_mode_create_from_cmdline_mode(struct drm_device *dev, return NULL; mode->type |= DRM_MODE_TYPE_USERDEF; + /* fix up 1368x768: GFT/CVT can't express 1366 width due to alignment */ + if (cmd->xres == 1366 && mode->hdisplay == 1368) { + mode->hdisplay = 1366; + mode->hsync_start--; + mode->hsync_end--; + drm_mode_set_name(mode); + } drm_mode_set_crtcinfo(mode, CRTC_INTERLACE_HALVE_V); return mode; } diff --git a/drivers/gpu/drm/drm_probe_helper.c b/drivers/gpu/drm/drm_probe_helper.c index f6b64d7d3528d7..276474d13763b9 100644 --- a/drivers/gpu/drm/drm_probe_helper.c +++ b/drivers/gpu/drm/drm_probe_helper.c @@ -143,8 +143,18 @@ void drm_kms_helper_poll_enable_locked(struct drm_device *dev) } if (dev->mode_config.delayed_event) { + /* + * FIXME: + * + * Use short (1s) delay to handle the initial delayed event. + * This delay should not be needed, but Optimus/nouveau will + * fail in a mysterious way if the delayed event is handled as + * soon as possible like it is done in + * drm_helper_probe_single_connector_modes() in case the poll + * was enabled before. + */ poll = true; - delay = 0; + delay = HZ; } if (poll) diff --git a/drivers/gpu/drm/gma500/psb_drv.c b/drivers/gpu/drm/gma500/psb_drv.c index 50eb944fb78a8e..8f3ca526bd1bd4 100644 --- a/drivers/gpu/drm/gma500/psb_drv.c +++ b/drivers/gpu/drm/gma500/psb_drv.c @@ -473,6 +473,9 @@ static const struct file_operations psb_gem_fops = { .open = drm_open, .release = drm_release, .unlocked_ioctl = psb_unlocked_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = drm_compat_ioctl, +#endif .mmap = drm_gem_mmap, .poll = drm_poll, .read = drm_read, diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c index 18dfdd5c1b3b1b..670beebc32f619 100644 --- a/drivers/gpu/drm/i915/i915_drv.c +++ b/drivers/gpu/drm/i915/i915_drv.c @@ -2372,7 +2372,7 @@ static int intel_runtime_suspend(struct device *kdev) assert_forcewakes_inactive(dev_priv); - if (!IS_VALLEYVIEW(dev_priv) || !IS_CHERRYVIEW(dev_priv)) + if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) intel_hpd_poll_init(dev_priv); DRM_DEBUG_KMS("Device suspended\n"); diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index 685e9e06528798..da832d3cdca7e0 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -3684,6 +3684,8 @@ extern void intel_display_print_error_state(struct drm_i915_error_state_buf *e, int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val); int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u32 mbox, u32 val); +int skl_pcode_request(struct drm_i915_private *dev_priv, u32 mbox, u32 request, + u32 reply_mask, u32 reply, int timeout_base_ms); /* intel_sideband.c */ u32 vlv_punit_read(struct drm_i915_private *dev_priv, u32 addr); diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c index 5b6f81c1dbca44..7467355e4a18e0 100644 --- a/drivers/gpu/drm/i915/i915_gem_evict.c +++ b/drivers/gpu/drm/i915/i915_gem_evict.c @@ -194,6 +194,7 @@ i915_gem_evict_something(struct i915_address_space *vm, } /* Unbinding will emit any required flushes */ + ret = 0; while (!list_empty(&eviction_list)) { vma = list_first_entry(&eviction_list, struct i915_vma, diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c index a218c2e395e759..0c400f852a7686 100644 --- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c +++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c @@ -1215,14 +1215,14 @@ validate_exec_list(struct drm_device *dev, if (exec[i].offset != gen8_canonical_addr(exec[i].offset & PAGE_MASK)) return -EINVAL; - - /* From drm_mm perspective address space is continuous, - * so from this point we're always using non-canonical - * form internally. - */ - exec[i].offset = gen8_noncanonical_addr(exec[i].offset); } + /* From drm_mm perspective address space is continuous, + * so from this point we're always using non-canonical + * form internally. + */ + exec[i].offset = gen8_noncanonical_addr(exec[i].offset); + if (exec[i].alignment && !is_power_of_2(exec[i].alignment)) return -EINVAL; diff --git a/drivers/gpu/drm/i915/i915_gem_request.h b/drivers/gpu/drm/i915/i915_gem_request.h index 974bd7bcc801f9..59ac9002555221 100644 --- a/drivers/gpu/drm/i915/i915_gem_request.h +++ b/drivers/gpu/drm/i915/i915_gem_request.h @@ -344,6 +344,25 @@ i915_gem_active_set(struct i915_gem_active *active, rcu_assign_pointer(active->request, request); } +/** + * i915_gem_active_set_retire_fn - updates the retirement callback + * @active - the active tracker + * @fn - the routine called when the request is retired + * @mutex - struct_mutex used to guard retirements + * + * i915_gem_active_set_retire_fn() updates the function pointer that + * is called when the final request associated with the @active tracker + * is retired. + */ +static inline void +i915_gem_active_set_retire_fn(struct i915_gem_active *active, + i915_gem_retire_fn fn, + struct mutex *mutex) +{ + lockdep_assert_held(mutex); + active->retire = fn ?: i915_gem_retire_noop; +} + static inline struct drm_i915_gem_request * __i915_gem_active_peek(const struct i915_gem_active *active) { diff --git a/drivers/gpu/drm/i915/i915_gem_stolen.c b/drivers/gpu/drm/i915/i915_gem_stolen.c index 59989e8ee5dc8b..9a71ed546b9063 100644 --- a/drivers/gpu/drm/i915/i915_gem_stolen.c +++ b/drivers/gpu/drm/i915/i915_gem_stolen.c @@ -55,10 +55,9 @@ int i915_gem_stolen_insert_node_in_range(struct drm_i915_private *dev_priv, return -ENODEV; /* See the comment at the drm_mm_init() call for more about this check. - * WaSkipStolenMemoryFirstPage:bdw,chv,kbl (incomplete) + * WaSkipStolenMemoryFirstPage:bdw+ (incomplete) */ - if (start < 4096 && (IS_GEN8(dev_priv) || - IS_KBL_REVID(dev_priv, 0, KBL_REVID_A0))) + if (start < 4096 && INTEL_GEN(dev_priv) >= 8) start = 4096; mutex_lock(&dev_priv->mm.stolen_lock); diff --git a/drivers/gpu/drm/i915/i915_sysfs.c b/drivers/gpu/drm/i915/i915_sysfs.c index 1012eeea132434..306fc54c161b1d 100644 --- a/drivers/gpu/drm/i915/i915_sysfs.c +++ b/drivers/gpu/drm/i915/i915_sysfs.c @@ -460,7 +460,7 @@ static ssize_t gt_min_freq_mhz_store(struct device *kdev, static DEVICE_ATTR(gt_act_freq_mhz, S_IRUGO, gt_act_freq_mhz_show, NULL); static DEVICE_ATTR(gt_cur_freq_mhz, S_IRUGO, gt_cur_freq_mhz_show, NULL); -static DEVICE_ATTR(gt_boost_freq_mhz, S_IRUGO, gt_boost_freq_mhz_show, gt_boost_freq_mhz_store); +static DEVICE_ATTR(gt_boost_freq_mhz, S_IRUGO | S_IWUSR, gt_boost_freq_mhz_show, gt_boost_freq_mhz_store); static DEVICE_ATTR(gt_max_freq_mhz, S_IRUGO | S_IWUSR, gt_max_freq_mhz_show, gt_max_freq_mhz_store); static DEVICE_ATTR(gt_min_freq_mhz, S_IRUGO | S_IWUSR, gt_min_freq_mhz_show, gt_min_freq_mhz_store); diff --git a/drivers/gpu/drm/i915/intel_crt.c b/drivers/gpu/drm/i915/intel_crt.c index dfbcf16b41dfb9..4149a0fbe8bd0f 100644 --- a/drivers/gpu/drm/i915/intel_crt.c +++ b/drivers/gpu/drm/i915/intel_crt.c @@ -499,6 +499,7 @@ static bool intel_crt_detect_ddc(struct drm_connector *connector) struct drm_i915_private *dev_priv = to_i915(crt->base.base.dev); struct edid *edid; struct i2c_adapter *i2c; + bool ret = false; BUG_ON(crt->base.type != INTEL_OUTPUT_ANALOG); @@ -515,17 +516,17 @@ static bool intel_crt_detect_ddc(struct drm_connector *connector) */ if (!is_digital) { DRM_DEBUG_KMS("CRT detected via DDC:0x50 [EDID]\n"); - return true; + ret = true; + } else { + DRM_DEBUG_KMS("CRT not detected via DDC:0x50 [EDID reports a digital panel]\n"); } - - DRM_DEBUG_KMS("CRT not detected via DDC:0x50 [EDID reports a digital panel]\n"); } else { DRM_DEBUG_KMS("CRT not detected via DDC:0x50 [no valid EDID found]\n"); } kfree(edid); - return false; + return ret; } static enum drm_connector_status diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c index 3cb70d73239bf4..b9be8a6141d845 100644 --- a/drivers/gpu/drm/i915/intel_display.c +++ b/drivers/gpu/drm/i915/intel_display.c @@ -2587,8 +2587,9 @@ intel_fill_fb_info(struct drm_i915_private *dev_priv, * We only keep the x/y offsets, so push all of the * gtt offset into the x/y offsets. */ - _intel_adjust_tile_offset(&x, &y, tile_size, - tile_width, tile_height, pitch_tiles, + _intel_adjust_tile_offset(&x, &y, + tile_width, tile_height, + tile_size, pitch_tiles, gtt_offset_rotated * tile_size, 0); gtt_offset_rotated += rot_info->plane[i].width * rot_info->plane[i].height; @@ -2975,6 +2976,9 @@ int skl_check_plane_surface(struct intel_plane_state *plane_state) unsigned int rotation = plane_state->base.rotation; int ret; + if (!plane_state->base.visible) + return 0; + /* Rotate src coordinates to match rotated GTT view */ if (intel_rotation_90_or_270(rotation)) drm_rect_rotate(&plane_state->base.src, @@ -4276,10 +4280,10 @@ static void page_flip_completed(struct intel_crtc *intel_crtc) drm_crtc_vblank_put(&intel_crtc->base); wake_up_all(&dev_priv->pending_flip_queue); - queue_work(dev_priv->wq, &work->unpin_work); - trace_i915_flip_complete(intel_crtc->plane, work->pending_flip_obj); + + queue_work(dev_priv->wq, &work->unpin_work); } static int intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc) @@ -6262,36 +6266,25 @@ skl_dpll0_disable(struct drm_i915_private *dev_priv) dev_priv->cdclk_pll.vco = 0; } -static bool skl_cdclk_pcu_ready(struct drm_i915_private *dev_priv) -{ - int ret; - u32 val; - - /* inform PCU we want to change CDCLK */ - val = SKL_CDCLK_PREPARE_FOR_CHANGE; - mutex_lock(&dev_priv->rps.hw_lock); - ret = sandybridge_pcode_read(dev_priv, SKL_PCODE_CDCLK_CONTROL, &val); - mutex_unlock(&dev_priv->rps.hw_lock); - - return ret == 0 && (val & SKL_CDCLK_READY_FOR_CHANGE); -} - -static bool skl_cdclk_wait_for_pcu_ready(struct drm_i915_private *dev_priv) -{ - return _wait_for(skl_cdclk_pcu_ready(dev_priv), 3000, 10) == 0; -} - static void skl_set_cdclk(struct drm_i915_private *dev_priv, int cdclk, int vco) { struct drm_device *dev = &dev_priv->drm; u32 freq_select, pcu_ack; + int ret; WARN_ON((cdclk == 24000) != (vco == 0)); DRM_DEBUG_DRIVER("Changing CDCLK to %d kHz (VCO %d kHz)\n", cdclk, vco); - if (!skl_cdclk_wait_for_pcu_ready(dev_priv)) { - DRM_ERROR("failed to inform PCU about cdclk change\n"); + mutex_lock(&dev_priv->rps.hw_lock); + ret = skl_pcode_request(dev_priv, SKL_PCODE_CDCLK_CONTROL, + SKL_CDCLK_PREPARE_FOR_CHANGE, + SKL_CDCLK_READY_FOR_CHANGE, + SKL_CDCLK_READY_FOR_CHANGE, 3); + mutex_unlock(&dev_priv->rps.hw_lock); + if (ret) { + DRM_ERROR("Failed to inform PCU about cdclk change (%d)\n", + ret); return; } @@ -6876,6 +6869,12 @@ static void intel_crtc_disable_noatomic(struct drm_crtc *crtc) } state = drm_atomic_state_alloc(crtc->dev); + if (!state) { + DRM_DEBUG_KMS("failed to disable [CRTC:%d:%s], out of memory", + crtc->base.id, crtc->name); + return; + } + state->acquire_ctx = crtc->dev->mode_config.acquire_ctx; /* Everything's already locked, -EDEADLK can't happen. */ @@ -13970,8 +13969,9 @@ static int intel_modeset_checks(struct drm_atomic_state *state) DRM_DEBUG_KMS("New cdclk calculated to be atomic %u, actual %u\n", intel_state->cdclk, intel_state->dev_cdclk); - } else + } else { to_intel_atomic_state(state)->cdclk = dev_priv->atomic_cdclk_freq; + } intel_modeset_clear_plls(state); @@ -14072,8 +14072,9 @@ static int intel_atomic_check(struct drm_device *dev, if (ret) return ret; - } else - intel_state->cdclk = dev_priv->cdclk_freq; + } else { + intel_state->cdclk = dev_priv->atomic_cdclk_freq; + } ret = drm_atomic_helper_check_planes(dev, state); if (ret) @@ -16441,6 +16442,7 @@ void intel_modeset_init(struct drm_device *dev) intel_update_czclk(dev_priv); intel_update_cdclk(dev); + dev_priv->atomic_cdclk_freq = dev_priv->cdclk_freq; intel_shared_dpll_init(dev); @@ -16757,7 +16759,6 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev) for_each_intel_crtc(dev, crtc) { struct intel_crtc_state *crtc_state = crtc->config; - int pixclk = 0; __drm_atomic_helper_crtc_destroy_state(&crtc_state->base); memset(crtc_state, 0, sizeof(*crtc_state)); @@ -16769,23 +16770,9 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev) crtc->base.enabled = crtc_state->base.enable; crtc->active = crtc_state->base.active; - if (crtc_state->base.active) { + if (crtc_state->base.active) dev_priv->active_crtcs |= 1 << crtc->pipe; - if (INTEL_GEN(dev_priv) >= 9 || IS_BROADWELL(dev_priv)) - pixclk = ilk_pipe_pixel_rate(crtc_state); - else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) - pixclk = crtc_state->base.adjusted_mode.crtc_clock; - else - WARN_ON(dev_priv->display.modeset_calc_cdclk); - - /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */ - if (IS_BROADWELL(dev_priv) && crtc_state->ips_enabled) - pixclk = DIV_ROUND_UP(pixclk * 100, 95); - } - - dev_priv->min_pixclk[crtc->pipe] = pixclk; - readout_plane_state(crtc); DRM_DEBUG_KMS("[CRTC:%d:%s] hw state readout: %s\n", @@ -16859,6 +16846,8 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev) } for_each_intel_crtc(dev, crtc) { + int pixclk = 0; + crtc->base.hwmode = crtc->config->base.adjusted_mode; memset(&crtc->base.mode, 0, sizeof(crtc->base.mode)); @@ -16886,10 +16875,23 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev) */ crtc->base.state->mode.private_flags = I915_MODE_FLAG_INHERITED; + if (INTEL_GEN(dev_priv) >= 9 || IS_BROADWELL(dev_priv)) + pixclk = ilk_pipe_pixel_rate(crtc->config); + else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) + pixclk = crtc->config->base.adjusted_mode.crtc_clock; + else + WARN_ON(dev_priv->display.modeset_calc_cdclk); + + /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */ + if (IS_BROADWELL(dev_priv) && crtc->config->ips_enabled) + pixclk = DIV_ROUND_UP(pixclk * 100, 95); + drm_calc_timestamping_constants(&crtc->base, &crtc->base.hwmode); update_scanline_offset(crtc); } + dev_priv->min_pixclk[crtc->pipe] = pixclk; + intel_pipe_config_sanity_check(dev_priv, crtc->config); } } diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c index bf344d08356a2b..055525013d2f2b 100644 --- a/drivers/gpu/drm/i915/intel_dp.c +++ b/drivers/gpu/drm/i915/intel_dp.c @@ -280,7 +280,8 @@ intel_dp_init_panel_power_sequencer(struct drm_device *dev, struct intel_dp *intel_dp); static void intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev, - struct intel_dp *intel_dp); + struct intel_dp *intel_dp, + bool force_disable_vdd); static void intel_dp_pps_init(struct drm_device *dev, struct intel_dp *intel_dp); @@ -442,7 +443,7 @@ vlv_power_sequencer_pipe(struct intel_dp *intel_dp) /* init power sequencer on this pipe and port */ intel_dp_init_panel_power_sequencer(dev, intel_dp); - intel_dp_init_panel_power_sequencer_registers(dev, intel_dp); + intel_dp_init_panel_power_sequencer_registers(dev, intel_dp, true); /* * Even vdd force doesn't work until we've made @@ -479,7 +480,7 @@ bxt_power_sequencer_idx(struct intel_dp *intel_dp) * Only the HW needs to be reprogrammed, the SW state is fixed and * has been setup during connector init. */ - intel_dp_init_panel_power_sequencer_registers(dev, intel_dp); + intel_dp_init_panel_power_sequencer_registers(dev, intel_dp, false); return 0; } @@ -562,7 +563,7 @@ vlv_initial_power_sequencer_setup(struct intel_dp *intel_dp) port_name(port), pipe_name(intel_dp->pps_pipe)); intel_dp_init_panel_power_sequencer(dev, intel_dp); - intel_dp_init_panel_power_sequencer_registers(dev, intel_dp); + intel_dp_init_panel_power_sequencer_registers(dev, intel_dp, false); } void intel_power_sequencer_reset(struct drm_i915_private *dev_priv) @@ -2924,7 +2925,7 @@ static void vlv_init_panel_power_sequencer(struct intel_dp *intel_dp) /* init power sequencer on this pipe and port */ intel_dp_init_panel_power_sequencer(dev, intel_dp); - intel_dp_init_panel_power_sequencer_registers(dev, intel_dp); + intel_dp_init_panel_power_sequencer_registers(dev, intel_dp, true); } static void vlv_pre_enable_dp(struct intel_encoder *encoder, @@ -4017,6 +4018,11 @@ intel_dp_check_link_status(struct intel_dp *intel_dp) if (!to_intel_crtc(intel_encoder->base.crtc)->active) return; + /* FIXME: we need to synchronize this sort of stuff with hardware + * readout. Currently fast link training doesn't work on boot-up. */ + if (!intel_dp->lane_count) + return; + /* if link training is requested we should perform it always */ if ((intel_dp->compliance_test_type == DP_TEST_LINK_TRAINING) || (!drm_dp_channel_eq_ok(link_status, intel_dp->lane_count))) { @@ -5054,7 +5060,8 @@ intel_dp_init_panel_power_sequencer(struct drm_device *dev, static void intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev, - struct intel_dp *intel_dp) + struct intel_dp *intel_dp, + bool force_disable_vdd) { struct drm_i915_private *dev_priv = to_i915(dev); u32 pp_on, pp_off, pp_div, port_sel = 0; @@ -5067,6 +5074,31 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev, intel_pps_get_registers(dev_priv, intel_dp, ®s); + /* + * On some VLV machines the BIOS can leave the VDD + * enabled even on power seqeuencers which aren't + * hooked up to any port. This would mess up the + * power domain tracking the first time we pick + * one of these power sequencers for use since + * edp_panel_vdd_on() would notice that the VDD was + * already on and therefore wouldn't grab the power + * domain reference. Disable VDD first to avoid this. + * This also avoids spuriously turning the VDD on as + * soon as the new power seqeuencer gets initialized. + */ + if (force_disable_vdd) { + u32 pp = ironlake_get_pp_control(intel_dp); + + WARN(pp & PANEL_POWER_ON, "Panel power already on\n"); + + if (pp & EDP_FORCE_VDD) + DRM_DEBUG_KMS("VDD already on, disabling first\n"); + + pp &= ~EDP_FORCE_VDD; + + I915_WRITE(regs.pp_ctrl, pp); + } + pp_on = (seq->t1_t3 << PANEL_POWER_UP_DELAY_SHIFT) | (seq->t8 << PANEL_LIGHT_ON_DELAY_SHIFT); pp_off = (seq->t9 << PANEL_LIGHT_OFF_DELAY_SHIFT) | @@ -5119,7 +5151,7 @@ static void intel_dp_pps_init(struct drm_device *dev, vlv_initial_power_sequencer_setup(intel_dp); } else { intel_dp_init_panel_power_sequencer(dev, intel_dp); - intel_dp_init_panel_power_sequencer_registers(dev, intel_dp); + intel_dp_init_panel_power_sequencer_registers(dev, intel_dp, false); } } diff --git a/drivers/gpu/drm/i915/intel_dpll_mgr.c b/drivers/gpu/drm/i915/intel_dpll_mgr.c index 1c59ca50c4309c..cae27c55dd9962 100644 --- a/drivers/gpu/drm/i915/intel_dpll_mgr.c +++ b/drivers/gpu/drm/i915/intel_dpll_mgr.c @@ -1723,7 +1723,8 @@ bxt_get_dpll(struct intel_crtc *crtc, return NULL; if ((encoder->type == INTEL_OUTPUT_DP || - encoder->type == INTEL_OUTPUT_EDP) && + encoder->type == INTEL_OUTPUT_EDP || + encoder->type == INTEL_OUTPUT_DP_MST) && !bxt_ddi_dp_set_dpll_hw_state(clock, &dpll_hw_state)) return NULL; diff --git a/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c b/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c index cd154ce6b6c1ed..34601574fc6edb 100644 --- a/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c +++ b/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c @@ -296,7 +296,8 @@ static void chv_exec_gpio(struct drm_i915_private *dev_priv, mutex_lock(&dev_priv->sb_lock); vlv_iosf_sb_write(dev_priv, port, cfg1, 0); vlv_iosf_sb_write(dev_priv, port, cfg0, - CHV_GPIO_GPIOCFG_GPO | CHV_GPIO_GPIOTXSTATE(value)); + CHV_GPIO_GPIOEN | CHV_GPIO_GPIOCFG_GPO | + CHV_GPIO_GPIOTXSTATE(value)); mutex_unlock(&dev_priv->sb_lock); } diff --git a/drivers/gpu/drm/i915/intel_fbdev.c b/drivers/gpu/drm/i915/intel_fbdev.c index b7098f98bb671d..9127e57f383c10 100644 --- a/drivers/gpu/drm/i915/intel_fbdev.c +++ b/drivers/gpu/drm/i915/intel_fbdev.c @@ -745,6 +745,9 @@ void intel_fbdev_initial_config_async(struct drm_device *dev) { struct intel_fbdev *ifbdev = to_i915(dev)->fbdev; + if (!ifbdev) + return; + ifbdev->cookie = async_schedule(intel_fbdev_initial_config, ifbdev); } diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c index 0adb879833ff0f..4147e51cf8930a 100644 --- a/drivers/gpu/drm/i915/intel_lrc.c +++ b/drivers/gpu/drm/i915/intel_lrc.c @@ -858,8 +858,7 @@ static inline int gen8_emit_flush_coherentl3_wa(struct intel_engine_cs *engine, * this batch updates GEN8_L3SQCREG4 with default value we need to * set this bit here to retain the WA during flush. */ - if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_E0) || - IS_KBL_REVID(dev_priv, 0, KBL_REVID_E0)) + if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_E0)) l3sqc4_flush |= GEN8_LQSC_RO_PERF_DIS; wa_ctx_emit(batch, index, (MI_STORE_REGISTER_MEM_GEN8 | @@ -2153,30 +2152,42 @@ static int execlists_context_deferred_alloc(struct i915_gem_context *ctx, void intel_lr_context_resume(struct drm_i915_private *dev_priv) { - struct i915_gem_context *ctx = dev_priv->kernel_context; struct intel_engine_cs *engine; + struct i915_gem_context *ctx; + + /* Because we emit WA_TAIL_DWORDS there may be a disparity + * between our bookkeeping in ce->ring->head and ce->ring->tail and + * that stored in context. As we only write new commands from + * ce->ring->tail onwards, everything before that is junk. If the GPU + * starts reading from its RING_HEAD from the context, it may try to + * execute that junk and die. + * + * So to avoid that we reset the context images upon resume. For + * simplicity, we just zero everything out. + */ + list_for_each_entry(ctx, &dev_priv->context_list, link) { + for_each_engine(engine, dev_priv) { + struct intel_context *ce = &ctx->engine[engine->id]; + u32 *reg; - for_each_engine(engine, dev_priv) { - struct intel_context *ce = &ctx->engine[engine->id]; - void *vaddr; - uint32_t *reg_state; - - if (!ce->state) - continue; - - vaddr = i915_gem_object_pin_map(ce->state->obj, I915_MAP_WB); - if (WARN_ON(IS_ERR(vaddr))) - continue; + if (!ce->state) + continue; - reg_state = vaddr + LRC_STATE_PN * PAGE_SIZE; + reg = i915_gem_object_pin_map(ce->state->obj, + I915_MAP_WB); + if (WARN_ON(IS_ERR(reg))) + continue; - reg_state[CTX_RING_HEAD+1] = 0; - reg_state[CTX_RING_TAIL+1] = 0; + reg += LRC_STATE_PN * PAGE_SIZE / sizeof(*reg); + reg[CTX_RING_HEAD+1] = 0; + reg[CTX_RING_TAIL+1] = 0; - ce->state->obj->dirty = true; - i915_gem_object_unpin_map(ce->state->obj); + ce->state->obj->dirty = true; + i915_gem_object_unpin_map(ce->state->obj); - ce->ring->head = 0; - ce->ring->tail = 0; + ce->ring->head = ce->ring->tail = 0; + ce->ring->last_retired_head = -1; + intel_ring_update_space(ce->ring); + } } } diff --git a/drivers/gpu/drm/i915/intel_overlay.c b/drivers/gpu/drm/i915/intel_overlay.c index a24bc8c7889f04..a2655cd5a84e2c 100644 --- a/drivers/gpu/drm/i915/intel_overlay.c +++ b/drivers/gpu/drm/i915/intel_overlay.c @@ -216,7 +216,8 @@ static void intel_overlay_submit_request(struct intel_overlay *overlay, { GEM_BUG_ON(i915_gem_active_peek(&overlay->last_flip, &overlay->i915->drm.struct_mutex)); - overlay->last_flip.retire = retire; + i915_gem_active_set_retire_fn(&overlay->last_flip, retire, + &overlay->i915->drm.struct_mutex); i915_gem_active_set(&overlay->last_flip, req); i915_add_request(req); } @@ -839,8 +840,8 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay, if (ret) goto out_unpin; - i915_gem_track_fb(overlay->vma->obj, new_bo, - INTEL_FRONTBUFFER_OVERLAY(pipe)); + i915_gem_track_fb(overlay->vma ? overlay->vma->obj : NULL, + vma->obj, INTEL_FRONTBUFFER_OVERLAY(pipe)); overlay->old_vma = overlay->vma; overlay->vma = vma; @@ -1430,6 +1431,8 @@ void intel_setup_overlay(struct drm_i915_private *dev_priv) overlay->contrast = 75; overlay->saturation = 146; + init_request_active(&overlay->last_flip, NULL); + regs = intel_overlay_map_regs(overlay); if (!regs) goto out_unpin_bo; diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c index db24f898853cbb..e559a45ff1f719 100644 --- a/drivers/gpu/drm/i915/intel_pm.c +++ b/drivers/gpu/drm/i915/intel_pm.c @@ -2879,6 +2879,21 @@ skl_wm_plane_id(const struct intel_plane *plane) } } +/* + * FIXME: We still don't have the proper code detect if we need to apply the WA, + * so assume we'll always need it in order to avoid underruns. + */ +static bool skl_needs_memory_bw_wa(struct intel_atomic_state *state) +{ + struct drm_i915_private *dev_priv = to_i915(state->base.dev); + + if (IS_SKYLAKE(dev_priv) || IS_BROXTON(dev_priv) || + IS_KABYLAKE(dev_priv)) + return true; + + return false; +} + static bool intel_has_sagv(struct drm_i915_private *dev_priv) { @@ -2940,24 +2955,10 @@ intel_enable_sagv(struct drm_i915_private *dev_priv) return 0; } -static int -intel_do_sagv_disable(struct drm_i915_private *dev_priv) -{ - int ret; - uint32_t temp = GEN9_SAGV_DISABLE; - - ret = sandybridge_pcode_read(dev_priv, GEN9_PCODE_SAGV_CONTROL, - &temp); - if (ret) - return ret; - else - return temp & GEN9_SAGV_IS_DISABLED; -} - int intel_disable_sagv(struct drm_i915_private *dev_priv) { - int ret, result; + int ret; if (!intel_has_sagv(dev_priv)) return 0; @@ -2969,25 +2970,23 @@ intel_disable_sagv(struct drm_i915_private *dev_priv) mutex_lock(&dev_priv->rps.hw_lock); /* bspec says to keep retrying for at least 1 ms */ - ret = wait_for(result = intel_do_sagv_disable(dev_priv), 1); + ret = skl_pcode_request(dev_priv, GEN9_PCODE_SAGV_CONTROL, + GEN9_SAGV_DISABLE, + GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED, + 1); mutex_unlock(&dev_priv->rps.hw_lock); - if (ret == -ETIMEDOUT) { - DRM_ERROR("Request to disable SAGV timed out\n"); - return -ETIMEDOUT; - } - /* * Some skl systems, pre-release machines in particular, * don't actually have an SAGV. */ - if (IS_SKYLAKE(dev_priv) && result == -ENXIO) { + if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) { DRM_DEBUG_DRIVER("No SAGV found on system, ignoring\n"); dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED; return 0; - } else if (result < 0) { - DRM_ERROR("Failed to disable the SAGV\n"); - return result; + } else if (ret < 0) { + DRM_ERROR("Failed to disable the SAGV (%d)\n", ret); + return ret; } dev_priv->sagv_status = I915_SAGV_DISABLED; @@ -2999,9 +2998,10 @@ bool intel_can_enable_sagv(struct drm_atomic_state *state) struct drm_device *dev = state->dev; struct drm_i915_private *dev_priv = to_i915(dev); struct intel_atomic_state *intel_state = to_intel_atomic_state(state); - struct drm_crtc *crtc; + struct intel_crtc *crtc; + struct intel_plane *plane; enum pipe pipe; - int level, plane; + int level, id, latency; if (!intel_has_sagv(dev_priv)) return false; @@ -3019,27 +3019,36 @@ bool intel_can_enable_sagv(struct drm_atomic_state *state) /* Since we're now guaranteed to only have one active CRTC... */ pipe = ffs(intel_state->active_crtcs) - 1; - crtc = dev_priv->pipe_to_crtc_mapping[pipe]; + crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]); - if (crtc->state->mode.flags & DRM_MODE_FLAG_INTERLACE) + if (crtc->base.state->mode.flags & DRM_MODE_FLAG_INTERLACE) return false; - for_each_plane(dev_priv, pipe, plane) { + for_each_intel_plane_on_crtc(dev, crtc, plane) { + id = skl_wm_plane_id(plane); + /* Skip this plane if it's not enabled */ - if (intel_state->wm_results.plane[pipe][plane][0] == 0) + if (intel_state->wm_results.plane[pipe][id][0] == 0) continue; /* Find the highest enabled wm level for this plane */ for (level = ilk_wm_max_level(dev); - intel_state->wm_results.plane[pipe][plane][level] == 0; --level) + intel_state->wm_results.plane[pipe][id][level] == 0; --level) { } + latency = dev_priv->wm.skl_latency[level]; + + if (skl_needs_memory_bw_wa(intel_state) && + plane->base.state->fb->modifier[0] == + I915_FORMAT_MOD_X_TILED) + latency += 15; + /* * If any of the planes on this pipe don't enable wm levels * that incur memory latencies higher then 30µs we can't enable * the SAGV */ - if (dev_priv->wm.skl_latency[level] < SKL_SAGV_BLOCK_TIME) + if (latency < SKL_SAGV_BLOCK_TIME) return false; } @@ -3549,12 +3558,18 @@ static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv, uint32_t width = 0, height = 0; uint32_t plane_pixel_rate; uint32_t y_tile_minimum, y_min_scanlines; + struct intel_atomic_state *state = + to_intel_atomic_state(cstate->base.state); + bool apply_memory_bw_wa = skl_needs_memory_bw_wa(state); if (latency == 0 || !cstate->base.active || !intel_pstate->base.visible) { *enabled = false; return 0; } + if (apply_memory_bw_wa && fb->modifier[0] == I915_FORMAT_MOD_X_TILED) + latency += 15; + width = drm_rect_width(&intel_pstate->base.src) >> 16; height = drm_rect_height(&intel_pstate->base.src) >> 16; @@ -3586,6 +3601,9 @@ static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv, y_min_scanlines = 4; } + if (apply_memory_bw_wa) + y_min_scanlines *= 2; + plane_bytes_per_line = width * cpp; if (fb->modifier[0] == I915_FORMAT_MOD_Y_TILED || fb->modifier[0] == I915_FORMAT_MOD_Yf_TILED) { @@ -7921,6 +7939,81 @@ int sandybridge_pcode_write(struct drm_i915_private *dev_priv, return 0; } +static bool skl_pcode_try_request(struct drm_i915_private *dev_priv, u32 mbox, + u32 request, u32 reply_mask, u32 reply, + u32 *status) +{ + u32 val = request; + + *status = sandybridge_pcode_read(dev_priv, mbox, &val); + + return *status || ((val & reply_mask) == reply); +} + +/** + * skl_pcode_request - send PCODE request until acknowledgment + * @dev_priv: device private + * @mbox: PCODE mailbox ID the request is targeted for + * @request: request ID + * @reply_mask: mask used to check for request acknowledgment + * @reply: value used to check for request acknowledgment + * @timeout_base_ms: timeout for polling with preemption enabled + * + * Keep resending the @request to @mbox until PCODE acknowledges it, PCODE + * reports an error or an overall timeout of @timeout_base_ms+10 ms expires. + * The request is acknowledged once the PCODE reply dword equals @reply after + * applying @reply_mask. Polling is first attempted with preemption enabled + * for @timeout_base_ms and if this times out for another 10 ms with + * preemption disabled. + * + * Returns 0 on success, %-ETIMEDOUT in case of a timeout, <0 in case of some + * other error as reported by PCODE. + */ +int skl_pcode_request(struct drm_i915_private *dev_priv, u32 mbox, u32 request, + u32 reply_mask, u32 reply, int timeout_base_ms) +{ + u32 status; + int ret; + + WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); + +#define COND skl_pcode_try_request(dev_priv, mbox, request, reply_mask, reply, \ + &status) + + /* + * Prime the PCODE by doing a request first. Normally it guarantees + * that a subsequent request, at most @timeout_base_ms later, succeeds. + * _wait_for() doesn't guarantee when its passed condition is evaluated + * first, so send the first request explicitly. + */ + if (COND) { + ret = 0; + goto out; + } + ret = _wait_for(COND, timeout_base_ms * 1000, 10); + if (!ret) + goto out; + + /* + * The above can time out if the number of requests was low (2 in the + * worst case) _and_ PCODE was busy for some reason even after a + * (queued) request and @timeout_base_ms delay. As a workaround retry + * the poll with preemption disabled to maximize the number of + * requests. Increase the timeout from @timeout_base_ms to 10ms to + * account for interrupts that could reduce the number of these + * requests. + */ + DRM_DEBUG_KMS("PCODE timeout, retrying with preemption disabled\n"); + WARN_ON_ONCE(timeout_base_ms > 3); + preempt_disable(); + ret = wait_for_atomic(COND, 10); + preempt_enable(); + +out: + return ret ? ret : status; +#undef COND +} + static int byt_gpu_freq(struct drm_i915_private *dev_priv, int val) { /* diff --git a/drivers/gpu/drm/i915/intel_psr.c b/drivers/gpu/drm/i915/intel_psr.c index 108ba1e5d65872..9b307cee300868 100644 --- a/drivers/gpu/drm/i915/intel_psr.c +++ b/drivers/gpu/drm/i915/intel_psr.c @@ -825,13 +825,9 @@ void intel_psr_init(struct drm_device *dev) dev_priv->psr_mmio_base = IS_HASWELL(dev_priv) ? HSW_EDP_PSR_BASE : BDW_EDP_PSR_BASE; - /* Per platform default */ - if (i915.enable_psr == -1) { - if (IS_HASWELL(dev) || IS_BROADWELL(dev)) - i915.enable_psr = 1; - else - i915.enable_psr = 0; - } + /* Per platform default: all disabled. */ + if (i915.enable_psr == -1) + i915.enable_psr = 0; /* Set link_standby x link_off defaults */ if (IS_HASWELL(dev) || IS_BROADWELL(dev)) diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c index ed9955dce156dd..8babfe0ce4e3d4 100644 --- a/drivers/gpu/drm/i915/intel_ringbuffer.c +++ b/drivers/gpu/drm/i915/intel_ringbuffer.c @@ -1153,14 +1153,6 @@ static int kbl_init_workarounds(struct intel_engine_cs *engine) WA_SET_BIT_MASKED(HDC_CHICKEN0, HDC_FENCE_DEST_SLM_DISABLE); - /* GEN8_L3SQCREG4 has a dependency with WA batch so any new changes - * involving this register should also be added to WA batch as required. - */ - if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_E0)) - /* WaDisableLSQCROPERFforOCL:kbl */ - I915_WRITE(GEN8_L3SQCREG4, I915_READ(GEN8_L3SQCREG4) | - GEN8_LQSC_RO_PERF_DIS); - /* WaToEnableHwFixForPushConstHWBug:kbl */ if (IS_KBL_REVID(dev_priv, KBL_REVID_C0, REVID_FOREVER)) WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2, diff --git a/drivers/gpu/drm/i915/intel_runtime_pm.c b/drivers/gpu/drm/i915/intel_runtime_pm.c index a38c2fefe85a6f..23ed3f5972fa4e 100644 --- a/drivers/gpu/drm/i915/intel_runtime_pm.c +++ b/drivers/gpu/drm/i915/intel_runtime_pm.c @@ -1065,7 +1065,18 @@ static bool vlv_power_well_enabled(struct drm_i915_private *dev_priv, static void vlv_init_display_clock_gating(struct drm_i915_private *dev_priv) { - I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE); + u32 val; + + /* + * On driver load, a pipe may be active and driving a DSI display. + * Preserve DPOUNIT_CLOCK_GATE_DISABLE to avoid the pipe getting stuck + * (and never recovering) in this case. intel_dsi_post_disable() will + * clear it when we turn off the display. + */ + val = I915_READ(DSPCLK_GATE_D); + val &= DPOUNIT_CLOCK_GATE_DISABLE; + val |= VRHUNIT_CLOCK_GATE_DISABLE; + I915_WRITE(DSPCLK_GATE_D, val); /* * Disable trickle feed and enable pnd deadline calculation diff --git a/drivers/gpu/drm/nouveau/dispnv04/hw.c b/drivers/gpu/drm/nouveau/dispnv04/hw.c index 74856a8b8f3594..e64f52464ecf55 100644 --- a/drivers/gpu/drm/nouveau/dispnv04/hw.c +++ b/drivers/gpu/drm/nouveau/dispnv04/hw.c @@ -222,6 +222,7 @@ nouveau_hw_get_clock(struct drm_device *dev, enum nvbios_pll_type plltype) uint32_t mpllP; pci_read_config_dword(pci_get_bus_and_slot(0, 3), 0x6c, &mpllP); + mpllP = (mpllP >> 8) & 0xf; if (!mpllP) mpllP = 4; @@ -232,7 +233,7 @@ nouveau_hw_get_clock(struct drm_device *dev, enum nvbios_pll_type plltype) uint32_t clock; pci_read_config_dword(pci_get_bus_and_slot(0, 5), 0x4c, &clock); - return clock; + return clock / 1000; } ret = nouveau_hw_get_pllvals(dev, plltype, &pllvals); diff --git a/drivers/gpu/drm/nouveau/nouveau_bios.c b/drivers/gpu/drm/nouveau/nouveau_bios.c index a1570b10943405..23ffe8571a990d 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bios.c +++ b/drivers/gpu/drm/nouveau/nouveau_bios.c @@ -333,6 +333,9 @@ get_fp_strap(struct drm_device *dev, struct nvbios *bios) if (bios->major_version < 5 && bios->data[0x48] & 0x4) return NVReadVgaCrtc5758(dev, 0, 0xf) & 0xf; + if (drm->device.info.family >= NV_DEVICE_INFO_V0_MAXWELL) + return nvif_rd32(device, 0x001800) & 0x0000000f; + else if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA) return (nvif_rd32(device, NV_PEXTDEV_BOOT_0) >> 24) & 0xf; else diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c index 343b8659472cfb..a2e6a81669e789 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bo.c +++ b/drivers/gpu/drm/nouveau/nouveau_bo.c @@ -1209,6 +1209,7 @@ nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem) nvbo->page_shift != vma->vm->mmu->lpg_shift)) { nvkm_vm_map(vma, new_mem->mm_node); } else { + WARN_ON(ttm_bo_wait(bo, false, false)); nvkm_vm_unmap(vma); } } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c index 7218a067a6c5e8..e0d7f8472ac652 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c @@ -1851,7 +1851,7 @@ nvf1_chipset = { .fb = gk104_fb_new, .fuse = gf100_fuse_new, .gpio = gk104_gpio_new, - .i2c = gf119_i2c_new, + .i2c = gk104_i2c_new, .ibus = gk104_ibus_new, .iccsense = gf100_iccsense_new, .imem = nv50_instmem_new, @@ -1965,7 +1965,7 @@ nv117_chipset = { .fb = gm107_fb_new, .fuse = gm107_fuse_new, .gpio = gk104_gpio_new, - .i2c = gf119_i2c_new, + .i2c = gk104_i2c_new, .ibus = gk104_ibus_new, .iccsense = gf100_iccsense_new, .imem = nv50_instmem_new, @@ -1999,7 +1999,7 @@ nv118_chipset = { .fb = gm107_fb_new, .fuse = gm107_fuse_new, .gpio = gk104_gpio_new, - .i2c = gf119_i2c_new, + .i2c = gk104_i2c_new, .ibus = gk104_ibus_new, .iccsense = gf100_iccsense_new, .imem = nv50_instmem_new, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdagt215.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdagt215.c index 6f0436df021953..f8f2f16c22a2a2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdagt215.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdagt215.c @@ -59,7 +59,7 @@ gt215_hda_eld(NV50_DISP_MTHD_V1) ); } for (i = 0; i < size; i++) - nvkm_wr32(device, 0x61c440 + soff, (i << 8) | args->v0.data[0]); + nvkm_wr32(device, 0x61c440 + soff, (i << 8) | args->v0.data[i]); for (; i < 0x60; i++) nvkm_wr32(device, 0x61c440 + soff, (i << 8)); nvkm_mask(device, 0x61c448 + soff, 0x80000003, 0x80000003); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c index cbc67f26232231..12d964260a2992 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c @@ -60,6 +60,7 @@ gf100_fifo_gpfifo_engine_fini(struct nvkm_fifo_chan *base, struct nvkm_gpuobj *inst = chan->base.inst; int ret = 0; + mutex_lock(&subdev->mutex); nvkm_wr32(device, 0x002634, chan->base.chid); if (nvkm_msec(device, 2000, if (nvkm_rd32(device, 0x002634) == chan->base.chid) @@ -67,10 +68,12 @@ gf100_fifo_gpfifo_engine_fini(struct nvkm_fifo_chan *base, ) < 0) { nvkm_error(subdev, "channel %d [%s] kick timeout\n", chan->base.chid, chan->base.object.client->name); - ret = -EBUSY; - if (suspend) - return ret; + ret = -ETIMEDOUT; } + mutex_unlock(&subdev->mutex); + + if (ret && suspend) + return ret; if (offset) { nvkm_kmap(inst); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c index ed4351032ed603..a2df4f3e776349 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c @@ -40,7 +40,9 @@ gk104_fifo_gpfifo_kick(struct gk104_fifo_chan *chan) struct nvkm_subdev *subdev = &fifo->base.engine.subdev; struct nvkm_device *device = subdev->device; struct nvkm_client *client = chan->base.object.client; + int ret = 0; + mutex_lock(&subdev->mutex); nvkm_wr32(device, 0x002634, chan->base.chid); if (nvkm_msec(device, 2000, if (!(nvkm_rd32(device, 0x002634) & 0x00100000)) @@ -48,10 +50,10 @@ gk104_fifo_gpfifo_kick(struct gk104_fifo_chan *chan) ) < 0) { nvkm_error(subdev, "channel %d [%s] kick timeout\n", chan->base.chid, client->name); - return -EBUSY; + ret = -ETIMEDOUT; } - - return 0; + mutex_unlock(&subdev->mutex); + return ret; } static u32 diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c index 157919c788e6f7..6584d505460ce4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c @@ -1755,6 +1755,50 @@ gf100_gr_ = { .object_get = gf100_gr_object_get, }; +int +gf100_gr_ctor_fw_legacy(struct gf100_gr *gr, const char *fwname, + struct gf100_gr_fuc *fuc, int ret) +{ + struct nvkm_subdev *subdev = &gr->base.engine.subdev; + struct nvkm_device *device = subdev->device; + const struct firmware *fw; + char f[32]; + + /* see if this firmware has a legacy path */ + if (!strcmp(fwname, "fecs_inst")) + fwname = "fuc409c"; + else if (!strcmp(fwname, "fecs_data")) + fwname = "fuc409d"; + else if (!strcmp(fwname, "gpccs_inst")) + fwname = "fuc41ac"; + else if (!strcmp(fwname, "gpccs_data")) + fwname = "fuc41ad"; + else { + /* nope, let's just return the error we got */ + nvkm_error(subdev, "failed to load %s\n", fwname); + return ret; + } + + /* yes, try to load from the legacy path */ + nvkm_debug(subdev, "%s: falling back to legacy path\n", fwname); + + snprintf(f, sizeof(f), "nouveau/nv%02x_%s", device->chipset, fwname); + ret = request_firmware(&fw, f, device->dev); + if (ret) { + snprintf(f, sizeof(f), "nouveau/%s", fwname); + ret = request_firmware(&fw, f, device->dev); + if (ret) { + nvkm_error(subdev, "failed to load %s\n", fwname); + return ret; + } + } + + fuc->size = fw->size; + fuc->data = kmemdup(fw->data, fuc->size, GFP_KERNEL); + release_firmware(fw); + return (fuc->data != NULL) ? 0 : -ENOMEM; +} + int gf100_gr_ctor_fw(struct gf100_gr *gr, const char *fwname, struct gf100_gr_fuc *fuc) @@ -1765,10 +1809,8 @@ gf100_gr_ctor_fw(struct gf100_gr *gr, const char *fwname, int ret; ret = nvkm_firmware_get(device, fwname, &fw); - if (ret) { - nvkm_error(subdev, "failed to load %s\n", fwname); - return ret; - } + if (ret) + return gf100_gr_ctor_fw_legacy(gr, fwname, fuc, ret); fuc->size = fw->size; fuc->data = kmemdup(fw->data, fuc->size, GFP_KERNEL); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/priv.h index 212800ecdce99e..7d1d3c6b4b728d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/priv.h @@ -12,6 +12,7 @@ struct nvbios_source { bool rw; bool ignore_checksum; bool no_pcir; + bool require_checksum; }; int nvbios_extend(struct nvkm_bios *, u32 length); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadow.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadow.c index b2557e87afdd6d..7deb81b6dbac6b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadow.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadow.c @@ -86,9 +86,12 @@ shadow_image(struct nvkm_bios *bios, int idx, u32 offset, struct shadow *mthd) nvbios_checksum(&bios->data[image.base], image.size)) { nvkm_debug(subdev, "%08x: checksum failed\n", image.base); - if (mthd->func->rw) + if (!mthd->func->require_checksum) { + if (mthd->func->rw) + score += 1; score += 1; - score += 1; + } else + return 0; } else { score += 3; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadowacpi.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadowacpi.c index 8fecb5ff22a0ef..06572f8ce91488 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadowacpi.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadowacpi.c @@ -99,6 +99,7 @@ nvbios_acpi_fast = { .init = acpi_init, .read = acpi_read_fast, .rw = false, + .require_checksum = true, }; const struct nvbios_source diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c index 39c2a38e54f74e..0c7ef250dcafc3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c @@ -47,8 +47,10 @@ nvkm_ltc_tags_clear(struct nvkm_ltc *ltc, u32 first, u32 count) BUG_ON((first > limit) || (limit >= ltc->num_tags)); + mutex_lock(<c->subdev.mutex); ltc->func->cbc_clear(ltc, first, limit); ltc->func->cbc_wait(ltc); + mutex_unlock(<c->subdev.mutex); } int diff --git a/drivers/gpu/drm/panel/panel-simple.c b/drivers/gpu/drm/panel/panel-simple.c index 113db3c4a633fc..27cb42467b2024 100644 --- a/drivers/gpu/drm/panel/panel-simple.c +++ b/drivers/gpu/drm/panel/panel-simple.c @@ -120,7 +120,7 @@ static int panel_simple_get_fixed_modes(struct panel_simple *panel) mode->type |= DRM_MODE_TYPE_DRIVER; - if (panel->desc->num_modes == 1) + if (panel->desc->num_timings == 1) mode->type |= DRM_MODE_TYPE_PREFERRED; drm_mode_probed_add(connector, mode); diff --git a/drivers/gpu/drm/radeon/radeon_cursor.c b/drivers/gpu/drm/radeon/radeon_cursor.c index 2a10e24b34b10b..fb16070b266e3f 100644 --- a/drivers/gpu/drm/radeon/radeon_cursor.c +++ b/drivers/gpu/drm/radeon/radeon_cursor.c @@ -90,6 +90,9 @@ static void radeon_show_cursor(struct drm_crtc *crtc) struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc); struct radeon_device *rdev = crtc->dev->dev_private; + if (radeon_crtc->cursor_out_of_bounds) + return; + if (ASIC_IS_DCE4(rdev)) { WREG32(EVERGREEN_CUR_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset, upper_32_bits(radeon_crtc->cursor_addr)); @@ -143,21 +146,25 @@ static int radeon_cursor_move_locked(struct drm_crtc *crtc, int x, int y) int xorigin = 0, yorigin = 0; int w = radeon_crtc->cursor_width; + radeon_crtc->cursor_x = x; + radeon_crtc->cursor_y = y; + if (ASIC_IS_AVIVO(rdev)) { /* avivo cursor are offset into the total surface */ x += crtc->x; y += crtc->y; } - DRM_DEBUG("x %d y %d c->x %d c->y %d\n", x, y, crtc->x, crtc->y); - if (x < 0) { + if (x < 0) xorigin = min(-x, radeon_crtc->max_cursor_width - 1); - x = 0; - } - if (y < 0) { + if (y < 0) yorigin = min(-y, radeon_crtc->max_cursor_height - 1); - y = 0; + + if (!ASIC_IS_AVIVO(rdev)) { + x += crtc->x; + y += crtc->y; } + DRM_DEBUG("x %d y %d c->x %d c->y %d\n", x, y, crtc->x, crtc->y); /* fixed on DCE6 and newer */ if (ASIC_IS_AVIVO(rdev) && !ASIC_IS_DCE6(rdev)) { @@ -180,27 +187,31 @@ static int radeon_cursor_move_locked(struct drm_crtc *crtc, int x, int y) if (i > 1) { int cursor_end, frame_end; - cursor_end = x - xorigin + w; + cursor_end = x + w; frame_end = crtc->x + crtc->mode.crtc_hdisplay; if (cursor_end >= frame_end) { w = w - (cursor_end - frame_end); if (!(frame_end & 0x7f)) w--; - } else { - if (!(cursor_end & 0x7f)) - w--; + } else if (cursor_end <= 0) { + goto out_of_bounds; + } else if (!(cursor_end & 0x7f)) { + w--; } if (w <= 0) { - w = 1; - cursor_end = x - xorigin + w; - if (!(cursor_end & 0x7f)) { - x--; - WARN_ON_ONCE(x < 0); - } + goto out_of_bounds; } } } + if (x <= (crtc->x - w) || y <= (crtc->y - radeon_crtc->cursor_height) || + x >= (crtc->x + crtc->mode.crtc_hdisplay) || + y >= (crtc->y + crtc->mode.crtc_vdisplay)) + goto out_of_bounds; + + x += xorigin; + y += yorigin; + if (ASIC_IS_DCE4(rdev)) { WREG32(EVERGREEN_CUR_POSITION + radeon_crtc->crtc_offset, (x << 16) | y); WREG32(EVERGREEN_CUR_HOT_SPOT + radeon_crtc->crtc_offset, (xorigin << 16) | yorigin); @@ -212,6 +223,9 @@ static int radeon_cursor_move_locked(struct drm_crtc *crtc, int x, int y) WREG32(AVIVO_D1CUR_SIZE + radeon_crtc->crtc_offset, ((w - 1) << 16) | (radeon_crtc->cursor_height - 1)); } else { + x -= crtc->x; + y -= crtc->y; + if (crtc->mode.flags & DRM_MODE_FLAG_DBLSCAN) y *= 2; @@ -229,10 +243,20 @@ static int radeon_cursor_move_locked(struct drm_crtc *crtc, int x, int y) yorigin * 256); } - radeon_crtc->cursor_x = x; - radeon_crtc->cursor_y = y; + if (radeon_crtc->cursor_out_of_bounds) { + radeon_crtc->cursor_out_of_bounds = false; + if (radeon_crtc->cursor_bo) + radeon_show_cursor(crtc); + } return 0; + + out_of_bounds: + if (!radeon_crtc->cursor_out_of_bounds) { + radeon_hide_cursor(crtc); + radeon_crtc->cursor_out_of_bounds = true; + } + return 0; } int radeon_crtc_cursor_move(struct drm_crtc *crtc, @@ -297,22 +321,23 @@ int radeon_crtc_cursor_set2(struct drm_crtc *crtc, return ret; } - radeon_crtc->cursor_width = width; - radeon_crtc->cursor_height = height; - radeon_lock_cursor(crtc, true); - if (hot_x != radeon_crtc->cursor_hot_x || + if (width != radeon_crtc->cursor_width || + height != radeon_crtc->cursor_height || + hot_x != radeon_crtc->cursor_hot_x || hot_y != radeon_crtc->cursor_hot_y) { int x, y; x = radeon_crtc->cursor_x + radeon_crtc->cursor_hot_x - hot_x; y = radeon_crtc->cursor_y + radeon_crtc->cursor_hot_y - hot_y; - radeon_cursor_move_locked(crtc, x, y); - + radeon_crtc->cursor_width = width; + radeon_crtc->cursor_height = height; radeon_crtc->cursor_hot_x = hot_x; radeon_crtc->cursor_hot_y = hot_y; + + radeon_cursor_move_locked(crtc, x, y); } radeon_show_cursor(crtc); diff --git a/drivers/gpu/drm/radeon/radeon_drv.c b/drivers/gpu/drm/radeon/radeon_drv.c index 00ea0002b539b9..e0c143b865f39c 100644 --- a/drivers/gpu/drm/radeon/radeon_drv.c +++ b/drivers/gpu/drm/radeon/radeon_drv.c @@ -366,11 +366,10 @@ static void radeon_pci_shutdown(struct pci_dev *pdev) { /* if we are running in a VM, make sure the device - * torn down properly on reboot/shutdown. - * unfortunately we can't detect certain - * hypervisors so just do this all the time. + * torn down properly on reboot/shutdown */ - radeon_pci_remove(pdev); + if (radeon_device_is_virtual()) + radeon_pci_remove(pdev); } static int radeon_pmops_suspend(struct device *dev) diff --git a/drivers/gpu/drm/radeon/radeon_mode.h b/drivers/gpu/drm/radeon/radeon_mode.h index bb75201a24ba92..f1da484864a9c5 100644 --- a/drivers/gpu/drm/radeon/radeon_mode.h +++ b/drivers/gpu/drm/radeon/radeon_mode.h @@ -330,6 +330,7 @@ struct radeon_crtc { u16 lut_r[256], lut_g[256], lut_b[256]; bool enabled; bool can_tile; + bool cursor_out_of_bounds; uint32_t crtc_offset; struct drm_gem_object *cursor_bo; uint64_t cursor_addr; diff --git a/drivers/gpu/drm/radeon/si.c b/drivers/gpu/drm/radeon/si.c index e402be8821c452..877af4a5ef685e 100644 --- a/drivers/gpu/drm/radeon/si.c +++ b/drivers/gpu/drm/radeon/si.c @@ -50,7 +50,6 @@ MODULE_FIRMWARE("radeon/tahiti_ce.bin"); MODULE_FIRMWARE("radeon/tahiti_mc.bin"); MODULE_FIRMWARE("radeon/tahiti_rlc.bin"); MODULE_FIRMWARE("radeon/tahiti_smc.bin"); -MODULE_FIRMWARE("radeon/tahiti_k_smc.bin"); MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin"); MODULE_FIRMWARE("radeon/PITCAIRN_me.bin"); @@ -1657,9 +1656,6 @@ static int si_init_microcode(struct radeon_device *rdev) switch (rdev->family) { case CHIP_TAHITI: chip_name = "TAHITI"; - /* XXX: figure out which Tahitis need the new ucode */ - if (0) - new_smc = true; new_chip_name = "tahiti"; pfp_req_size = SI_PFP_UCODE_SIZE * 4; me_req_size = SI_PM4_UCODE_SIZE * 4; @@ -1671,12 +1667,9 @@ static int si_init_microcode(struct radeon_device *rdev) break; case CHIP_PITCAIRN: chip_name = "PITCAIRN"; - if ((rdev->pdev->revision == 0x81) || - (rdev->pdev->device == 0x6810) || - (rdev->pdev->device == 0x6811) || - (rdev->pdev->device == 0x6816) || - (rdev->pdev->device == 0x6817) || - (rdev->pdev->device == 0x6806)) + if ((rdev->pdev->revision == 0x81) && + ((rdev->pdev->device == 0x6810) || + (rdev->pdev->device == 0x6811))) new_smc = true; new_chip_name = "pitcairn"; pfp_req_size = SI_PFP_UCODE_SIZE * 4; @@ -1689,15 +1682,15 @@ static int si_init_microcode(struct radeon_device *rdev) break; case CHIP_VERDE: chip_name = "VERDE"; - if ((rdev->pdev->revision == 0x81) || - (rdev->pdev->revision == 0x83) || - (rdev->pdev->revision == 0x87) || - (rdev->pdev->device == 0x6820) || - (rdev->pdev->device == 0x6821) || - (rdev->pdev->device == 0x6822) || - (rdev->pdev->device == 0x6823) || - (rdev->pdev->device == 0x682A) || - (rdev->pdev->device == 0x682B)) + if (((rdev->pdev->device == 0x6820) && + ((rdev->pdev->revision == 0x81) || + (rdev->pdev->revision == 0x83))) || + ((rdev->pdev->device == 0x6821) && + ((rdev->pdev->revision == 0x83) || + (rdev->pdev->revision == 0x87))) || + ((rdev->pdev->revision == 0x87) && + ((rdev->pdev->device == 0x6823) || + (rdev->pdev->device == 0x682b)))) new_smc = true; new_chip_name = "verde"; pfp_req_size = SI_PFP_UCODE_SIZE * 4; @@ -1710,12 +1703,13 @@ static int si_init_microcode(struct radeon_device *rdev) break; case CHIP_OLAND: chip_name = "OLAND"; - if ((rdev->pdev->revision == 0xC7) || - (rdev->pdev->revision == 0x80) || - (rdev->pdev->revision == 0x81) || - (rdev->pdev->revision == 0x83) || - (rdev->pdev->device == 0x6604) || - (rdev->pdev->device == 0x6605)) + if (((rdev->pdev->revision == 0x81) && + ((rdev->pdev->device == 0x6600) || + (rdev->pdev->device == 0x6604) || + (rdev->pdev->device == 0x6605) || + (rdev->pdev->device == 0x6610))) || + ((rdev->pdev->revision == 0x83) && + (rdev->pdev->device == 0x6610))) new_smc = true; new_chip_name = "oland"; pfp_req_size = SI_PFP_UCODE_SIZE * 4; @@ -1727,12 +1721,15 @@ static int si_init_microcode(struct radeon_device *rdev) break; case CHIP_HAINAN: chip_name = "HAINAN"; - if ((rdev->pdev->revision == 0x81) || - (rdev->pdev->revision == 0x83) || - (rdev->pdev->revision == 0xC3) || - (rdev->pdev->device == 0x6664) || - (rdev->pdev->device == 0x6665) || - (rdev->pdev->device == 0x6667)) + if (((rdev->pdev->revision == 0x81) && + (rdev->pdev->device == 0x6660)) || + ((rdev->pdev->revision == 0x83) && + ((rdev->pdev->device == 0x6660) || + (rdev->pdev->device == 0x6663) || + (rdev->pdev->device == 0x6665) || + (rdev->pdev->device == 0x6667))) || + ((rdev->pdev->revision == 0xc3) && + (rdev->pdev->device == 0x6665))) new_smc = true; new_chip_name = "hainan"; pfp_req_size = SI_PFP_UCODE_SIZE * 4; diff --git a/drivers/gpu/drm/radeon/si_dpm.c b/drivers/gpu/drm/radeon/si_dpm.c index c49934527a8785..13ba73fd9b6884 100644 --- a/drivers/gpu/drm/radeon/si_dpm.c +++ b/drivers/gpu/drm/radeon/si_dpm.c @@ -3008,24 +3008,12 @@ static void si_apply_state_adjust_rules(struct radeon_device *rdev, (rdev->pdev->device == 0x6817) || (rdev->pdev->device == 0x6806)) max_mclk = 120000; - } else if (rdev->family == CHIP_VERDE) { - if ((rdev->pdev->revision == 0x81) || - (rdev->pdev->revision == 0x83) || - (rdev->pdev->revision == 0x87) || - (rdev->pdev->device == 0x6820) || - (rdev->pdev->device == 0x6821) || - (rdev->pdev->device == 0x6822) || - (rdev->pdev->device == 0x6823) || - (rdev->pdev->device == 0x682A) || - (rdev->pdev->device == 0x682B)) { - max_sclk = 75000; - max_mclk = 80000; - } } else if (rdev->family == CHIP_OLAND) { if ((rdev->pdev->revision == 0xC7) || (rdev->pdev->revision == 0x80) || (rdev->pdev->revision == 0x81) || (rdev->pdev->revision == 0x83) || + (rdev->pdev->revision == 0x87) || (rdev->pdev->device == 0x6604) || (rdev->pdev->device == 0x6605)) { max_sclk = 75000; diff --git a/drivers/gpu/drm/savage/savage_state.c b/drivers/gpu/drm/savage/savage_state.c index 3dc0d8ff95ec6d..2db89bed52e806 100644 --- a/drivers/gpu/drm/savage/savage_state.c +++ b/drivers/gpu/drm/savage/savage_state.c @@ -1004,6 +1004,7 @@ int savage_bci_cmdbuf(struct drm_device *dev, void *data, struct drm_file *file_ kvb_addr = memdup_user(cmdbuf->vb_addr, cmdbuf->vb_size); if (IS_ERR(kvb_addr)) { ret = PTR_ERR(kvb_addr); + kvb_addr = NULL; goto done; } cmdbuf->vb_addr = kvb_addr; diff --git a/drivers/gpu/drm/tegra/dpaux.c b/drivers/gpu/drm/tegra/dpaux.c index 059f409556d55d..2fde44c3a1b30f 100644 --- a/drivers/gpu/drm/tegra/dpaux.c +++ b/drivers/gpu/drm/tegra/dpaux.c @@ -539,9 +539,9 @@ static int tegra_dpaux_probe(struct platform_device *pdev) dpaux->desc.owner = THIS_MODULE; dpaux->pinctrl = devm_pinctrl_register(&pdev->dev, &dpaux->desc, dpaux); - if (!dpaux->pinctrl) { + if (IS_ERR(dpaux->pinctrl)) { dev_err(&pdev->dev, "failed to register pincontrol\n"); - return -ENODEV; + return PTR_ERR(dpaux->pinctrl); } #endif /* enable and clear all interrupts */ diff --git a/drivers/gpu/drm/vc4/vc4_crtc.c b/drivers/gpu/drm/vc4/vc4_crtc.c index 7f08d681a74b4e..d544ff9b0d4609 100644 --- a/drivers/gpu/drm/vc4/vc4_crtc.c +++ b/drivers/gpu/drm/vc4/vc4_crtc.c @@ -832,7 +832,7 @@ static void vc4_crtc_destroy_state(struct drm_crtc *crtc, } - __drm_atomic_helper_crtc_destroy_state(state); + drm_atomic_helper_crtc_destroy_state(crtc, state); } static const struct drm_crtc_funcs vc4_crtc_funcs = { diff --git a/drivers/gpu/drm/vc4/vc4_gem.c b/drivers/gpu/drm/vc4/vc4_gem.c index 47a095f392f8a7..18e37171e9c8e2 100644 --- a/drivers/gpu/drm/vc4/vc4_gem.c +++ b/drivers/gpu/drm/vc4/vc4_gem.c @@ -544,14 +544,15 @@ vc4_cl_lookup_bos(struct drm_device *dev, handles = drm_malloc_ab(exec->bo_count, sizeof(uint32_t)); if (!handles) { + ret = -ENOMEM; DRM_ERROR("Failed to allocate incoming GEM handles\n"); goto fail; } - ret = copy_from_user(handles, - (void __user *)(uintptr_t)args->bo_handles, - exec->bo_count * sizeof(uint32_t)); - if (ret) { + if (copy_from_user(handles, + (void __user *)(uintptr_t)args->bo_handles, + exec->bo_count * sizeof(uint32_t))) { + ret = -EFAULT; DRM_ERROR("Failed to copy in GEM handles\n"); goto fail; } @@ -593,12 +594,14 @@ vc4_get_bcl(struct drm_device *dev, struct vc4_exec_info *exec) args->shader_rec_count); struct vc4_bo *bo; - if (uniforms_offset < shader_rec_offset || + if (shader_rec_offset < args->bin_cl_size || + uniforms_offset < shader_rec_offset || exec_size < uniforms_offset || args->shader_rec_count >= (UINT_MAX / sizeof(struct vc4_shader_state)) || temp_size < exec_size) { DRM_ERROR("overflow in exec arguments\n"); + ret = -EINVAL; goto fail; } diff --git a/drivers/gpu/drm/vc4/vc4_render_cl.c b/drivers/gpu/drm/vc4/vc4_render_cl.c index 08886a30975772..5cdd003605f57c 100644 --- a/drivers/gpu/drm/vc4/vc4_render_cl.c +++ b/drivers/gpu/drm/vc4/vc4_render_cl.c @@ -461,7 +461,7 @@ static int vc4_rcl_surface_setup(struct vc4_exec_info *exec, } ret = vc4_full_res_bounds_check(exec, *obj, surf); - if (!ret) + if (ret) return ret; return 0; diff --git a/drivers/hid/hid-corsair.c b/drivers/hid/hid-corsair.c index 717704e9ae07bd..c0303f61c26a94 100644 --- a/drivers/hid/hid-corsair.c +++ b/drivers/hid/hid-corsair.c @@ -148,26 +148,36 @@ static enum led_brightness k90_backlight_get(struct led_classdev *led_cdev) struct usb_interface *usbif = to_usb_interface(dev->parent); struct usb_device *usbdev = interface_to_usbdev(usbif); int brightness; - char data[8]; + char *data; + + data = kmalloc(8, GFP_KERNEL); + if (!data) + return -ENOMEM; ret = usb_control_msg(usbdev, usb_rcvctrlpipe(usbdev, 0), K90_REQUEST_STATUS, USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_DEVICE, 0, 0, data, 8, USB_CTRL_SET_TIMEOUT); - if (ret < 0) { + if (ret < 5) { dev_warn(dev, "Failed to get K90 initial state (error %d).\n", ret); - return -EIO; + ret = -EIO; + goto out; } brightness = data[4]; if (brightness < 0 || brightness > 3) { dev_warn(dev, "Read invalid backlight brightness: %02hhx.\n", data[4]); - return -EIO; + ret = -EIO; + goto out; } - return brightness; + ret = brightness; +out: + kfree(data); + + return ret; } static enum led_brightness k90_record_led_get(struct led_classdev *led_cdev) @@ -253,17 +263,22 @@ static ssize_t k90_show_macro_mode(struct device *dev, struct usb_interface *usbif = to_usb_interface(dev->parent); struct usb_device *usbdev = interface_to_usbdev(usbif); const char *macro_mode; - char data[8]; + char *data; + + data = kmalloc(2, GFP_KERNEL); + if (!data) + return -ENOMEM; ret = usb_control_msg(usbdev, usb_rcvctrlpipe(usbdev, 0), K90_REQUEST_GET_MODE, USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_DEVICE, 0, 0, data, 2, USB_CTRL_SET_TIMEOUT); - if (ret < 0) { + if (ret < 1) { dev_warn(dev, "Failed to get K90 initial mode (error %d).\n", ret); - return -EIO; + ret = -EIO; + goto out; } switch (data[0]) { @@ -277,10 +292,15 @@ static ssize_t k90_show_macro_mode(struct device *dev, default: dev_warn(dev, "K90 in unknown mode: %02hhx.\n", data[0]); - return -EIO; + ret = -EIO; + goto out; } - return snprintf(buf, PAGE_SIZE, "%s\n", macro_mode); + ret = snprintf(buf, PAGE_SIZE, "%s\n", macro_mode); +out: + kfree(data); + + return ret; } static ssize_t k90_store_macro_mode(struct device *dev, @@ -320,26 +340,36 @@ static ssize_t k90_show_current_profile(struct device *dev, struct usb_interface *usbif = to_usb_interface(dev->parent); struct usb_device *usbdev = interface_to_usbdev(usbif); int current_profile; - char data[8]; + char *data; + + data = kmalloc(8, GFP_KERNEL); + if (!data) + return -ENOMEM; ret = usb_control_msg(usbdev, usb_rcvctrlpipe(usbdev, 0), K90_REQUEST_STATUS, USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_DEVICE, 0, 0, data, 8, USB_CTRL_SET_TIMEOUT); - if (ret < 0) { + if (ret < 8) { dev_warn(dev, "Failed to get K90 initial state (error %d).\n", ret); - return -EIO; + ret = -EIO; + goto out; } current_profile = data[7]; if (current_profile < 1 || current_profile > 3) { dev_warn(dev, "Read invalid current profile: %02hhx.\n", data[7]); - return -EIO; + ret = -EIO; + goto out; } - return snprintf(buf, PAGE_SIZE, "%d\n", current_profile); + ret = snprintf(buf, PAGE_SIZE, "%d\n", current_profile); +out: + kfree(data); + + return ret; } static ssize_t k90_store_current_profile(struct device *dev, diff --git a/drivers/hid/hid-cp2112.c b/drivers/hid/hid-cp2112.c index 60d30203a5faf9..e06c1344c9138e 100644 --- a/drivers/hid/hid-cp2112.c +++ b/drivers/hid/hid-cp2112.c @@ -167,7 +167,7 @@ struct cp2112_device { atomic_t xfer_avail; struct gpio_chip gc; u8 *in_out_buffer; - spinlock_t lock; + struct mutex lock; }; static int gpio_push_pull = 0xFF; @@ -179,10 +179,9 @@ static int cp2112_gpio_direction_input(struct gpio_chip *chip, unsigned offset) struct cp2112_device *dev = gpiochip_get_data(chip); struct hid_device *hdev = dev->hdev; u8 *buf = dev->in_out_buffer; - unsigned long flags; int ret; - spin_lock_irqsave(&dev->lock, flags); + mutex_lock(&dev->lock); ret = hid_hw_raw_request(hdev, CP2112_GPIO_CONFIG, buf, CP2112_GPIO_CONFIG_LENGTH, HID_FEATURE_REPORT, @@ -206,8 +205,8 @@ static int cp2112_gpio_direction_input(struct gpio_chip *chip, unsigned offset) ret = 0; exit: - spin_unlock_irqrestore(&dev->lock, flags); - return ret <= 0 ? ret : -EIO; + mutex_unlock(&dev->lock); + return ret < 0 ? ret : -EIO; } static void cp2112_gpio_set(struct gpio_chip *chip, unsigned offset, int value) @@ -215,10 +214,9 @@ static void cp2112_gpio_set(struct gpio_chip *chip, unsigned offset, int value) struct cp2112_device *dev = gpiochip_get_data(chip); struct hid_device *hdev = dev->hdev; u8 *buf = dev->in_out_buffer; - unsigned long flags; int ret; - spin_lock_irqsave(&dev->lock, flags); + mutex_lock(&dev->lock); buf[0] = CP2112_GPIO_SET; buf[1] = value ? 0xff : 0; @@ -230,7 +228,7 @@ static void cp2112_gpio_set(struct gpio_chip *chip, unsigned offset, int value) if (ret < 0) hid_err(hdev, "error setting GPIO values: %d\n", ret); - spin_unlock_irqrestore(&dev->lock, flags); + mutex_unlock(&dev->lock); } static int cp2112_gpio_get(struct gpio_chip *chip, unsigned offset) @@ -238,10 +236,9 @@ static int cp2112_gpio_get(struct gpio_chip *chip, unsigned offset) struct cp2112_device *dev = gpiochip_get_data(chip); struct hid_device *hdev = dev->hdev; u8 *buf = dev->in_out_buffer; - unsigned long flags; int ret; - spin_lock_irqsave(&dev->lock, flags); + mutex_lock(&dev->lock); ret = hid_hw_raw_request(hdev, CP2112_GPIO_GET, buf, CP2112_GPIO_GET_LENGTH, HID_FEATURE_REPORT, @@ -255,7 +252,7 @@ static int cp2112_gpio_get(struct gpio_chip *chip, unsigned offset) ret = (buf[1] >> offset) & 1; exit: - spin_unlock_irqrestore(&dev->lock, flags); + mutex_unlock(&dev->lock); return ret; } @@ -266,10 +263,9 @@ static int cp2112_gpio_direction_output(struct gpio_chip *chip, struct cp2112_device *dev = gpiochip_get_data(chip); struct hid_device *hdev = dev->hdev; u8 *buf = dev->in_out_buffer; - unsigned long flags; int ret; - spin_lock_irqsave(&dev->lock, flags); + mutex_lock(&dev->lock); ret = hid_hw_raw_request(hdev, CP2112_GPIO_CONFIG, buf, CP2112_GPIO_CONFIG_LENGTH, HID_FEATURE_REPORT, @@ -290,7 +286,7 @@ static int cp2112_gpio_direction_output(struct gpio_chip *chip, goto fail; } - spin_unlock_irqrestore(&dev->lock, flags); + mutex_unlock(&dev->lock); /* * Set gpio value when output direction is already set, @@ -301,7 +297,7 @@ static int cp2112_gpio_direction_output(struct gpio_chip *chip, return 0; fail: - spin_unlock_irqrestore(&dev->lock, flags); + mutex_unlock(&dev->lock); return ret < 0 ? ret : -EIO; } @@ -1057,7 +1053,7 @@ static int cp2112_probe(struct hid_device *hdev, const struct hid_device_id *id) if (!dev->in_out_buffer) return -ENOMEM; - spin_lock_init(&dev->lock); + mutex_init(&dev->lock); ret = hid_parse(hdev); if (ret) { diff --git a/drivers/hid/hid-cypress.c b/drivers/hid/hid-cypress.c index 1b764d1745f3da..1689568b597d4e 100644 --- a/drivers/hid/hid-cypress.c +++ b/drivers/hid/hid-cypress.c @@ -39,6 +39,9 @@ static __u8 *cp_report_fixup(struct hid_device *hdev, __u8 *rdesc, if (!(quirks & CP_RDESC_SWAPPED_MIN_MAX)) return rdesc; + if (*rsize < 4) + return rdesc; + for (i = 0; i < *rsize - 4; i++) if (rdesc[i] == 0x29 && rdesc[i + 2] == 0x19) { rdesc[i] = 0x19; diff --git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h index 575aa65436d182..9845189fae925a 100644 --- a/drivers/hid/hid-ids.h +++ b/drivers/hid/hid-ids.h @@ -76,6 +76,9 @@ #define USB_VENDOR_ID_ALPS_JP 0x044E #define HID_DEVICE_ID_ALPS_U1_DUAL 0x120B +#define USB_VENDOR_ID_AMI 0x046b +#define USB_DEVICE_ID_AMI_VIRT_KEYBOARD_AND_MOUSE 0xff10 + #define USB_VENDOR_ID_ANTON 0x1130 #define USB_DEVICE_ID_ANTON_TOUCH_PAD 0x3101 diff --git a/drivers/hid/hid-lg.c b/drivers/hid/hid-lg.c index c5c5fbe9d60577..52026dc94d5c4b 100644 --- a/drivers/hid/hid-lg.c +++ b/drivers/hid/hid-lg.c @@ -872,7 +872,7 @@ static const struct hid_device_id lg_devices[] = { { HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_WINGMAN_FFG), .driver_data = LG_NOGET | LG_FF4 }, { HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_RUMBLEPAD2), - .driver_data = LG_FF2 }, + .driver_data = LG_NOGET | LG_FF2 }, { HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_FLIGHT_SYSTEM_G940), .driver_data = LG_FF3 }, { HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_SPACENAVIGATOR), diff --git a/drivers/hid/hid-sensor-hub.c b/drivers/hid/hid-sensor-hub.c index 60875625cbdff4..8f6c35370f661d 100644 --- a/drivers/hid/hid-sensor-hub.c +++ b/drivers/hid/hid-sensor-hub.c @@ -212,7 +212,6 @@ int sensor_hub_set_feature(struct hid_sensor_hub_device *hsdev, u32 report_id, __s32 value; int ret = 0; - memset(buffer, 0, buffer_size); mutex_lock(&data->mutex); report = sensor_hub_report(report_id, hsdev->hdev, HID_FEATURE_REPORT); if (!report || (field_index >= report->maxfield)) { @@ -256,6 +255,8 @@ int sensor_hub_get_feature(struct hid_sensor_hub_device *hsdev, u32 report_id, int buffer_index = 0; int i; + memset(buffer, 0, buffer_size); + mutex_lock(&data->mutex); report = sensor_hub_report(report_id, hsdev->hdev, HID_FEATURE_REPORT); if (!report || (field_index >= report->maxfield) || diff --git a/drivers/hid/usbhid/hid-quirks.c b/drivers/hid/usbhid/hid-quirks.c index e6cfd323babc62..cde060fefa9116 100644 --- a/drivers/hid/usbhid/hid-quirks.c +++ b/drivers/hid/usbhid/hid-quirks.c @@ -57,6 +57,7 @@ static const struct hid_blacklist { { USB_VENDOR_ID_AIREN, USB_DEVICE_ID_AIREN_SLIMPLUS, HID_QUIRK_NOGET }, { USB_VENDOR_ID_AKAI, USB_DEVICE_ID_AKAI_MPKMINI2, HID_QUIRK_NO_INIT_REPORTS }, { USB_VENDOR_ID_AKAI_09E8, USB_DEVICE_ID_AKAI_09E8_MIDIMIX, HID_QUIRK_NO_INIT_REPORTS }, + { USB_VENDOR_ID_AMI, USB_DEVICE_ID_AMI_VIRT_KEYBOARD_AND_MOUSE, HID_QUIRK_ALWAYS_POLL }, { USB_VENDOR_ID_ATEN, USB_DEVICE_ID_ATEN_UC100KM, HID_QUIRK_NOGET }, { USB_VENDOR_ID_ATEN, USB_DEVICE_ID_ATEN_CS124U, HID_QUIRK_NOGET }, { USB_VENDOR_ID_ATEN, USB_DEVICE_ID_ATEN_2PORTKVM, HID_QUIRK_NOGET }, diff --git a/drivers/hid/wacom_wac.c b/drivers/hid/wacom_wac.c index 1cb79925730d93..623be90704abfb 100644 --- a/drivers/hid/wacom_wac.c +++ b/drivers/hid/wacom_wac.c @@ -164,19 +164,21 @@ static int wacom_pl_irq(struct wacom_wac *wacom) wacom->id[0] = STYLUS_DEVICE_ID; } - pressure = (signed char)((data[7] << 1) | ((data[4] >> 2) & 1)); - if (features->pressure_max > 255) - pressure = (pressure << 1) | ((data[4] >> 6) & 1); - pressure += (features->pressure_max + 1) / 2; - - input_report_abs(input, ABS_X, data[3] | (data[2] << 7) | ((data[1] & 0x03) << 14)); - input_report_abs(input, ABS_Y, data[6] | (data[5] << 7) | ((data[4] & 0x03) << 14)); - input_report_abs(input, ABS_PRESSURE, pressure); - - input_report_key(input, BTN_TOUCH, data[4] & 0x08); - input_report_key(input, BTN_STYLUS, data[4] & 0x10); - /* Only allow the stylus2 button to be reported for the pen tool. */ - input_report_key(input, BTN_STYLUS2, (wacom->tool[0] == BTN_TOOL_PEN) && (data[4] & 0x20)); + if (prox) { + pressure = (signed char)((data[7] << 1) | ((data[4] >> 2) & 1)); + if (features->pressure_max > 255) + pressure = (pressure << 1) | ((data[4] >> 6) & 1); + pressure += (features->pressure_max + 1) / 2; + + input_report_abs(input, ABS_X, data[3] | (data[2] << 7) | ((data[1] & 0x03) << 14)); + input_report_abs(input, ABS_Y, data[6] | (data[5] << 7) | ((data[4] & 0x03) << 14)); + input_report_abs(input, ABS_PRESSURE, pressure); + + input_report_key(input, BTN_TOUCH, data[4] & 0x08); + input_report_key(input, BTN_STYLUS, data[4] & 0x10); + /* Only allow the stylus2 button to be reported for the pen tool. */ + input_report_key(input, BTN_STYLUS2, (wacom->tool[0] == BTN_TOOL_PEN) && (data[4] & 0x20)); + } if (!prox) wacom->id[0] = 0; diff --git a/drivers/hv/channel.c b/drivers/hv/channel.c index 16f91c8490fe01..5fb4c6d9209b75 100644 --- a/drivers/hv/channel.c +++ b/drivers/hv/channel.c @@ -39,7 +39,7 @@ * vmbus_setevent- Trigger an event notification on the specified * channel. */ -static void vmbus_setevent(struct vmbus_channel *channel) +void vmbus_setevent(struct vmbus_channel *channel) { struct hv_monitor_page *monitorpage; @@ -65,6 +65,7 @@ static void vmbus_setevent(struct vmbus_channel *channel) vmbus_set_event(channel); } } +EXPORT_SYMBOL_GPL(vmbus_setevent); /* * vmbus_open - Open the specified channel. @@ -635,8 +636,6 @@ int vmbus_sendpacket_ctl(struct vmbus_channel *channel, void *buffer, u32 packetlen_aligned = ALIGN(packetlen, sizeof(u64)); struct kvec bufferlist[3]; u64 aligned_data = 0; - int ret; - bool signal = false; bool lock = channel->acquire_ring_lock; int num_vecs = ((bufferlen != 0) ? 3 : 1); @@ -656,33 +655,9 @@ int vmbus_sendpacket_ctl(struct vmbus_channel *channel, void *buffer, bufferlist[2].iov_base = &aligned_data; bufferlist[2].iov_len = (packetlen_aligned - packetlen); - ret = hv_ringbuffer_write(&channel->outbound, bufferlist, num_vecs, - &signal, lock, channel->signal_policy); - - /* - * Signalling the host is conditional on many factors: - * 1. The ring state changed from being empty to non-empty. - * This is tracked by the variable "signal". - * 2. The variable kick_q tracks if more data will be placed - * on the ring. We will not signal if more data is - * to be placed. - * - * Based on the channel signal state, we will decide - * which signaling policy will be applied. - * - * If we cannot write to the ring-buffer; signal the host - * even if we may not have written anything. This is a rare - * enough condition that it should not matter. - * NOTE: in this case, the hvsock channel is an exception, because - * it looks the host side's hvsock implementation has a throttling - * mechanism which can hurt the performance otherwise. - */ - - if (((ret == 0) && kick_q && signal) || - (ret && !is_hvsock_channel(channel))) - vmbus_setevent(channel); + return hv_ringbuffer_write(channel, bufferlist, num_vecs, + lock, kick_q); - return ret; } EXPORT_SYMBOL(vmbus_sendpacket_ctl); @@ -723,7 +698,6 @@ int vmbus_sendpacket_pagebuffer_ctl(struct vmbus_channel *channel, u32 flags, bool kick_q) { - int ret; int i; struct vmbus_channel_packet_page_buffer desc; u32 descsize; @@ -731,7 +705,6 @@ int vmbus_sendpacket_pagebuffer_ctl(struct vmbus_channel *channel, u32 packetlen_aligned; struct kvec bufferlist[3]; u64 aligned_data = 0; - bool signal = false; bool lock = channel->acquire_ring_lock; if (pagecount > MAX_PAGE_BUFFER_COUNT) @@ -769,29 +742,8 @@ int vmbus_sendpacket_pagebuffer_ctl(struct vmbus_channel *channel, bufferlist[2].iov_base = &aligned_data; bufferlist[2].iov_len = (packetlen_aligned - packetlen); - ret = hv_ringbuffer_write(&channel->outbound, bufferlist, 3, - &signal, lock, channel->signal_policy); - - /* - * Signalling the host is conditional on many factors: - * 1. The ring state changed from being empty to non-empty. - * This is tracked by the variable "signal". - * 2. The variable kick_q tracks if more data will be placed - * on the ring. We will not signal if more data is - * to be placed. - * - * Based on the channel signal state, we will decide - * which signaling policy will be applied. - * - * If we cannot write to the ring-buffer; signal the host - * even if we may not have written anything. This is a rare - * enough condition that it should not matter. - */ - - if (((ret == 0) && kick_q && signal) || (ret)) - vmbus_setevent(channel); - - return ret; + return hv_ringbuffer_write(channel, bufferlist, 3, + lock, kick_q); } EXPORT_SYMBOL_GPL(vmbus_sendpacket_pagebuffer_ctl); @@ -822,12 +774,10 @@ int vmbus_sendpacket_mpb_desc(struct vmbus_channel *channel, u32 desc_size, void *buffer, u32 bufferlen, u64 requestid) { - int ret; u32 packetlen; u32 packetlen_aligned; struct kvec bufferlist[3]; u64 aligned_data = 0; - bool signal = false; bool lock = channel->acquire_ring_lock; packetlen = desc_size + bufferlen; @@ -848,13 +798,8 @@ int vmbus_sendpacket_mpb_desc(struct vmbus_channel *channel, bufferlist[2].iov_base = &aligned_data; bufferlist[2].iov_len = (packetlen_aligned - packetlen); - ret = hv_ringbuffer_write(&channel->outbound, bufferlist, 3, - &signal, lock, channel->signal_policy); - - if (ret == 0 && signal) - vmbus_setevent(channel); - - return ret; + return hv_ringbuffer_write(channel, bufferlist, 3, + lock, true); } EXPORT_SYMBOL_GPL(vmbus_sendpacket_mpb_desc); @@ -866,14 +811,12 @@ int vmbus_sendpacket_multipagebuffer(struct vmbus_channel *channel, struct hv_multipage_buffer *multi_pagebuffer, void *buffer, u32 bufferlen, u64 requestid) { - int ret; struct vmbus_channel_packet_multipage_buffer desc; u32 descsize; u32 packetlen; u32 packetlen_aligned; struct kvec bufferlist[3]; u64 aligned_data = 0; - bool signal = false; bool lock = channel->acquire_ring_lock; u32 pfncount = NUM_PAGES_SPANNED(multi_pagebuffer->offset, multi_pagebuffer->len); @@ -913,13 +856,8 @@ int vmbus_sendpacket_multipagebuffer(struct vmbus_channel *channel, bufferlist[2].iov_base = &aligned_data; bufferlist[2].iov_len = (packetlen_aligned - packetlen); - ret = hv_ringbuffer_write(&channel->outbound, bufferlist, 3, - &signal, lock, channel->signal_policy); - - if (ret == 0 && signal) - vmbus_setevent(channel); - - return ret; + return hv_ringbuffer_write(channel, bufferlist, 3, + lock, true); } EXPORT_SYMBOL_GPL(vmbus_sendpacket_multipagebuffer); @@ -941,16 +879,9 @@ __vmbus_recvpacket(struct vmbus_channel *channel, void *buffer, u32 bufferlen, u32 *buffer_actual_len, u64 *requestid, bool raw) { - int ret; - bool signal = false; + return hv_ringbuffer_read(channel, buffer, bufferlen, + buffer_actual_len, requestid, raw); - ret = hv_ringbuffer_read(&channel->inbound, buffer, bufferlen, - buffer_actual_len, requestid, &signal, raw); - - if (signal) - vmbus_setevent(channel); - - return ret; } int vmbus_recvpacket(struct vmbus_channel *channel, void *buffer, diff --git a/drivers/hv/channel_mgmt.c b/drivers/hv/channel_mgmt.c index 96a85cd39580a9..caf34184246421 100644 --- a/drivers/hv/channel_mgmt.c +++ b/drivers/hv/channel_mgmt.c @@ -389,6 +389,7 @@ void vmbus_free_channels(void) { struct vmbus_channel *channel, *tmp; + mutex_lock(&vmbus_connection.channel_mutex); list_for_each_entry_safe(channel, tmp, &vmbus_connection.chn_list, listentry) { /* hv_process_channel_removal() needs this */ @@ -396,6 +397,7 @@ void vmbus_free_channels(void) vmbus_device_unregister(channel->device_obj); } + mutex_unlock(&vmbus_connection.channel_mutex); } /* @@ -447,8 +449,6 @@ static void vmbus_process_offer(struct vmbus_channel *newchannel) } dev_type = hv_get_dev_type(newchannel); - if (dev_type == HV_NIC) - set_channel_signal_state(newchannel, HV_SIGNAL_POLICY_EXPLICIT); init_vp_index(newchannel, dev_type); diff --git a/drivers/hv/hyperv_vmbus.h b/drivers/hv/hyperv_vmbus.h index a5b4442433c892..2b13f2a0a71e10 100644 --- a/drivers/hv/hyperv_vmbus.h +++ b/drivers/hv/hyperv_vmbus.h @@ -527,14 +527,14 @@ int hv_ringbuffer_init(struct hv_ring_buffer_info *ring_info, void hv_ringbuffer_cleanup(struct hv_ring_buffer_info *ring_info); -int hv_ringbuffer_write(struct hv_ring_buffer_info *ring_info, +int hv_ringbuffer_write(struct vmbus_channel *channel, struct kvec *kv_list, - u32 kv_count, bool *signal, bool lock, - enum hv_signal_policy policy); + u32 kv_count, bool lock, + bool kick_q); -int hv_ringbuffer_read(struct hv_ring_buffer_info *inring_info, +int hv_ringbuffer_read(struct vmbus_channel *channel, void *buffer, u32 buflen, u32 *buffer_actual_len, - u64 *requestid, bool *signal, bool raw); + u64 *requestid, bool raw); void hv_ringbuffer_get_debuginfo(struct hv_ring_buffer_info *ring_info, struct hv_ring_buffer_debug_info *debug_info); diff --git a/drivers/hv/ring_buffer.c b/drivers/hv/ring_buffer.c index 08043da1a61c5f..308dbda700ebda 100644 --- a/drivers/hv/ring_buffer.c +++ b/drivers/hv/ring_buffer.c @@ -66,21 +66,25 @@ u32 hv_end_read(struct hv_ring_buffer_info *rbi) * once the ring buffer is empty, it will clear the * interrupt_mask and re-check to see if new data has * arrived. + * + * KYS: Oct. 30, 2016: + * It looks like Windows hosts have logic to deal with DOS attacks that + * can be triggered if it receives interrupts when it is not expecting + * the interrupt. The host expects interrupts only when the ring + * transitions from empty to non-empty (or full to non full on the guest + * to host ring). + * So, base the signaling decision solely on the ring state until the + * host logic is fixed. */ -static bool hv_need_to_signal(u32 old_write, struct hv_ring_buffer_info *rbi, - enum hv_signal_policy policy) +static void hv_signal_on_write(u32 old_write, struct vmbus_channel *channel, + bool kick_q) { + struct hv_ring_buffer_info *rbi = &channel->outbound; + virt_mb(); if (READ_ONCE(rbi->ring_buffer->interrupt_mask)) - return false; - - /* - * When the client wants to control signaling, - * we only honour the host interrupt mask. - */ - if (policy == HV_SIGNAL_POLICY_EXPLICIT) - return true; + return; /* check interrupt_mask before read_index */ virt_rmb(); @@ -89,9 +93,9 @@ static bool hv_need_to_signal(u32 old_write, struct hv_ring_buffer_info *rbi, * ring transitions from being empty to non-empty. */ if (old_write == READ_ONCE(rbi->ring_buffer->read_index)) - return true; + vmbus_setevent(channel); - return false; + return; } /* Get the next write location for the specified ring buffer. */ @@ -280,9 +284,9 @@ void hv_ringbuffer_cleanup(struct hv_ring_buffer_info *ring_info) } /* Write to the ring buffer. */ -int hv_ringbuffer_write(struct hv_ring_buffer_info *outring_info, - struct kvec *kv_list, u32 kv_count, bool *signal, bool lock, - enum hv_signal_policy policy) +int hv_ringbuffer_write(struct vmbus_channel *channel, + struct kvec *kv_list, u32 kv_count, bool lock, + bool kick_q) { int i = 0; u32 bytes_avail_towrite; @@ -292,6 +296,7 @@ int hv_ringbuffer_write(struct hv_ring_buffer_info *outring_info, u32 old_write; u64 prev_indices = 0; unsigned long flags = 0; + struct hv_ring_buffer_info *outring_info = &channel->outbound; for (i = 0; i < kv_count; i++) totalbytes_towrite += kv_list[i].iov_len; @@ -344,13 +349,13 @@ int hv_ringbuffer_write(struct hv_ring_buffer_info *outring_info, if (lock) spin_unlock_irqrestore(&outring_info->ring_lock, flags); - *signal = hv_need_to_signal(old_write, outring_info, policy); + hv_signal_on_write(old_write, channel, kick_q); return 0; } -int hv_ringbuffer_read(struct hv_ring_buffer_info *inring_info, +int hv_ringbuffer_read(struct vmbus_channel *channel, void *buffer, u32 buflen, u32 *buffer_actual_len, - u64 *requestid, bool *signal, bool raw) + u64 *requestid, bool raw) { u32 bytes_avail_toread; u32 next_read_location = 0; @@ -359,6 +364,7 @@ int hv_ringbuffer_read(struct hv_ring_buffer_info *inring_info, u32 offset; u32 packetlen; int ret = 0; + struct hv_ring_buffer_info *inring_info = &channel->inbound; if (buflen <= 0) return -EINVAL; @@ -377,6 +383,7 @@ int hv_ringbuffer_read(struct hv_ring_buffer_info *inring_info, return ret; } + init_cached_read_index(channel); next_read_location = hv_get_next_read_location(inring_info); next_read_location = hv_copyfrom_ringbuffer(inring_info, &desc, sizeof(desc), @@ -416,7 +423,7 @@ int hv_ringbuffer_read(struct hv_ring_buffer_info *inring_info, /* Update the read index */ hv_set_next_read_location(inring_info, next_read_location); - *signal = hv_need_to_signal_on_read(inring_info); + hv_signal_on_read(channel); return ret; } diff --git a/drivers/hwmon/amc6821.c b/drivers/hwmon/amc6821.c index 12e851a5af48d0..46b4e35fd5555b 100644 --- a/drivers/hwmon/amc6821.c +++ b/drivers/hwmon/amc6821.c @@ -188,8 +188,8 @@ static struct amc6821_data *amc6821_update_device(struct device *dev) !data->valid) { for (i = 0; i < TEMP_IDX_LEN; i++) - data->temp[i] = i2c_smbus_read_byte_data(client, - temp_reg[i]); + data->temp[i] = (int8_t)i2c_smbus_read_byte_data( + client, temp_reg[i]); data->stat1 = i2c_smbus_read_byte_data(client, AMC6821_REG_STAT1); diff --git a/drivers/hwmon/ds620.c b/drivers/hwmon/ds620.c index edf550fc4eef5c..0043a4c02b85b4 100644 --- a/drivers/hwmon/ds620.c +++ b/drivers/hwmon/ds620.c @@ -166,7 +166,7 @@ static ssize_t set_temp(struct device *dev, struct device_attribute *da, if (res) return res; - val = (val * 10 / 625) * 8; + val = (clamp_val(val, -128000, 128000) * 10 / 625) * 8; mutex_lock(&data->update_lock); data->temp[attr->index] = val; diff --git a/drivers/hwmon/g762.c b/drivers/hwmon/g762.c index b96a2a9e4df7d5..628be9c95ff9db 100644 --- a/drivers/hwmon/g762.c +++ b/drivers/hwmon/g762.c @@ -193,14 +193,17 @@ static inline unsigned int rpm_from_cnt(u8 cnt, u32 clk_freq, u16 p, * Convert fan RPM value from sysfs into count value for fan controller * register (FAN_SET_CNT). */ -static inline unsigned char cnt_from_rpm(u32 rpm, u32 clk_freq, u16 p, +static inline unsigned char cnt_from_rpm(unsigned long rpm, u32 clk_freq, u16 p, u8 clk_div, u8 gear_mult) { - if (!rpm) /* to stop the fan, set cnt to 255 */ + unsigned long f1 = clk_freq * 30 * gear_mult; + unsigned long f2 = p * clk_div; + + if (!rpm) /* to stop the fan, set cnt to 255 */ return 0xff; - return clamp_val(((clk_freq * 30 * gear_mult) / (rpm * p * clk_div)), - 0, 255); + rpm = clamp_val(rpm, f1 / (255 * f2), ULONG_MAX / f2); + return DIV_ROUND_CLOSEST(f1, rpm * f2); } /* helper to grab and cache data, at most one time per second */ diff --git a/drivers/hwmon/lm90.c b/drivers/hwmon/lm90.c index 322ed927281181..841f2428e84a39 100644 --- a/drivers/hwmon/lm90.c +++ b/drivers/hwmon/lm90.c @@ -1036,7 +1036,7 @@ static const u8 lm90_temp_emerg_index[3] = { }; static const u8 lm90_min_alarm_bits[3] = { 5, 3, 11 }; -static const u8 lm90_max_alarm_bits[3] = { 0, 4, 12 }; +static const u8 lm90_max_alarm_bits[3] = { 6, 4, 12 }; static const u8 lm90_crit_alarm_bits[3] = { 0, 1, 9 }; static const u8 lm90_emergency_alarm_bits[3] = { 15, 13, 14 }; static const u8 lm90_fault_bits[3] = { 0, 2, 10 }; diff --git a/drivers/hwmon/nct7802.c b/drivers/hwmon/nct7802.c index 3ce33d244cc0f7..12b94b094c0d15 100644 --- a/drivers/hwmon/nct7802.c +++ b/drivers/hwmon/nct7802.c @@ -259,13 +259,15 @@ static int nct7802_read_fan_min(struct nct7802_data *data, u8 reg_fan_low, ret = 0; else if (ret) ret = DIV_ROUND_CLOSEST(1350000U, ret); + else + ret = 1350000U; abort: mutex_unlock(&data->access_lock); return ret; } static int nct7802_write_fan_min(struct nct7802_data *data, u8 reg_fan_low, - u8 reg_fan_high, unsigned int limit) + u8 reg_fan_high, unsigned long limit) { int err; @@ -326,8 +328,8 @@ static int nct7802_write_voltage(struct nct7802_data *data, int nr, int index, int shift = 8 - REG_VOLTAGE_LIMIT_MSB_SHIFT[index - 1][nr]; int err; + voltage = clamp_val(voltage, 0, 0x3ff * nct7802_vmul[nr]); voltage = DIV_ROUND_CLOSEST(voltage, nct7802_vmul[nr]); - voltage = clamp_val(voltage, 0, 0x3ff); mutex_lock(&data->access_lock); err = regmap_write(data->regmap, @@ -402,7 +404,7 @@ static ssize_t store_temp(struct device *dev, struct device_attribute *attr, if (err < 0) return err; - val = clamp_val(DIV_ROUND_CLOSEST(val, 1000), -128, 127); + val = DIV_ROUND_CLOSEST(clamp_val(val, -128000, 127000), 1000); err = regmap_write(data->regmap, nr, val & 0xff); return err ? : count; diff --git a/drivers/hwmon/scpi-hwmon.c b/drivers/hwmon/scpi-hwmon.c index 559a3dcd64d8d6..094f948f99ffb0 100644 --- a/drivers/hwmon/scpi-hwmon.c +++ b/drivers/hwmon/scpi-hwmon.c @@ -251,6 +251,7 @@ static const struct of_device_id scpi_of_match[] = { {.compatible = "arm,scpi-sensors"}, {}, }; +MODULE_DEVICE_TABLE(of, scpi_of_match); static struct platform_driver scpi_hwmon_platdrv = { .driver = { diff --git a/drivers/hwtracing/stm/core.c b/drivers/hwtracing/stm/core.c index 51f81d64ca37f9..a6ea387b5b0038 100644 --- a/drivers/hwtracing/stm/core.c +++ b/drivers/hwtracing/stm/core.c @@ -361,7 +361,7 @@ static int stm_char_open(struct inode *inode, struct file *file) struct stm_file *stmf; struct device *dev; unsigned int major = imajor(inode); - int err = -ENODEV; + int err = -ENOMEM; dev = class_find_device(&stm_class, NULL, &major, major_match); if (!dev) @@ -369,8 +369,9 @@ static int stm_char_open(struct inode *inode, struct file *file) stmf = kzalloc(sizeof(*stmf), GFP_KERNEL); if (!stmf) - return -ENOMEM; + goto err_put_device; + err = -ENODEV; stm_output_init(&stmf->output); stmf->stm = to_stm_device(dev); @@ -382,9 +383,10 @@ static int stm_char_open(struct inode *inode, struct file *file) return nonseekable_open(inode, file); err_free: + kfree(stmf); +err_put_device: /* matches class_find_device() above */ put_device(dev); - kfree(stmf); return err; } diff --git a/drivers/i2c/busses/i2c-piix4.c b/drivers/i2c/busses/i2c-piix4.c index c2268cdf38e823..e34d82e79b988a 100644 --- a/drivers/i2c/busses/i2c-piix4.c +++ b/drivers/i2c/busses/i2c-piix4.c @@ -585,10 +585,29 @@ static s32 piix4_access_sb800(struct i2c_adapter *adap, u16 addr, u8 command, int size, union i2c_smbus_data *data) { struct i2c_piix4_adapdata *adapdata = i2c_get_adapdata(adap); + unsigned short piix4_smba = adapdata->smba; + int retries = MAX_TIMEOUT; + int smbslvcnt; u8 smba_en_lo; u8 port; int retval; + /* Request the SMBUS semaphore, avoid conflicts with the IMC */ + smbslvcnt = inb_p(SMBSLVCNT); + do { + outb_p(smbslvcnt | 0x10, SMBSLVCNT); + + /* Check the semaphore status */ + smbslvcnt = inb_p(SMBSLVCNT); + if (smbslvcnt & 0x10) + break; + + usleep_range(1000, 2000); + } while (--retries); + /* SMBus is still owned by the IMC, we give up */ + if (!retries) + return -EBUSY; + mutex_lock(&piix4_mutex_sb800); outb_p(piix4_port_sel_sb800, SB800_PIIX4_SMB_IDX); @@ -606,6 +625,9 @@ static s32 piix4_access_sb800(struct i2c_adapter *adap, u16 addr, mutex_unlock(&piix4_mutex_sb800); + /* Release the semaphore */ + outb_p(smbslvcnt | 0x20, SMBSLVCNT); + return retval; } diff --git a/drivers/i2c/i2c-core.c b/drivers/i2c/i2c-core.c index b432b64e307a81..7484aac1e14d6e 100644 --- a/drivers/i2c/i2c-core.c +++ b/drivers/i2c/i2c-core.c @@ -1657,7 +1657,7 @@ static struct i2c_client *of_i2c_register_device(struct i2c_adapter *adap, if (i2c_check_addr_validity(addr, info.flags)) { dev_err(&adap->dev, "of_i2c: invalid addr=%x on %s\n", - info.addr, node->full_name); + addr, node->full_name); return ERR_PTR(-EINVAL); } diff --git a/drivers/i2c/i2c-dev.c b/drivers/i2c/i2c-dev.c index 66f323fd39826e..6f638bbc922db4 100644 --- a/drivers/i2c/i2c-dev.c +++ b/drivers/i2c/i2c-dev.c @@ -331,7 +331,7 @@ static noinline int i2cdev_ioctl_smbus(struct i2c_client *client, unsigned long arg) { struct i2c_smbus_ioctl_data data_arg; - union i2c_smbus_data temp; + union i2c_smbus_data temp = {}; int datasize, res; if (copy_from_user(&data_arg, diff --git a/drivers/i2c/muxes/i2c-mux-pca954x.c b/drivers/i2c/muxes/i2c-mux-pca954x.c index 8bc3d36d28379e..9c4ac26c014e16 100644 --- a/drivers/i2c/muxes/i2c-mux-pca954x.c +++ b/drivers/i2c/muxes/i2c-mux-pca954x.c @@ -151,6 +151,9 @@ static int pca954x_reg_write(struct i2c_adapter *adap, buf[0] = val; msg.buf = buf; ret = __i2c_transfer(adap, &msg, 1); + + if (ret >= 0 && ret != 1) + ret = -EREMOTEIO; } else { union i2c_smbus_data data; ret = adap->algo->smbus_xfer(adap, client->addr, @@ -179,7 +182,7 @@ static int pca954x_select_chan(struct i2c_mux_core *muxc, u32 chan) /* Only select the channel if its different from the last channel */ if (data->last_chan != regval) { ret = pca954x_reg_write(muxc->parent, client, regval); - data->last_chan = ret ? 0 : regval; + data->last_chan = ret < 0 ? 0 : regval; } return ret; diff --git a/drivers/iio/accel/st_accel_core.c b/drivers/iio/accel/st_accel_core.c index ce69048c88e98b..3a557e3181eac5 100644 --- a/drivers/iio/accel/st_accel_core.c +++ b/drivers/iio/accel/st_accel_core.c @@ -154,8 +154,8 @@ #define ST_ACCEL_4_FS_MASK 0x80 #define ST_ACCEL_4_FS_AVL_2_VAL 0X00 #define ST_ACCEL_4_FS_AVL_6_VAL 0X01 -#define ST_ACCEL_4_FS_AVL_2_GAIN IIO_G_TO_M_S_2(1024) -#define ST_ACCEL_4_FS_AVL_6_GAIN IIO_G_TO_M_S_2(340) +#define ST_ACCEL_4_FS_AVL_2_GAIN IIO_G_TO_M_S_2(1000) +#define ST_ACCEL_4_FS_AVL_6_GAIN IIO_G_TO_M_S_2(3000) #define ST_ACCEL_4_BDU_ADDR 0x21 #define ST_ACCEL_4_BDU_MASK 0x40 #define ST_ACCEL_4_DRDY_IRQ_ADDR 0x21 @@ -346,6 +346,14 @@ static const struct st_sensor_settings st_accel_sensors_settings[] = { .addr = ST_ACCEL_1_BDU_ADDR, .mask = ST_ACCEL_1_BDU_MASK, }, + /* + * Data Alignment Setting - needs to be set to get + * left-justified data like all other sensors. + */ + .das = { + .addr = 0x21, + .mask = 0x01, + }, .drdy_irq = { .addr = ST_ACCEL_1_DRDY_IRQ_ADDR, .mask_int1 = ST_ACCEL_1_DRDY_IRQ_INT1_MASK, diff --git a/drivers/iio/adc/palmas_gpadc.c b/drivers/iio/adc/palmas_gpadc.c index 2bbf0c521bebb5..7d61b566e148dd 100644 --- a/drivers/iio/adc/palmas_gpadc.c +++ b/drivers/iio/adc/palmas_gpadc.c @@ -775,7 +775,7 @@ static int palmas_adc_wakeup_reset(struct palmas_gpadc *adc) static int palmas_gpadc_suspend(struct device *dev) { - struct iio_dev *indio_dev = dev_to_iio_dev(dev); + struct iio_dev *indio_dev = dev_get_drvdata(dev); struct palmas_gpadc *adc = iio_priv(indio_dev); int wakeup = adc->wakeup1_enable || adc->wakeup2_enable; int ret; @@ -798,7 +798,7 @@ static int palmas_gpadc_suspend(struct device *dev) static int palmas_gpadc_resume(struct device *dev) { - struct iio_dev *indio_dev = dev_to_iio_dev(dev); + struct iio_dev *indio_dev = dev_get_drvdata(dev); struct palmas_gpadc *adc = iio_priv(indio_dev); int wakeup = adc->wakeup1_enable || adc->wakeup2_enable; int ret; diff --git a/drivers/iio/common/st_sensors/st_sensors_buffer.c b/drivers/iio/common/st_sensors/st_sensors_buffer.c index fe7775bb37402e..df4045203a0717 100644 --- a/drivers/iio/common/st_sensors/st_sensors_buffer.c +++ b/drivers/iio/common/st_sensors/st_sensors_buffer.c @@ -30,7 +30,9 @@ static int st_sensors_get_buffer_element(struct iio_dev *indio_dev, u8 *buf) for_each_set_bit(i, indio_dev->active_scan_mask, num_data_channels) { const struct iio_chan_spec *channel = &indio_dev->channels[i]; - unsigned int bytes_to_read = channel->scan_type.realbits >> 3; + unsigned int bytes_to_read = + DIV_ROUND_UP(channel->scan_type.realbits + + channel->scan_type.shift, 8); unsigned int storage_bytes = channel->scan_type.storagebits >> 3; diff --git a/drivers/iio/common/st_sensors/st_sensors_core.c b/drivers/iio/common/st_sensors/st_sensors_core.c index 975a1f19f74760..79c8c7cd70d5c6 100644 --- a/drivers/iio/common/st_sensors/st_sensors_core.c +++ b/drivers/iio/common/st_sensors/st_sensors_core.c @@ -401,6 +401,15 @@ int st_sensors_init_sensor(struct iio_dev *indio_dev, return err; } + /* set DAS */ + if (sdata->sensor_settings->das.addr) { + err = st_sensors_write_data_with_mask(indio_dev, + sdata->sensor_settings->das.addr, + sdata->sensor_settings->das.mask, 1); + if (err < 0) + return err; + } + if (sdata->int_pin_open_drain) { dev_info(&indio_dev->dev, "set interrupt line to open drain mode\n"); @@ -483,8 +492,10 @@ static int st_sensors_read_axis_data(struct iio_dev *indio_dev, int err; u8 *outdata; struct st_sensor_data *sdata = iio_priv(indio_dev); - unsigned int byte_for_channel = ch->scan_type.realbits >> 3; + unsigned int byte_for_channel; + byte_for_channel = DIV_ROUND_UP(ch->scan_type.realbits + + ch->scan_type.shift, 8); outdata = kmalloc(byte_for_channel, GFP_KERNEL); if (!outdata) return -ENOMEM; diff --git a/drivers/iio/health/afe4403.c b/drivers/iio/health/afe4403.c index 9a081465c42f42..6bb23a49e81eb8 100644 --- a/drivers/iio/health/afe4403.c +++ b/drivers/iio/health/afe4403.c @@ -422,7 +422,7 @@ MODULE_DEVICE_TABLE(of, afe4403_of_match); static int __maybe_unused afe4403_suspend(struct device *dev) { - struct iio_dev *indio_dev = dev_to_iio_dev(dev); + struct iio_dev *indio_dev = spi_get_drvdata(to_spi_device(dev)); struct afe4403_data *afe = iio_priv(indio_dev); int ret; @@ -443,7 +443,7 @@ static int __maybe_unused afe4403_suspend(struct device *dev) static int __maybe_unused afe4403_resume(struct device *dev) { - struct iio_dev *indio_dev = dev_to_iio_dev(dev); + struct iio_dev *indio_dev = spi_get_drvdata(to_spi_device(dev)); struct afe4403_data *afe = iio_priv(indio_dev); int ret; diff --git a/drivers/iio/health/afe4404.c b/drivers/iio/health/afe4404.c index 45266404f7e3b5..964f5231a831c4 100644 --- a/drivers/iio/health/afe4404.c +++ b/drivers/iio/health/afe4404.c @@ -428,7 +428,7 @@ MODULE_DEVICE_TABLE(of, afe4404_of_match); static int __maybe_unused afe4404_suspend(struct device *dev) { - struct iio_dev *indio_dev = dev_to_iio_dev(dev); + struct iio_dev *indio_dev = i2c_get_clientdata(to_i2c_client(dev)); struct afe4404_data *afe = iio_priv(indio_dev); int ret; @@ -449,7 +449,7 @@ static int __maybe_unused afe4404_suspend(struct device *dev) static int __maybe_unused afe4404_resume(struct device *dev) { - struct iio_dev *indio_dev = dev_to_iio_dev(dev); + struct iio_dev *indio_dev = i2c_get_clientdata(to_i2c_client(dev)); struct afe4404_data *afe = iio_priv(indio_dev); int ret; diff --git a/drivers/iio/health/max30100.c b/drivers/iio/health/max30100.c index 90ab8a2d2846f8..183c14329d6e35 100644 --- a/drivers/iio/health/max30100.c +++ b/drivers/iio/health/max30100.c @@ -238,7 +238,7 @@ static irqreturn_t max30100_interrupt_handler(int irq, void *private) mutex_lock(&data->lock); - while (cnt || (cnt = max30100_fifo_count(data) > 0)) { + while (cnt || (cnt = max30100_fifo_count(data)) > 0) { ret = max30100_read_measurement(data); if (ret) break; diff --git a/drivers/iio/humidity/dht11.c b/drivers/iio/humidity/dht11.c index 9c47bc98f3acde..2a22ad92033306 100644 --- a/drivers/iio/humidity/dht11.c +++ b/drivers/iio/humidity/dht11.c @@ -71,7 +71,8 @@ * a) select an implementation using busy loop polling on those systems * b) use the checksum to do some probabilistic decoding */ -#define DHT11_START_TRANSMISSION 18 /* ms */ +#define DHT11_START_TRANSMISSION_MIN 18000 /* us */ +#define DHT11_START_TRANSMISSION_MAX 20000 /* us */ #define DHT11_MIN_TIMERES 34000 /* ns */ #define DHT11_THRESHOLD 49000 /* ns */ #define DHT11_AMBIG_LOW 23000 /* ns */ @@ -228,7 +229,8 @@ static int dht11_read_raw(struct iio_dev *iio_dev, ret = gpio_direction_output(dht11->gpio, 0); if (ret) goto err; - msleep(DHT11_START_TRANSMISSION); + usleep_range(DHT11_START_TRANSMISSION_MIN, + DHT11_START_TRANSMISSION_MAX); ret = gpio_direction_input(dht11->gpio); if (ret) goto err; diff --git a/drivers/iio/imu/bmi160/bmi160_core.c b/drivers/iio/imu/bmi160/bmi160_core.c index e0251b8c1a527a..5fb571d0315377 100644 --- a/drivers/iio/imu/bmi160/bmi160_core.c +++ b/drivers/iio/imu/bmi160/bmi160_core.c @@ -66,10 +66,8 @@ #define BMI160_REG_DUMMY 0x7F -#define BMI160_ACCEL_PMU_MIN_USLEEP 3200 -#define BMI160_ACCEL_PMU_MAX_USLEEP 3800 -#define BMI160_GYRO_PMU_MIN_USLEEP 55000 -#define BMI160_GYRO_PMU_MAX_USLEEP 80000 +#define BMI160_ACCEL_PMU_MIN_USLEEP 3800 +#define BMI160_GYRO_PMU_MIN_USLEEP 80000 #define BMI160_SOFTRESET_USLEEP 1000 #define BMI160_CHANNEL(_type, _axis, _index) { \ @@ -151,20 +149,9 @@ static struct bmi160_regs bmi160_regs[] = { }, }; -struct bmi160_pmu_time { - unsigned long min; - unsigned long max; -}; - -static struct bmi160_pmu_time bmi160_pmu_time[] = { - [BMI160_ACCEL] = { - .min = BMI160_ACCEL_PMU_MIN_USLEEP, - .max = BMI160_ACCEL_PMU_MAX_USLEEP - }, - [BMI160_GYRO] = { - .min = BMI160_GYRO_PMU_MIN_USLEEP, - .max = BMI160_GYRO_PMU_MIN_USLEEP, - }, +static unsigned long bmi160_pmu_time[] = { + [BMI160_ACCEL] = BMI160_ACCEL_PMU_MIN_USLEEP, + [BMI160_GYRO] = BMI160_GYRO_PMU_MIN_USLEEP, }; struct bmi160_scale { @@ -289,7 +276,7 @@ int bmi160_set_mode(struct bmi160_data *data, enum bmi160_sensor_type t, if (ret < 0) return ret; - usleep_range(bmi160_pmu_time[t].min, bmi160_pmu_time[t].max); + usleep_range(bmi160_pmu_time[t], bmi160_pmu_time[t] + 1000); return 0; } diff --git a/drivers/iio/light/max44000.c b/drivers/iio/light/max44000.c index 6511b20a2a2966..a8ffa432bf0deb 100644 --- a/drivers/iio/light/max44000.c +++ b/drivers/iio/light/max44000.c @@ -113,7 +113,7 @@ static const char max44000_int_time_avail_str[] = "0.100 " "0.025 " "0.00625 " - "0.001625"; + "0.0015625"; /* Available scales (internal to ulux) with pretty manual alignment: */ static const int max44000_scale_avail_ulux_array[] = { diff --git a/drivers/infiniband/core/cache.c b/drivers/infiniband/core/cache.c index 1a2984c28b9589..ae04826e82fc09 100644 --- a/drivers/infiniband/core/cache.c +++ b/drivers/infiniband/core/cache.c @@ -770,12 +770,8 @@ static int _gid_table_setup_one(struct ib_device *ib_dev) int err = 0; table = kcalloc(ib_dev->phys_port_cnt, sizeof(*table), GFP_KERNEL); - - if (!table) { - pr_warn("failed to allocate ib gid cache for %s\n", - ib_dev->name); + if (!table) return -ENOMEM; - } for (port = 0; port < ib_dev->phys_port_cnt; port++) { u8 rdma_port = port + rdma_start_port(ib_dev); @@ -1170,14 +1166,13 @@ int ib_cache_setup_one(struct ib_device *device) GFP_KERNEL); if (!device->cache.pkey_cache || !device->cache.lmc_cache) { - pr_warn("Couldn't allocate cache for %s\n", device->name); - return -ENOMEM; + err = -ENOMEM; + goto free; } err = gid_table_setup_one(device); if (err) - /* Allocated memory will be cleaned in the release function */ - return err; + goto free; for (p = 0; p <= rdma_end_port(device) - rdma_start_port(device); ++p) ib_cache_update(device, p + rdma_start_port(device)); @@ -1192,6 +1187,9 @@ int ib_cache_setup_one(struct ib_device *device) err: gid_table_cleanup_one(device); +free: + kfree(device->cache.pkey_cache); + kfree(device->cache.lmc_cache); return err; } diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c index 2a6fc47a1dfbb5..c25768c2dd3b14 100644 --- a/drivers/infiniband/core/cma.c +++ b/drivers/infiniband/core/cma.c @@ -2768,7 +2768,8 @@ static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr, if (!src_addr || !src_addr->sa_family) { src_addr = (struct sockaddr *) &id->route.addr.src_addr; src_addr->sa_family = dst_addr->sa_family; - if (dst_addr->sa_family == AF_INET6) { + if (IS_ENABLED(CONFIG_IPV6) && + dst_addr->sa_family == AF_INET6) { struct sockaddr_in6 *src_addr6 = (struct sockaddr_in6 *) src_addr; struct sockaddr_in6 *dst_addr6 = (struct sockaddr_in6 *) dst_addr; src_addr6->sin6_scope_id = dst_addr6->sin6_scope_id; diff --git a/drivers/infiniband/core/mad.c b/drivers/infiniband/core/mad.c index 40cbd6bdb73be1..2395fe2021c961 100644 --- a/drivers/infiniband/core/mad.c +++ b/drivers/infiniband/core/mad.c @@ -1746,7 +1746,7 @@ find_mad_agent(struct ib_mad_port_private *port_priv, if (!class) goto out; if (convert_mgmt_class(mad_hdr->mgmt_class) >= - IB_MGMT_MAX_METHODS) + ARRAY_SIZE(class->method_table)) goto out; method = class->method_table[convert_mgmt_class( mad_hdr->mgmt_class)]; diff --git a/drivers/infiniband/core/multicast.c b/drivers/infiniband/core/multicast.c index e51b739f6ea3d2..322cb67b07a961 100644 --- a/drivers/infiniband/core/multicast.c +++ b/drivers/infiniband/core/multicast.c @@ -518,8 +518,11 @@ static void join_handler(int status, struct ib_sa_mcmember_rec *rec, process_join_error(group, status); else { int mgids_changed, is_mgid0; - ib_find_pkey(group->port->dev->device, group->port->port_num, - be16_to_cpu(rec->pkey), &pkey_index); + + if (ib_find_pkey(group->port->dev->device, + group->port->port_num, be16_to_cpu(rec->pkey), + &pkey_index)) + pkey_index = MCAST_INVALID_PKEY_INDEX; spin_lock_irq(&group->port->lock); if (group->state == MCAST_BUSY && diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c index 84b4eff90395e5..c22fde6207d141 100644 --- a/drivers/infiniband/core/umem.c +++ b/drivers/infiniband/core/umem.c @@ -134,6 +134,7 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr, IB_ACCESS_REMOTE_ATOMIC | IB_ACCESS_MW_BIND)); if (access & IB_ACCESS_ON_DEMAND) { + put_pid(umem->pid); ret = ib_umem_odp_get(context, umem); if (ret) { kfree(umem); @@ -149,6 +150,7 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr, page_list = (struct page **) __get_free_page(GFP_KERNEL); if (!page_list) { + put_pid(umem->pid); kfree(umem); return ERR_PTR(-ENOMEM); } diff --git a/drivers/infiniband/hw/cxgb4/device.c b/drivers/infiniband/hw/cxgb4/device.c index 93e3d270a98af6..b85a1a983e07c8 100644 --- a/drivers/infiniband/hw/cxgb4/device.c +++ b/drivers/infiniband/hw/cxgb4/device.c @@ -828,8 +828,10 @@ static int c4iw_rdev_open(struct c4iw_rdev *rdev) } rdev->status_page = (struct t4_dev_status_page *) __get_free_page(GFP_KERNEL); - if (!rdev->status_page) + if (!rdev->status_page) { + err = -ENOMEM; goto destroy_ocqp_pool; + } rdev->status_page->qp_start = rdev->lldi.vr->qp.start; rdev->status_page->qp_size = rdev->lldi.vr->qp.size; rdev->status_page->cq_start = rdev->lldi.vr->cq.start; @@ -846,9 +848,17 @@ static int c4iw_rdev_open(struct c4iw_rdev *rdev) } } + rdev->free_workq = create_singlethread_workqueue("iw_cxgb4_free"); + if (!rdev->free_workq) { + err = -ENOMEM; + goto err_free_status_page; + } + rdev->status_page->db_off = 0; return 0; +err_free_status_page: + free_page((unsigned long)rdev->status_page); destroy_ocqp_pool: c4iw_ocqp_pool_destroy(rdev); destroy_rqtpool: @@ -862,6 +872,7 @@ static int c4iw_rdev_open(struct c4iw_rdev *rdev) static void c4iw_rdev_close(struct c4iw_rdev *rdev) { + destroy_workqueue(rdev->free_workq); kfree(rdev->wr_log); free_page((unsigned long)rdev->status_page); c4iw_pblpool_destroy(rdev); diff --git a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h index 4788e1a46fdee2..7d540667dad26a 100644 --- a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h +++ b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h @@ -45,6 +45,7 @@ #include #include #include +#include #include @@ -107,6 +108,7 @@ struct c4iw_dev_ucontext { struct list_head qpids; struct list_head cqids; struct mutex lock; + struct kref kref; }; enum c4iw_rdev_flags { @@ -183,6 +185,7 @@ struct c4iw_rdev { atomic_t wr_log_idx; struct wr_log_entry *wr_log; int wr_log_size; + struct workqueue_struct *free_workq; }; static inline int c4iw_fatal_error(struct c4iw_rdev *rdev) @@ -482,6 +485,8 @@ struct c4iw_qp { int sq_sig_all; struct completion rq_drained; struct completion sq_drained; + struct work_struct free_work; + struct c4iw_ucontext *ucontext; }; static inline struct c4iw_qp *to_c4iw_qp(struct ib_qp *ibqp) @@ -495,6 +500,7 @@ struct c4iw_ucontext { u32 key; spinlock_t mmap_lock; struct list_head mmaps; + struct kref kref; }; static inline struct c4iw_ucontext *to_c4iw_ucontext(struct ib_ucontext *c) @@ -502,6 +508,18 @@ static inline struct c4iw_ucontext *to_c4iw_ucontext(struct ib_ucontext *c) return container_of(c, struct c4iw_ucontext, ibucontext); } +void _c4iw_free_ucontext(struct kref *kref); + +static inline void c4iw_put_ucontext(struct c4iw_ucontext *ucontext) +{ + kref_put(&ucontext->kref, _c4iw_free_ucontext); +} + +static inline void c4iw_get_ucontext(struct c4iw_ucontext *ucontext) +{ + kref_get(&ucontext->kref); +} + struct c4iw_mm_entry { struct list_head entry; u64 addr; diff --git a/drivers/infiniband/hw/cxgb4/provider.c b/drivers/infiniband/hw/cxgb4/provider.c index 645e606a17c569..8278ba06f9958a 100644 --- a/drivers/infiniband/hw/cxgb4/provider.c +++ b/drivers/infiniband/hw/cxgb4/provider.c @@ -91,17 +91,28 @@ static int c4iw_process_mad(struct ib_device *ibdev, int mad_flags, return -ENOSYS; } -static int c4iw_dealloc_ucontext(struct ib_ucontext *context) +void _c4iw_free_ucontext(struct kref *kref) { - struct c4iw_dev *rhp = to_c4iw_dev(context->device); - struct c4iw_ucontext *ucontext = to_c4iw_ucontext(context); + struct c4iw_ucontext *ucontext; + struct c4iw_dev *rhp; struct c4iw_mm_entry *mm, *tmp; - PDBG("%s context %p\n", __func__, context); + ucontext = container_of(kref, struct c4iw_ucontext, kref); + rhp = to_c4iw_dev(ucontext->ibucontext.device); + + PDBG("%s ucontext %p\n", __func__, ucontext); list_for_each_entry_safe(mm, tmp, &ucontext->mmaps, entry) kfree(mm); c4iw_release_dev_ucontext(&rhp->rdev, &ucontext->uctx); kfree(ucontext); +} + +static int c4iw_dealloc_ucontext(struct ib_ucontext *context) +{ + struct c4iw_ucontext *ucontext = to_c4iw_ucontext(context); + + PDBG("%s context %p\n", __func__, context); + c4iw_put_ucontext(ucontext); return 0; } @@ -125,6 +136,7 @@ static struct ib_ucontext *c4iw_alloc_ucontext(struct ib_device *ibdev, c4iw_init_dev_ucontext(&rhp->rdev, &context->uctx); INIT_LIST_HEAD(&context->mmaps); spin_lock_init(&context->mmap_lock); + kref_init(&context->kref); if (udata->outlen < sizeof(uresp) - sizeof(uresp.reserved)) { if (!warned++) diff --git a/drivers/infiniband/hw/cxgb4/qp.c b/drivers/infiniband/hw/cxgb4/qp.c index b7ac97b27c88c2..cc2243f6cc7f56 100644 --- a/drivers/infiniband/hw/cxgb4/qp.c +++ b/drivers/infiniband/hw/cxgb4/qp.c @@ -321,7 +321,8 @@ static int create_qp(struct c4iw_rdev *rdev, struct t4_wq *wq, FW_RI_RES_WR_DCAEN_V(0) | FW_RI_RES_WR_DCACPU_V(0) | FW_RI_RES_WR_FBMIN_V(2) | - FW_RI_RES_WR_FBMAX_V(2) | + (t4_sq_onchip(&wq->sq) ? FW_RI_RES_WR_FBMAX_V(2) : + FW_RI_RES_WR_FBMAX_V(3)) | FW_RI_RES_WR_CIDXFTHRESHO_V(0) | FW_RI_RES_WR_CIDXFTHRESH_V(0) | FW_RI_RES_WR_EQSIZE_V(eqsize)); @@ -345,7 +346,7 @@ static int create_qp(struct c4iw_rdev *rdev, struct t4_wq *wq, FW_RI_RES_WR_DCAEN_V(0) | FW_RI_RES_WR_DCACPU_V(0) | FW_RI_RES_WR_FBMIN_V(2) | - FW_RI_RES_WR_FBMAX_V(2) | + FW_RI_RES_WR_FBMAX_V(3) | FW_RI_RES_WR_CIDXFTHRESHO_V(0) | FW_RI_RES_WR_CIDXFTHRESH_V(0) | FW_RI_RES_WR_EQSIZE_V(eqsize)); @@ -714,13 +715,32 @@ static int build_inv_stag(union t4_wr *wqe, struct ib_send_wr *wr, u8 *len16) return 0; } -static void _free_qp(struct kref *kref) +static void free_qp_work(struct work_struct *work) +{ + struct c4iw_ucontext *ucontext; + struct c4iw_qp *qhp; + struct c4iw_dev *rhp; + + qhp = container_of(work, struct c4iw_qp, free_work); + ucontext = qhp->ucontext; + rhp = qhp->rhp; + + PDBG("%s qhp %p ucontext %p\n", __func__, qhp, ucontext); + destroy_qp(&rhp->rdev, &qhp->wq, + ucontext ? &ucontext->uctx : &rhp->rdev.uctx); + + if (ucontext) + c4iw_put_ucontext(ucontext); + kfree(qhp); +} + +static void queue_qp_free(struct kref *kref) { struct c4iw_qp *qhp; qhp = container_of(kref, struct c4iw_qp, kref); PDBG("%s qhp %p\n", __func__, qhp); - kfree(qhp); + queue_work(qhp->rhp->rdev.free_workq, &qhp->free_work); } void c4iw_qp_add_ref(struct ib_qp *qp) @@ -732,7 +752,7 @@ void c4iw_qp_add_ref(struct ib_qp *qp) void c4iw_qp_rem_ref(struct ib_qp *qp) { PDBG("%s ib_qp %p\n", __func__, qp); - kref_put(&to_c4iw_qp(qp)->kref, _free_qp); + kref_put(&to_c4iw_qp(qp)->kref, queue_qp_free); } static void add_to_fc_list(struct list_head *head, struct list_head *entry) @@ -1642,7 +1662,6 @@ int c4iw_destroy_qp(struct ib_qp *ib_qp) struct c4iw_dev *rhp; struct c4iw_qp *qhp; struct c4iw_qp_attributes attrs; - struct c4iw_ucontext *ucontext; qhp = to_c4iw_qp(ib_qp); rhp = qhp->rhp; @@ -1662,11 +1681,6 @@ int c4iw_destroy_qp(struct ib_qp *ib_qp) spin_unlock_irq(&rhp->lock); free_ird(rhp, qhp->attr.max_ird); - ucontext = ib_qp->uobject ? - to_c4iw_ucontext(ib_qp->uobject->context) : NULL; - destroy_qp(&rhp->rdev, &qhp->wq, - ucontext ? &ucontext->uctx : &rhp->rdev.uctx); - c4iw_qp_rem_ref(ib_qp); PDBG("%s ib_qp %p qpid 0x%0x\n", __func__, ib_qp, qhp->wq.sq.qid); @@ -1767,6 +1781,7 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs, mutex_init(&qhp->mutex); init_waitqueue_head(&qhp->wait); kref_init(&qhp->kref); + INIT_WORK(&qhp->free_work, free_qp_work); ret = insert_handle(rhp, &rhp->qpidr, qhp, qhp->wq.sq.qid); if (ret) @@ -1853,6 +1868,9 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs, ma_sync_key_mm->len = PAGE_SIZE; insert_mmap(ucontext, ma_sync_key_mm); } + + c4iw_get_ucontext(ucontext); + qhp->ucontext = ucontext; } qhp->ibqp.qp_num = qhp->wq.sq.qid; init_timer(&(qhp->timer)); diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.c b/drivers/infiniband/hw/i40iw/i40iw_verbs.c index 6329c971c22fc3..4b892ca2b13a49 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_verbs.c +++ b/drivers/infiniband/hw/i40iw/i40iw_verbs.c @@ -2501,7 +2501,7 @@ static int i40iw_get_hw_stats(struct ib_device *ibdev, return -ENOSYS; } - memcpy(&stats->value[0], &hw_stats, sizeof(*hw_stats)); + memcpy(&stats->value[0], hw_stats, sizeof(*hw_stats)); return stats->num_counters; } diff --git a/drivers/infiniband/hw/mlx4/ah.c b/drivers/infiniband/hw/mlx4/ah.c index b9bf0759f10a54..8dfc76f8cbb410 100644 --- a/drivers/infiniband/hw/mlx4/ah.c +++ b/drivers/infiniband/hw/mlx4/ah.c @@ -114,7 +114,9 @@ static struct ib_ah *create_iboe_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr !(1 << ah->av.eth.stat_rate & dev->caps.stat_rate_support)) --ah->av.eth.stat_rate; } - + ah->av.eth.sl_tclass_flowlabel |= + cpu_to_be32((ah_attr->grh.traffic_class << 20) | + ah_attr->grh.flow_label); /* * HW requires multicast LID so we just choose one. */ @@ -122,7 +124,7 @@ static struct ib_ah *create_iboe_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr ah->av.ib.dlid = cpu_to_be16(0xc000); memcpy(ah->av.eth.dgid, ah_attr->grh.dgid.raw, 16); - ah->av.eth.sl_tclass_flowlabel = cpu_to_be32(ah_attr->sl << 29); + ah->av.eth.sl_tclass_flowlabel |= cpu_to_be32(ah_attr->sl << 29); return &ah->ibah; } diff --git a/drivers/infiniband/hw/mlx4/mad.c b/drivers/infiniband/hw/mlx4/mad.c index 1672907ff21977..18d309e40f1b92 100644 --- a/drivers/infiniband/hw/mlx4/mad.c +++ b/drivers/infiniband/hw/mlx4/mad.c @@ -702,10 +702,18 @@ static int mlx4_ib_demux_mad(struct ib_device *ibdev, u8 port, /* If a grh is present, we demux according to it */ if (wc->wc_flags & IB_WC_GRH) { - slave = mlx4_ib_find_real_gid(ibdev, port, grh->dgid.global.interface_id); - if (slave < 0) { - mlx4_ib_warn(ibdev, "failed matching grh\n"); - return -ENOENT; + if (grh->dgid.global.interface_id == + cpu_to_be64(IB_SA_WELL_KNOWN_GUID) && + grh->dgid.global.subnet_prefix == cpu_to_be64( + atomic64_read(&dev->sriov.demux[port - 1].subnet_prefix))) { + slave = 0; + } else { + slave = mlx4_ib_find_real_gid(ibdev, port, + grh->dgid.global.interface_id); + if (slave < 0) { + mlx4_ib_warn(ibdev, "failed matching grh\n"); + return -ENOENT; + } } } /* Class-specific handling */ diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c index b597e822759139..46ad99595fd208 100644 --- a/drivers/infiniband/hw/mlx4/main.c +++ b/drivers/infiniband/hw/mlx4/main.c @@ -697,9 +697,11 @@ static int eth_link_query_port(struct ib_device *ibdev, u8 port, if (err) goto out; - props->active_width = (((u8 *)mailbox->buf)[5] == 0x40) ? - IB_WIDTH_4X : IB_WIDTH_1X; - props->active_speed = IB_SPEED_QDR; + props->active_width = (((u8 *)mailbox->buf)[5] == 0x40) || + (((u8 *)mailbox->buf)[5] == 0x20 /*56Gb*/) ? + IB_WIDTH_4X : IB_WIDTH_1X; + props->active_speed = (((u8 *)mailbox->buf)[5] == 0x20 /*56Gb*/) ? + IB_SPEED_FDR : IB_SPEED_QDR; props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_IP_BASED_GIDS; props->gid_tbl_len = mdev->dev->caps.gid_table_len[port]; props->max_msg_sz = mdev->dev->caps.max_msg_sz; @@ -2820,14 +2822,19 @@ static void *mlx4_ib_add(struct mlx4_dev *dev) goto err_steer_qp_release; } - bitmap_zero(ibdev->ib_uc_qpns_bitmap, ibdev->steer_qpn_count); - - err = mlx4_FLOW_STEERING_IB_UC_QP_RANGE( - dev, ibdev->steer_qpn_base, - ibdev->steer_qpn_base + - ibdev->steer_qpn_count - 1); - if (err) - goto err_steer_free_bitmap; + if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_DMFS_IPOIB) { + bitmap_zero(ibdev->ib_uc_qpns_bitmap, + ibdev->steer_qpn_count); + err = mlx4_FLOW_STEERING_IB_UC_QP_RANGE( + dev, ibdev->steer_qpn_base, + ibdev->steer_qpn_base + + ibdev->steer_qpn_count - 1); + if (err) + goto err_steer_free_bitmap; + } else { + bitmap_fill(ibdev->ib_uc_qpns_bitmap, + ibdev->steer_qpn_count); + } } for (j = 1; j <= ibdev->dev->caps.num_ports; j++) diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c index 570bc866b1d622..c22454383976c4 100644 --- a/drivers/infiniband/hw/mlx4/qp.c +++ b/drivers/infiniband/hw/mlx4/qp.c @@ -1280,7 +1280,8 @@ static int _mlx4_ib_destroy_qp(struct ib_qp *qp) if (is_qp0(dev, mqp)) mlx4_CLOSE_PORT(dev->dev, mqp->port); - if (dev->qp1_proxy[mqp->port - 1] == mqp) { + if (mqp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_GSI && + dev->qp1_proxy[mqp->port - 1] == mqp) { mutex_lock(&dev->qp1_proxy_lock[mqp->port - 1]); dev->qp1_proxy[mqp->port - 1] = NULL; mutex_unlock(&dev->qp1_proxy_lock[mqp->port - 1]); @@ -1764,14 +1765,14 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp, u8 port_num = mlx4_is_bonded(to_mdev(ibqp->device)->dev) ? 1 : attr_mask & IB_QP_PORT ? attr->port_num : qp->port; union ib_gid gid; - struct ib_gid_attr gid_attr; + struct ib_gid_attr gid_attr = {.gid_type = IB_GID_TYPE_IB}; u16 vlan = 0xffff; u8 smac[ETH_ALEN]; int status = 0; int is_eth = rdma_cap_eth_ah(&dev->ib_dev, port_num) && attr->ah_attr.ah_flags & IB_AH_GRH; - if (is_eth) { + if (is_eth && attr->ah_attr.ah_flags & IB_AH_GRH) { int index = attr->ah_attr.grh.sgid_index; status = ib_get_cached_gid(ibqp->device, port_num, diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 32b09f059c84a4..4cab29ea394cf0 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -496,6 +496,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev, struct mlx5_ib_dev *dev = to_mdev(ibdev); struct mlx5_core_dev *mdev = dev->mdev; int err = -ENOMEM; + int max_sq_desc; int max_rq_sg; int max_sq_sg; u64 min_page_size = 1ull << MLX5_CAP_GEN(mdev, log_pg_sz); @@ -618,9 +619,10 @@ static int mlx5_ib_query_device(struct ib_device *ibdev, props->max_qp_wr = 1 << MLX5_CAP_GEN(mdev, log_max_qp_sz); max_rq_sg = MLX5_CAP_GEN(mdev, max_wqe_sz_rq) / sizeof(struct mlx5_wqe_data_seg); - max_sq_sg = (MLX5_CAP_GEN(mdev, max_wqe_sz_sq) - - sizeof(struct mlx5_wqe_ctrl_seg)) / - sizeof(struct mlx5_wqe_data_seg); + max_sq_desc = min_t(int, MLX5_CAP_GEN(mdev, max_wqe_sz_sq), 512); + max_sq_sg = (max_sq_desc - sizeof(struct mlx5_wqe_ctrl_seg) - + sizeof(struct mlx5_wqe_raddr_seg)) / + sizeof(struct mlx5_wqe_data_seg); props->max_sge = min(max_rq_sg, max_sq_sg); props->max_sge_rd = MLX5_MAX_SGE_RD; props->max_cq = 1 << MLX5_CAP_GEN(mdev, log_max_cq); diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index 4e9012463c37de..be2d02b6a6aa6d 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -628,7 +628,8 @@ int mlx5_mr_cache_init(struct mlx5_ib_dev *dev) ent->order = i + 2; ent->dev = dev; - if (dev->mdev->profile->mask & MLX5_PROF_MASK_MR_CACHE) + if ((dev->mdev->profile->mask & MLX5_PROF_MASK_MR_CACHE) && + (mlx5_core_is_pf(dev->mdev))) limit = dev->mdev->profile->mr_cache[i].limit; else limit = 0; @@ -646,6 +647,33 @@ int mlx5_mr_cache_init(struct mlx5_ib_dev *dev) return 0; } +static void wait_for_async_commands(struct mlx5_ib_dev *dev) +{ + struct mlx5_mr_cache *cache = &dev->cache; + struct mlx5_cache_ent *ent; + int total = 0; + int i; + int j; + + for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) { + ent = &cache->ent[i]; + for (j = 0 ; j < 1000; j++) { + if (!ent->pending) + break; + msleep(50); + } + } + for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) { + ent = &cache->ent[i]; + total += ent->pending; + } + + if (total) + mlx5_ib_warn(dev, "aborted while there are %d pending mr requests\n", total); + else + mlx5_ib_warn(dev, "done with all pending requests\n"); +} + int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev) { int i; @@ -659,6 +687,7 @@ int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev) clean_keys(dev, i); destroy_workqueue(dev->cache.wq); + wait_for_async_commands(dev); del_timer_sync(&dev->delay_timer); return 0; diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index d1e921816bfee3..aee3942ec68d85 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -351,6 +351,29 @@ static int calc_send_wqe(struct ib_qp_init_attr *attr) return ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB); } +static int get_send_sge(struct ib_qp_init_attr *attr, int wqe_size) +{ + int max_sge; + + if (attr->qp_type == IB_QPT_RC) + max_sge = (min_t(int, wqe_size, 512) - + sizeof(struct mlx5_wqe_ctrl_seg) - + sizeof(struct mlx5_wqe_raddr_seg)) / + sizeof(struct mlx5_wqe_data_seg); + else if (attr->qp_type == IB_QPT_XRC_INI) + max_sge = (min_t(int, wqe_size, 512) - + sizeof(struct mlx5_wqe_ctrl_seg) - + sizeof(struct mlx5_wqe_xrc_seg) - + sizeof(struct mlx5_wqe_raddr_seg)) / + sizeof(struct mlx5_wqe_data_seg); + else + max_sge = (wqe_size - sq_overhead(attr)) / + sizeof(struct mlx5_wqe_data_seg); + + return min_t(int, max_sge, wqe_size - sq_overhead(attr) / + sizeof(struct mlx5_wqe_data_seg)); +} + static int calc_sq_size(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr, struct mlx5_ib_qp *qp) { @@ -387,7 +410,11 @@ static int calc_sq_size(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr, return -ENOMEM; } qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB); - qp->sq.max_gs = attr->cap.max_send_sge; + qp->sq.max_gs = get_send_sge(attr, wqe_size); + if (qp->sq.max_gs < attr->cap.max_send_sge) + return -ENOMEM; + + attr->cap.max_send_sge = qp->sq.max_gs; qp->sq.max_post = wq_size / wqe_size; attr->cap.max_send_wr = qp->sq.max_post; diff --git a/drivers/infiniband/hw/mlx5/srq.c b/drivers/infiniband/hw/mlx5/srq.c index 3857dbd9c95604..729b0696626e41 100644 --- a/drivers/infiniband/hw/mlx5/srq.c +++ b/drivers/infiniband/hw/mlx5/srq.c @@ -282,6 +282,7 @@ struct ib_srq *mlx5_ib_create_srq(struct ib_pd *pd, mlx5_ib_dbg(dev, "desc_size 0x%x, req wr 0x%x, srq size 0x%x, max_gs 0x%x, max_avail_gather 0x%x\n", desc_size, init_attr->attr.max_wr, srq->msrq.max, srq->msrq.max_gs, srq->msrq.max_avail_gather); + in.type = init_attr->srq_type; if (pd->uobject) err = create_srq_user(pd, srq, &in, udata, buf_size); @@ -294,7 +295,6 @@ struct ib_srq *mlx5_ib_create_srq(struct ib_pd *pd, goto err_srq; } - in.type = init_attr->srq_type; in.log_size = ilog2(srq->msrq.max); in.wqe_shift = srq->msrq.wqe_shift - 4; if (srq->wq_sig) diff --git a/drivers/infiniband/sw/rxe/rxe_mr.c b/drivers/infiniband/sw/rxe/rxe_mr.c index 1869152f1d2363..9b732c5f89e16c 100644 --- a/drivers/infiniband/sw/rxe/rxe_mr.c +++ b/drivers/infiniband/sw/rxe/rxe_mr.c @@ -59,9 +59,11 @@ int mem_check_range(struct rxe_mem *mem, u64 iova, size_t length) case RXE_MEM_TYPE_MR: case RXE_MEM_TYPE_FMR: - return ((iova < mem->iova) || - ((iova + length) > (mem->iova + mem->length))) ? - -EFAULT : 0; + if (iova < mem->iova || + length > mem->length || + iova > mem->iova + mem->length - length) + return -EFAULT; + return 0; default: return -EFAULT; diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c index ffff5a54cb340d..f4f3942ebbd133 100644 --- a/drivers/infiniband/sw/rxe/rxe_net.c +++ b/drivers/infiniband/sw/rxe/rxe_net.c @@ -554,7 +554,7 @@ struct rxe_dev *rxe_net_add(struct net_device *ndev) } spin_lock_bh(&dev_list_lock); - list_add_tail(&rxe_dev_list, &rxe->list); + list_add_tail(&rxe->list, &rxe_dev_list); spin_unlock_bh(&dev_list_lock); return rxe; } diff --git a/drivers/infiniband/sw/rxe/rxe_param.h b/drivers/infiniband/sw/rxe/rxe_param.h index f459c43a77c833..13ed2cc6eaa2ac 100644 --- a/drivers/infiniband/sw/rxe/rxe_param.h +++ b/drivers/infiniband/sw/rxe/rxe_param.h @@ -82,7 +82,7 @@ enum rxe_device_param { RXE_MAX_SGE = 32, RXE_MAX_SGE_RD = 32, RXE_MAX_CQ = 16384, - RXE_MAX_LOG_CQE = 13, + RXE_MAX_LOG_CQE = 15, RXE_MAX_MR = 2 * 1024, RXE_MAX_PD = 0x7ffc, RXE_MAX_QP_RD_ATOM = 128, diff --git a/drivers/infiniband/sw/rxe/rxe_qp.c b/drivers/infiniband/sw/rxe/rxe_qp.c index c3e60e4bde6e2a..44b2108253bd98 100644 --- a/drivers/infiniband/sw/rxe/rxe_qp.c +++ b/drivers/infiniband/sw/rxe/rxe_qp.c @@ -813,8 +813,7 @@ void rxe_qp_destroy(struct rxe_qp *qp) del_timer_sync(&qp->rnr_nak_timer); rxe_cleanup_task(&qp->req.task); - if (qp_type(qp) == IB_QPT_RC) - rxe_cleanup_task(&qp->comp.task); + rxe_cleanup_task(&qp->comp.task); /* flush out any receive wr's or pending requests */ __rxe_do_task(&qp->req.task); @@ -855,4 +854,5 @@ void rxe_qp_cleanup(void *arg) free_rd_atomic_resources(qp); kernel_sock_shutdown(qp->sk, SHUT_RDWR); + sock_release(qp->sk); } diff --git a/drivers/infiniband/sw/rxe/rxe_req.c b/drivers/infiniband/sw/rxe/rxe_req.c index 22bd9630dcd924..9f46be52335e7a 100644 --- a/drivers/infiniband/sw/rxe/rxe_req.c +++ b/drivers/infiniband/sw/rxe/rxe_req.c @@ -548,23 +548,23 @@ static void update_wqe_psn(struct rxe_qp *qp, static void save_state(struct rxe_send_wqe *wqe, struct rxe_qp *qp, struct rxe_send_wqe *rollback_wqe, - struct rxe_qp *rollback_qp) + u32 *rollback_psn) { rollback_wqe->state = wqe->state; rollback_wqe->first_psn = wqe->first_psn; rollback_wqe->last_psn = wqe->last_psn; - rollback_qp->req.psn = qp->req.psn; + *rollback_psn = qp->req.psn; } static void rollback_state(struct rxe_send_wqe *wqe, struct rxe_qp *qp, struct rxe_send_wqe *rollback_wqe, - struct rxe_qp *rollback_qp) + u32 rollback_psn) { wqe->state = rollback_wqe->state; wqe->first_psn = rollback_wqe->first_psn; wqe->last_psn = rollback_wqe->last_psn; - qp->req.psn = rollback_qp->req.psn; + qp->req.psn = rollback_psn; } static void update_state(struct rxe_qp *qp, struct rxe_send_wqe *wqe, @@ -593,8 +593,8 @@ int rxe_requester(void *arg) int mtu; int opcode; int ret; - struct rxe_qp rollback_qp; struct rxe_send_wqe rollback_wqe; + u32 rollback_psn; next_wqe: if (unlikely(!qp->valid || qp->req.state == QP_STATE_ERROR)) @@ -719,7 +719,7 @@ int rxe_requester(void *arg) * rxe_xmit_packet(). * Otherwise, completer might initiate an unjustified retry flow. */ - save_state(wqe, qp, &rollback_wqe, &rollback_qp); + save_state(wqe, qp, &rollback_wqe, &rollback_psn); update_wqe_state(qp, wqe, &pkt); update_wqe_psn(qp, wqe, &pkt, payload); ret = rxe_xmit_packet(to_rdev(qp->ibqp.device), qp, &pkt, skb); @@ -727,7 +727,7 @@ int rxe_requester(void *arg) qp->need_req_skb = 1; kfree_skb(skb); - rollback_state(wqe, qp, &rollback_wqe, &rollback_qp); + rollback_state(wqe, qp, &rollback_wqe, rollback_psn); if (ret == -EAGAIN) { rxe_run_task(&qp->req.task, 1); diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c index dd3d88adc00308..ccf62476356515 100644 --- a/drivers/infiniband/sw/rxe/rxe_resp.c +++ b/drivers/infiniband/sw/rxe/rxe_resp.c @@ -472,7 +472,7 @@ static enum resp_states check_rkey(struct rxe_qp *qp, goto err2; } - resid = mtu; + qp->resp.resid = mtu; } else { if (pktlen != resid) { state = RESPST_ERR_LENGTH; diff --git a/drivers/infiniband/ulp/ipoib/ipoib_cm.c b/drivers/infiniband/ulp/ipoib/ipoib_cm.c index 339a1eecdfe308..81a8080c18b358 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_cm.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_cm.c @@ -1054,8 +1054,6 @@ static struct ib_qp *ipoib_cm_create_tx_qp(struct net_device *dev, struct ipoib_ tx_qp = ib_create_qp(priv->pd, &attr); if (PTR_ERR(tx_qp) == -EINVAL) { - ipoib_warn(priv, "can't use GFP_NOIO for QPs on device %s, using GFP_KERNEL\n", - priv->ca->name); attr.create_flags &= ~IB_QP_CREATE_USE_GFP_NOIO; tx_qp = ib_create_qp(priv->pd, &attr); } diff --git a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c index 1909dd252c9406..fddff403d5d21b 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c @@ -575,8 +575,11 @@ void ipoib_mcast_join_task(struct work_struct *work) if (!test_bit(IPOIB_FLAG_OPER_UP, &priv->flags)) return; - if (ib_query_port(priv->ca, priv->port, &port_attr) || - port_attr.state != IB_PORT_ACTIVE) { + if (ib_query_port(priv->ca, priv->port, &port_attr)) { + ipoib_dbg(priv, "ib_query_port() failed\n"); + return; + } + if (port_attr.state != IB_PORT_ACTIVE) { ipoib_dbg(priv, "port state is not ACTIVE (state = %d) suspending join task\n", port_attr.state); return; diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.c b/drivers/infiniband/ulp/iser/iscsi_iser.c index 64b3d11dcf1ed5..140f3f354cf302 100644 --- a/drivers/infiniband/ulp/iser/iscsi_iser.c +++ b/drivers/infiniband/ulp/iser/iscsi_iser.c @@ -651,13 +651,6 @@ iscsi_iser_session_create(struct iscsi_endpoint *ep, SHOST_DIX_GUARD_CRC); } - /* - * Limit the sg_tablesize and max_sectors based on the device - * max fastreg page list length. - */ - shost->sg_tablesize = min_t(unsigned short, shost->sg_tablesize, - ib_conn->device->ib_device->attrs.max_fast_reg_page_list_len); - if (iscsi_host_add(shost, ib_conn->device->ib_device->dma_device)) { mutex_unlock(&iser_conn->state_mutex); diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c index d980fb458ad49c..e7dcf14a76e24a 100644 --- a/drivers/infiniband/ulp/srp/ib_srp.c +++ b/drivers/infiniband/ulp/srp/ib_srp.c @@ -366,6 +366,7 @@ static struct srp_fr_pool *srp_create_fr_pool(struct ib_device *device, struct srp_fr_desc *d; struct ib_mr *mr; int i, ret = -EINVAL; + enum ib_mr_type mr_type; if (pool_size <= 0) goto err; @@ -379,9 +380,13 @@ static struct srp_fr_pool *srp_create_fr_pool(struct ib_device *device, spin_lock_init(&pool->lock); INIT_LIST_HEAD(&pool->free_list); + if (device->attrs.device_cap_flags & IB_DEVICE_SG_GAPS_REG) + mr_type = IB_MR_TYPE_SG_GAPS; + else + mr_type = IB_MR_TYPE_MEM_REG; + for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) { - mr = ib_alloc_mr(pd, IB_MR_TYPE_MEM_REG, - max_page_list_len); + mr = ib_alloc_mr(pd, mr_type, max_page_list_len); if (IS_ERR(mr)) { ret = PTR_ERR(mr); goto destroy_pool; @@ -3678,6 +3683,12 @@ static int __init srp_init_module(void) indirect_sg_entries = cmd_sg_entries; } + if (indirect_sg_entries > SG_MAX_SEGMENTS) { + pr_warn("Clamping indirect_sg_entries to %u\n", + SG_MAX_SEGMENTS); + indirect_sg_entries = SG_MAX_SEGMENTS; + } + srp_remove_wq = create_workqueue("srp_remove"); if (!srp_remove_wq) { ret = -ENOMEM; diff --git a/drivers/input/Kconfig b/drivers/input/Kconfig index 6261874c07c9a7..34ffa0257b4b2c 100644 --- a/drivers/input/Kconfig +++ b/drivers/input/Kconfig @@ -187,6 +187,19 @@ config INPUT_APMPOWER To compile this driver as a module, choose M here: the module will be called apm-power. +config INPUT_KEYRESET + bool "Reset key" + depends on INPUT + select INPUT_KEYCOMBO + ---help--- + Say Y here if you want to reboot when some keys are pressed; + +config INPUT_KEYCOMBO + bool "Key combo" + depends on INPUT + ---help--- + Say Y here if you want to take action when some keys are pressed; + comment "Input Device Drivers" source "drivers/input/keyboard/Kconfig" diff --git a/drivers/input/Makefile b/drivers/input/Makefile index 595820bbabe95d..6a3281ca33060e 100644 --- a/drivers/input/Makefile +++ b/drivers/input/Makefile @@ -26,5 +26,7 @@ obj-$(CONFIG_INPUT_TOUCHSCREEN) += touchscreen/ obj-$(CONFIG_INPUT_MISC) += misc/ obj-$(CONFIG_INPUT_APMPOWER) += apm-power.o +obj-$(CONFIG_INPUT_KEYRESET) += keyreset.o +obj-$(CONFIG_INPUT_KEYCOMBO) += keycombo.o obj-$(CONFIG_RMI4_CORE) += rmi4/ diff --git a/drivers/input/joystick/xpad.c b/drivers/input/joystick/xpad.c index 83af17ad0f1f13..bbe15243b8e7ab 100644 --- a/drivers/input/joystick/xpad.c +++ b/drivers/input/joystick/xpad.c @@ -1376,6 +1376,12 @@ static int xpad_init_input(struct usb_xpad *xpad) input_dev->name = xpad->name; input_dev->phys = xpad->phys; usb_to_input_id(xpad->udev, &input_dev->id); + + if (xpad->xtype == XTYPE_XBOX360W) { + /* x360w controllers and the receiver have different ids */ + input_dev->id.product = 0x02a1; + } + input_dev->dev.parent = &xpad->intf->dev; input_set_drvdata(input_dev, xpad); diff --git a/drivers/input/keyboard/goldfish_events.c b/drivers/input/keyboard/goldfish_events.c index f6e643b589b616..c877e56a9bd576 100644 --- a/drivers/input/keyboard/goldfish_events.c +++ b/drivers/input/keyboard/goldfish_events.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -24,6 +25,8 @@ #include #include +#define GOLDFISH_MAX_FINGERS 5 + enum { REG_READ = 0x00, REG_SET_PAGE = 0x00, @@ -52,7 +55,21 @@ static irqreturn_t events_interrupt(int irq, void *dev_id) value = __raw_readl(edev->addr + REG_READ); input_event(edev->input, type, code, value); - input_sync(edev->input); + // Send an extra (EV_SYN, SYN_REPORT, 0x0) event + // if a key was pressed. Some keyboard device + // drivers may only send the EV_KEY event and + // not EV_SYN. + // Note that sending an extra SYN_REPORT is not + // necessary nor correct protocol with other + // devices such as touchscreens, which will send + // their own SYN_REPORT's when sufficient event + // information has been collected (e.g., for + // touchscreens, when pressure and X/Y coordinates + // have been received). Hence, we will only send + // this extra SYN_REPORT if type == EV_KEY. + if (type == EV_KEY) { + input_sync(edev->input); + } return IRQ_HANDLED; } @@ -154,6 +171,15 @@ static int events_probe(struct platform_device *pdev) input_dev->name = edev->name; input_dev->id.bustype = BUS_HOST; + // Set the Goldfish Device to be multi-touch. + // In the Ranchu kernel, there is multi-touch-specific + // code for handling ABS_MT_SLOT events. + // See drivers/input/input.c:input_handle_abs_event. + // If we do not issue input_mt_init_slots, + // the kernel will filter out needed ABS_MT_SLOT + // events when we touch the screen in more than one place, + // preventing multi-touch with more than one finger from working. + input_mt_init_slots(input_dev, GOLDFISH_MAX_FINGERS, 0); events_import_bits(edev, input_dev->evbit, EV_SYN, EV_MAX); events_import_bits(edev, input_dev->keybit, EV_KEY, KEY_MAX); diff --git a/drivers/input/keycombo.c b/drivers/input/keycombo.c new file mode 100644 index 00000000000000..2fba451b91d594 --- /dev/null +++ b/drivers/input/keycombo.c @@ -0,0 +1,261 @@ +/* drivers/input/keycombo.c + * + * Copyright (C) 2014 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include +#include +#include +#include +#include +#include + +struct keycombo_state { + struct input_handler input_handler; + unsigned long keybit[BITS_TO_LONGS(KEY_CNT)]; + unsigned long upbit[BITS_TO_LONGS(KEY_CNT)]; + unsigned long key[BITS_TO_LONGS(KEY_CNT)]; + spinlock_t lock; + struct workqueue_struct *wq; + int key_down_target; + int key_down; + int key_up; + struct delayed_work key_down_work; + int delay; + struct work_struct key_up_work; + void (*key_up_fn)(void *); + void (*key_down_fn)(void *); + void *priv; + int key_is_down; + struct wakeup_source combo_held_wake_source; + struct wakeup_source combo_up_wake_source; +}; + +static void do_key_down(struct work_struct *work) +{ + struct delayed_work *dwork = container_of(work, struct delayed_work, + work); + struct keycombo_state *state = container_of(dwork, + struct keycombo_state, key_down_work); + if (state->key_down_fn) + state->key_down_fn(state->priv); +} + +static void do_key_up(struct work_struct *work) +{ + struct keycombo_state *state = container_of(work, struct keycombo_state, + key_up_work); + if (state->key_up_fn) + state->key_up_fn(state->priv); + __pm_relax(&state->combo_up_wake_source); +} + +static void keycombo_event(struct input_handle *handle, unsigned int type, + unsigned int code, int value) +{ + unsigned long flags; + struct keycombo_state *state = handle->private; + + if (type != EV_KEY) + return; + + if (code >= KEY_MAX) + return; + + if (!test_bit(code, state->keybit)) + return; + + spin_lock_irqsave(&state->lock, flags); + if (!test_bit(code, state->key) == !value) + goto done; + __change_bit(code, state->key); + if (test_bit(code, state->upbit)) { + if (value) + state->key_up++; + else + state->key_up--; + } else { + if (value) + state->key_down++; + else + state->key_down--; + } + if (state->key_down == state->key_down_target && state->key_up == 0) { + __pm_stay_awake(&state->combo_held_wake_source); + state->key_is_down = 1; + if (queue_delayed_work(state->wq, &state->key_down_work, + state->delay)) + pr_debug("Key down work already queued!"); + } else if (state->key_is_down) { + if (!cancel_delayed_work(&state->key_down_work)) { + __pm_stay_awake(&state->combo_up_wake_source); + queue_work(state->wq, &state->key_up_work); + } + __pm_relax(&state->combo_held_wake_source); + state->key_is_down = 0; + } +done: + spin_unlock_irqrestore(&state->lock, flags); +} + +static int keycombo_connect(struct input_handler *handler, + struct input_dev *dev, + const struct input_device_id *id) +{ + int i; + int ret; + struct input_handle *handle; + struct keycombo_state *state = + container_of(handler, struct keycombo_state, input_handler); + for (i = 0; i < KEY_MAX; i++) { + if (test_bit(i, state->keybit) && test_bit(i, dev->keybit)) + break; + } + if (i == KEY_MAX) + return -ENODEV; + + handle = kzalloc(sizeof(*handle), GFP_KERNEL); + if (!handle) + return -ENOMEM; + + handle->dev = dev; + handle->handler = handler; + handle->name = KEYCOMBO_NAME; + handle->private = state; + + ret = input_register_handle(handle); + if (ret) + goto err_input_register_handle; + + ret = input_open_device(handle); + if (ret) + goto err_input_open_device; + + return 0; + +err_input_open_device: + input_unregister_handle(handle); +err_input_register_handle: + kfree(handle); + return ret; +} + +static void keycombo_disconnect(struct input_handle *handle) +{ + input_close_device(handle); + input_unregister_handle(handle); + kfree(handle); +} + +static const struct input_device_id keycombo_ids[] = { + { + .flags = INPUT_DEVICE_ID_MATCH_EVBIT, + .evbit = { BIT_MASK(EV_KEY) }, + }, + { }, +}; +MODULE_DEVICE_TABLE(input, keycombo_ids); + +static int keycombo_probe(struct platform_device *pdev) +{ + int ret; + int key, *keyp; + struct keycombo_state *state; + struct keycombo_platform_data *pdata = pdev->dev.platform_data; + + if (!pdata) + return -EINVAL; + + state = kzalloc(sizeof(*state), GFP_KERNEL); + if (!state) + return -ENOMEM; + + spin_lock_init(&state->lock); + keyp = pdata->keys_down; + while ((key = *keyp++)) { + if (key >= KEY_MAX) + continue; + state->key_down_target++; + __set_bit(key, state->keybit); + } + if (pdata->keys_up) { + keyp = pdata->keys_up; + while ((key = *keyp++)) { + if (key >= KEY_MAX) + continue; + __set_bit(key, state->keybit); + __set_bit(key, state->upbit); + } + } + + state->wq = alloc_ordered_workqueue("keycombo", 0); + if (!state->wq) + return -ENOMEM; + + state->priv = pdata->priv; + + if (pdata->key_down_fn) + state->key_down_fn = pdata->key_down_fn; + INIT_DELAYED_WORK(&state->key_down_work, do_key_down); + + if (pdata->key_up_fn) + state->key_up_fn = pdata->key_up_fn; + INIT_WORK(&state->key_up_work, do_key_up); + + wakeup_source_init(&state->combo_held_wake_source, "key combo"); + wakeup_source_init(&state->combo_up_wake_source, "key combo up"); + state->delay = msecs_to_jiffies(pdata->key_down_delay); + + state->input_handler.event = keycombo_event; + state->input_handler.connect = keycombo_connect; + state->input_handler.disconnect = keycombo_disconnect; + state->input_handler.name = KEYCOMBO_NAME; + state->input_handler.id_table = keycombo_ids; + ret = input_register_handler(&state->input_handler); + if (ret) { + kfree(state); + return ret; + } + platform_set_drvdata(pdev, state); + return 0; +} + +int keycombo_remove(struct platform_device *pdev) +{ + struct keycombo_state *state = platform_get_drvdata(pdev); + input_unregister_handler(&state->input_handler); + destroy_workqueue(state->wq); + kfree(state); + return 0; +} + + +struct platform_driver keycombo_driver = { + .driver.name = KEYCOMBO_NAME, + .probe = keycombo_probe, + .remove = keycombo_remove, +}; + +static int __init keycombo_init(void) +{ + return platform_driver_register(&keycombo_driver); +} + +static void __exit keycombo_exit(void) +{ + return platform_driver_unregister(&keycombo_driver); +} + +module_init(keycombo_init); +module_exit(keycombo_exit); diff --git a/drivers/input/keyreset.c b/drivers/input/keyreset.c new file mode 100644 index 00000000000000..7e5222aec7c1e5 --- /dev/null +++ b/drivers/input/keyreset.c @@ -0,0 +1,144 @@ +/* drivers/input/keyreset.c + * + * Copyright (C) 2014 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct keyreset_state { + int restart_requested; + int (*reset_fn)(void); + struct platform_device *pdev_child; + struct work_struct restart_work; +}; + +static void do_restart(struct work_struct *unused) +{ + orderly_reboot(); +} + +static void do_reset_fn(void *priv) +{ + struct keyreset_state *state = priv; + if (state->restart_requested) + panic("keyboard reset failed, %d", state->restart_requested); + if (state->reset_fn) { + state->restart_requested = state->reset_fn(); + } else { + pr_info("keyboard reset\n"); + schedule_work(&state->restart_work); + state->restart_requested = 1; + } +} + +static int keyreset_probe(struct platform_device *pdev) +{ + int ret = -ENOMEM; + struct keycombo_platform_data *pdata_child; + struct keyreset_platform_data *pdata = pdev->dev.platform_data; + int up_size = 0, down_size = 0, size; + int key, *keyp; + struct keyreset_state *state; + + if (!pdata) + return -EINVAL; + state = devm_kzalloc(&pdev->dev, sizeof(*state), GFP_KERNEL); + if (!state) + return -ENOMEM; + + state->pdev_child = platform_device_alloc(KEYCOMBO_NAME, + PLATFORM_DEVID_AUTO); + if (!state->pdev_child) + return -ENOMEM; + state->pdev_child->dev.parent = &pdev->dev; + INIT_WORK(&state->restart_work, do_restart); + + keyp = pdata->keys_down; + while ((key = *keyp++)) { + if (key >= KEY_MAX) + continue; + down_size++; + } + if (pdata->keys_up) { + keyp = pdata->keys_up; + while ((key = *keyp++)) { + if (key >= KEY_MAX) + continue; + up_size++; + } + } + size = sizeof(struct keycombo_platform_data) + + sizeof(int) * (down_size + 1); + pdata_child = devm_kzalloc(&pdev->dev, size, GFP_KERNEL); + if (!pdata_child) + goto error; + memcpy(pdata_child->keys_down, pdata->keys_down, + sizeof(int) * down_size); + if (up_size > 0) { + pdata_child->keys_up = devm_kzalloc(&pdev->dev, up_size + 1, + GFP_KERNEL); + if (!pdata_child->keys_up) + goto error; + memcpy(pdata_child->keys_up, pdata->keys_up, + sizeof(int) * up_size); + if (!pdata_child->keys_up) + goto error; + } + state->reset_fn = pdata->reset_fn; + pdata_child->key_down_fn = do_reset_fn; + pdata_child->priv = state; + pdata_child->key_down_delay = pdata->key_down_delay; + ret = platform_device_add_data(state->pdev_child, pdata_child, size); + if (ret) + goto error; + platform_set_drvdata(pdev, state); + return platform_device_add(state->pdev_child); +error: + platform_device_put(state->pdev_child); + return ret; +} + +int keyreset_remove(struct platform_device *pdev) +{ + struct keyreset_state *state = platform_get_drvdata(pdev); + platform_device_put(state->pdev_child); + return 0; +} + + +struct platform_driver keyreset_driver = { + .driver.name = KEYRESET_NAME, + .probe = keyreset_probe, + .remove = keyreset_remove, +}; + +static int __init keyreset_init(void) +{ + return platform_driver_register(&keyreset_driver); +} + +static void __exit keyreset_exit(void) +{ + return platform_driver_unregister(&keyreset_driver); +} + +module_init(keyreset_init); +module_exit(keyreset_exit); diff --git a/drivers/input/misc/Kconfig b/drivers/input/misc/Kconfig index 7ffb614ce5664c..94360fe63f41a9 100644 --- a/drivers/input/misc/Kconfig +++ b/drivers/input/misc/Kconfig @@ -367,6 +367,17 @@ config INPUT_ATI_REMOTE2 To compile this driver as a module, choose M here: the module will be called ati_remote2. +config INPUT_KEYCHORD + tristate "Key chord input driver support" + help + Say Y here if you want to enable the key chord driver + accessible at /dev/keychord. This driver can be used + for receiving notifications when client specified key + combinations are pressed. + + To compile this driver as a module, choose M here: the + module will be called keychord. + config INPUT_KEYSPAN_REMOTE tristate "Keyspan DMR USB remote control" depends on USB_ARCH_HAS_HCD @@ -535,6 +546,11 @@ config INPUT_SGI_BTNS To compile this driver as a module, choose M here: the module will be called sgi_btns. +config INPUT_GPIO + tristate "GPIO driver support" + help + Say Y here if you want to support gpio based keys, wheels etc... + config HP_SDC_RTC tristate "HP SDC Real Time Clock" depends on (GSC || HP300) && SERIO diff --git a/drivers/input/misc/Makefile b/drivers/input/misc/Makefile index 0b6d025f0487a5..64bf231faf8c77 100644 --- a/drivers/input/misc/Makefile +++ b/drivers/input/misc/Makefile @@ -36,10 +36,12 @@ obj-$(CONFIG_INPUT_GP2A) += gp2ap002a00f.o obj-$(CONFIG_INPUT_GPIO_BEEPER) += gpio-beeper.o obj-$(CONFIG_INPUT_GPIO_TILT_POLLED) += gpio_tilt_polled.o obj-$(CONFIG_INPUT_GPIO_DECODER) += gpio_decoder.o +obj-$(CONFIG_INPUT_GPIO) += gpio_event.o gpio_matrix.o gpio_input.o gpio_output.o gpio_axis.o obj-$(CONFIG_INPUT_HISI_POWERKEY) += hisi_powerkey.o obj-$(CONFIG_HP_SDC_RTC) += hp_sdc_rtc.o obj-$(CONFIG_INPUT_IMS_PCU) += ims-pcu.o obj-$(CONFIG_INPUT_IXP4XX_BEEPER) += ixp4xx-beeper.o +obj-$(CONFIG_INPUT_KEYCHORD) += keychord.o obj-$(CONFIG_INPUT_KEYSPAN_REMOTE) += keyspan_remote.o obj-$(CONFIG_INPUT_KXTJ9) += kxtj9.o obj-$(CONFIG_INPUT_M68K_BEEP) += m68kspkr.o diff --git a/drivers/input/misc/drv260x.c b/drivers/input/misc/drv260x.c index 2adfd86c869a53..930424e55439d4 100644 --- a/drivers/input/misc/drv260x.c +++ b/drivers/input/misc/drv260x.c @@ -592,7 +592,6 @@ static int drv260x_probe(struct i2c_client *client, } haptics->input_dev->name = "drv260x:haptics"; - haptics->input_dev->dev.parent = client->dev.parent; haptics->input_dev->close = drv260x_close; input_set_drvdata(haptics->input_dev, haptics); input_set_capability(haptics->input_dev, EV_FF, FF_RUMBLE); diff --git a/drivers/input/misc/gpio_axis.c b/drivers/input/misc/gpio_axis.c new file mode 100644 index 00000000000000..0acf4a576f53a2 --- /dev/null +++ b/drivers/input/misc/gpio_axis.c @@ -0,0 +1,192 @@ +/* drivers/input/misc/gpio_axis.c + * + * Copyright (C) 2007 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include +#include +#include +#include + +struct gpio_axis_state { + struct gpio_event_input_devs *input_devs; + struct gpio_event_axis_info *info; + uint32_t pos; +}; + +uint16_t gpio_axis_4bit_gray_map_table[] = { + [0x0] = 0x0, [0x1] = 0x1, /* 0000 0001 */ + [0x3] = 0x2, [0x2] = 0x3, /* 0011 0010 */ + [0x6] = 0x4, [0x7] = 0x5, /* 0110 0111 */ + [0x5] = 0x6, [0x4] = 0x7, /* 0101 0100 */ + [0xc] = 0x8, [0xd] = 0x9, /* 1100 1101 */ + [0xf] = 0xa, [0xe] = 0xb, /* 1111 1110 */ + [0xa] = 0xc, [0xb] = 0xd, /* 1010 1011 */ + [0x9] = 0xe, [0x8] = 0xf, /* 1001 1000 */ +}; +uint16_t gpio_axis_4bit_gray_map(struct gpio_event_axis_info *info, uint16_t in) +{ + return gpio_axis_4bit_gray_map_table[in]; +} + +uint16_t gpio_axis_5bit_singletrack_map_table[] = { + [0x10] = 0x00, [0x14] = 0x01, [0x1c] = 0x02, /* 10000 10100 11100 */ + [0x1e] = 0x03, [0x1a] = 0x04, [0x18] = 0x05, /* 11110 11010 11000 */ + [0x08] = 0x06, [0x0a] = 0x07, [0x0e] = 0x08, /* 01000 01010 01110 */ + [0x0f] = 0x09, [0x0d] = 0x0a, [0x0c] = 0x0b, /* 01111 01101 01100 */ + [0x04] = 0x0c, [0x05] = 0x0d, [0x07] = 0x0e, /* 00100 00101 00111 */ + [0x17] = 0x0f, [0x16] = 0x10, [0x06] = 0x11, /* 10111 10110 00110 */ + [0x02] = 0x12, [0x12] = 0x13, [0x13] = 0x14, /* 00010 10010 10011 */ + [0x1b] = 0x15, [0x0b] = 0x16, [0x03] = 0x17, /* 11011 01011 00011 */ + [0x01] = 0x18, [0x09] = 0x19, [0x19] = 0x1a, /* 00001 01001 11001 */ + [0x1d] = 0x1b, [0x15] = 0x1c, [0x11] = 0x1d, /* 11101 10101 10001 */ +}; +uint16_t gpio_axis_5bit_singletrack_map( + struct gpio_event_axis_info *info, uint16_t in) +{ + return gpio_axis_5bit_singletrack_map_table[in]; +} + +static void gpio_event_update_axis(struct gpio_axis_state *as, int report) +{ + struct gpio_event_axis_info *ai = as->info; + int i; + int change; + uint16_t state = 0; + uint16_t pos; + uint16_t old_pos = as->pos; + for (i = ai->count - 1; i >= 0; i--) + state = (state << 1) | gpio_get_value(ai->gpio[i]); + pos = ai->map(ai, state); + if (ai->flags & GPIOEAF_PRINT_RAW) + pr_info("axis %d-%d raw %x, pos %d -> %d\n", + ai->type, ai->code, state, old_pos, pos); + if (report && pos != old_pos) { + if (ai->type == EV_REL) { + change = (ai->decoded_size + pos - old_pos) % + ai->decoded_size; + if (change > ai->decoded_size / 2) + change -= ai->decoded_size; + if (change == ai->decoded_size / 2) { + if (ai->flags & GPIOEAF_PRINT_EVENT) + pr_info("axis %d-%d unknown direction, " + "pos %d -> %d\n", ai->type, + ai->code, old_pos, pos); + change = 0; /* no closest direction */ + } + if (ai->flags & GPIOEAF_PRINT_EVENT) + pr_info("axis %d-%d change %d\n", + ai->type, ai->code, change); + input_report_rel(as->input_devs->dev[ai->dev], + ai->code, change); + } else { + if (ai->flags & GPIOEAF_PRINT_EVENT) + pr_info("axis %d-%d now %d\n", + ai->type, ai->code, pos); + input_event(as->input_devs->dev[ai->dev], + ai->type, ai->code, pos); + } + input_sync(as->input_devs->dev[ai->dev]); + } + as->pos = pos; +} + +static irqreturn_t gpio_axis_irq_handler(int irq, void *dev_id) +{ + struct gpio_axis_state *as = dev_id; + gpio_event_update_axis(as, 1); + return IRQ_HANDLED; +} + +int gpio_event_axis_func(struct gpio_event_input_devs *input_devs, + struct gpio_event_info *info, void **data, int func) +{ + int ret; + int i; + int irq; + struct gpio_event_axis_info *ai; + struct gpio_axis_state *as; + + ai = container_of(info, struct gpio_event_axis_info, info); + if (func == GPIO_EVENT_FUNC_SUSPEND) { + for (i = 0; i < ai->count; i++) + disable_irq(gpio_to_irq(ai->gpio[i])); + return 0; + } + if (func == GPIO_EVENT_FUNC_RESUME) { + for (i = 0; i < ai->count; i++) + enable_irq(gpio_to_irq(ai->gpio[i])); + return 0; + } + + if (func == GPIO_EVENT_FUNC_INIT) { + *data = as = kmalloc(sizeof(*as), GFP_KERNEL); + if (as == NULL) { + ret = -ENOMEM; + goto err_alloc_axis_state_failed; + } + as->input_devs = input_devs; + as->info = ai; + if (ai->dev >= input_devs->count) { + pr_err("gpio_event_axis: bad device index %d >= %d " + "for %d:%d\n", ai->dev, input_devs->count, + ai->type, ai->code); + ret = -EINVAL; + goto err_bad_device_index; + } + + input_set_capability(input_devs->dev[ai->dev], + ai->type, ai->code); + if (ai->type == EV_ABS) { + input_set_abs_params(input_devs->dev[ai->dev], ai->code, + 0, ai->decoded_size - 1, 0, 0); + } + for (i = 0; i < ai->count; i++) { + ret = gpio_request(ai->gpio[i], "gpio_event_axis"); + if (ret < 0) + goto err_request_gpio_failed; + ret = gpio_direction_input(ai->gpio[i]); + if (ret < 0) + goto err_gpio_direction_input_failed; + ret = irq = gpio_to_irq(ai->gpio[i]); + if (ret < 0) + goto err_get_irq_num_failed; + ret = request_irq(irq, gpio_axis_irq_handler, + IRQF_TRIGGER_RISING | + IRQF_TRIGGER_FALLING, + "gpio_event_axis", as); + if (ret < 0) + goto err_request_irq_failed; + } + gpio_event_update_axis(as, 0); + return 0; + } + + ret = 0; + as = *data; + for (i = ai->count - 1; i >= 0; i--) { + free_irq(gpio_to_irq(ai->gpio[i]), as); +err_request_irq_failed: +err_get_irq_num_failed: +err_gpio_direction_input_failed: + gpio_free(ai->gpio[i]); +err_request_gpio_failed: + ; + } +err_bad_device_index: + kfree(as); + *data = NULL; +err_alloc_axis_state_failed: + return ret; +} diff --git a/drivers/input/misc/gpio_event.c b/drivers/input/misc/gpio_event.c new file mode 100644 index 00000000000000..90f07eba3ce9e9 --- /dev/null +++ b/drivers/input/misc/gpio_event.c @@ -0,0 +1,228 @@ +/* drivers/input/misc/gpio_event.c + * + * Copyright (C) 2007 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include +#include +#include +#include +#include + +struct gpio_event { + struct gpio_event_input_devs *input_devs; + const struct gpio_event_platform_data *info; + void *state[0]; +}; + +static int gpio_input_event( + struct input_dev *dev, unsigned int type, unsigned int code, int value) +{ + int i; + int devnr; + int ret = 0; + int tmp_ret; + struct gpio_event_info **ii; + struct gpio_event *ip = input_get_drvdata(dev); + + for (devnr = 0; devnr < ip->input_devs->count; devnr++) + if (ip->input_devs->dev[devnr] == dev) + break; + if (devnr == ip->input_devs->count) { + pr_err("gpio_input_event: unknown device %p\n", dev); + return -EIO; + } + + for (i = 0, ii = ip->info->info; i < ip->info->info_count; i++, ii++) { + if ((*ii)->event) { + tmp_ret = (*ii)->event(ip->input_devs, *ii, + &ip->state[i], + devnr, type, code, value); + if (tmp_ret) + ret = tmp_ret; + } + } + return ret; +} + +static int gpio_event_call_all_func(struct gpio_event *ip, int func) +{ + int i; + int ret; + struct gpio_event_info **ii; + + if (func == GPIO_EVENT_FUNC_INIT || func == GPIO_EVENT_FUNC_RESUME) { + ii = ip->info->info; + for (i = 0; i < ip->info->info_count; i++, ii++) { + if ((*ii)->func == NULL) { + ret = -ENODEV; + pr_err("gpio_event_probe: Incomplete pdata, " + "no function\n"); + goto err_no_func; + } + if (func == GPIO_EVENT_FUNC_RESUME && (*ii)->no_suspend) + continue; + ret = (*ii)->func(ip->input_devs, *ii, &ip->state[i], + func); + if (ret) { + pr_err("gpio_event_probe: function failed\n"); + goto err_func_failed; + } + } + return 0; + } + + ret = 0; + i = ip->info->info_count; + ii = ip->info->info + i; + while (i > 0) { + i--; + ii--; + if ((func & ~1) == GPIO_EVENT_FUNC_SUSPEND && (*ii)->no_suspend) + continue; + (*ii)->func(ip->input_devs, *ii, &ip->state[i], func & ~1); +err_func_failed: +err_no_func: + ; + } + return ret; +} + +static void __maybe_unused gpio_event_suspend(struct gpio_event *ip) +{ + gpio_event_call_all_func(ip, GPIO_EVENT_FUNC_SUSPEND); + if (ip->info->power) + ip->info->power(ip->info, 0); +} + +static void __maybe_unused gpio_event_resume(struct gpio_event *ip) +{ + if (ip->info->power) + ip->info->power(ip->info, 1); + gpio_event_call_all_func(ip, GPIO_EVENT_FUNC_RESUME); +} + +static int gpio_event_probe(struct platform_device *pdev) +{ + int err; + struct gpio_event *ip; + struct gpio_event_platform_data *event_info; + int dev_count = 1; + int i; + int registered = 0; + + event_info = pdev->dev.platform_data; + if (event_info == NULL) { + pr_err("gpio_event_probe: No pdata\n"); + return -ENODEV; + } + if ((!event_info->name && !event_info->names[0]) || + !event_info->info || !event_info->info_count) { + pr_err("gpio_event_probe: Incomplete pdata\n"); + return -ENODEV; + } + if (!event_info->name) + while (event_info->names[dev_count]) + dev_count++; + ip = kzalloc(sizeof(*ip) + + sizeof(ip->state[0]) * event_info->info_count + + sizeof(*ip->input_devs) + + sizeof(ip->input_devs->dev[0]) * dev_count, GFP_KERNEL); + if (ip == NULL) { + err = -ENOMEM; + pr_err("gpio_event_probe: Failed to allocate private data\n"); + goto err_kp_alloc_failed; + } + ip->input_devs = (void*)&ip->state[event_info->info_count]; + platform_set_drvdata(pdev, ip); + + for (i = 0; i < dev_count; i++) { + struct input_dev *input_dev = input_allocate_device(); + if (input_dev == NULL) { + err = -ENOMEM; + pr_err("gpio_event_probe: " + "Failed to allocate input device\n"); + goto err_input_dev_alloc_failed; + } + input_set_drvdata(input_dev, ip); + input_dev->name = event_info->name ? + event_info->name : event_info->names[i]; + input_dev->event = gpio_input_event; + ip->input_devs->dev[i] = input_dev; + } + ip->input_devs->count = dev_count; + ip->info = event_info; + if (event_info->power) + ip->info->power(ip->info, 1); + + err = gpio_event_call_all_func(ip, GPIO_EVENT_FUNC_INIT); + if (err) + goto err_call_all_func_failed; + + for (i = 0; i < dev_count; i++) { + err = input_register_device(ip->input_devs->dev[i]); + if (err) { + pr_err("gpio_event_probe: Unable to register %s " + "input device\n", ip->input_devs->dev[i]->name); + goto err_input_register_device_failed; + } + registered++; + } + + return 0; + +err_input_register_device_failed: + gpio_event_call_all_func(ip, GPIO_EVENT_FUNC_UNINIT); +err_call_all_func_failed: + if (event_info->power) + ip->info->power(ip->info, 0); + for (i = 0; i < registered; i++) + input_unregister_device(ip->input_devs->dev[i]); + for (i = dev_count - 1; i >= registered; i--) { + input_free_device(ip->input_devs->dev[i]); +err_input_dev_alloc_failed: + ; + } + kfree(ip); +err_kp_alloc_failed: + return err; +} + +static int gpio_event_remove(struct platform_device *pdev) +{ + struct gpio_event *ip = platform_get_drvdata(pdev); + int i; + + gpio_event_call_all_func(ip, GPIO_EVENT_FUNC_UNINIT); + if (ip->info->power) + ip->info->power(ip->info, 0); + for (i = 0; i < ip->input_devs->count; i++) + input_unregister_device(ip->input_devs->dev[i]); + kfree(ip); + return 0; +} + +static struct platform_driver gpio_event_driver = { + .probe = gpio_event_probe, + .remove = gpio_event_remove, + .driver = { + .name = GPIO_EVENT_DEV_NAME, + }, +}; + +module_platform_driver(gpio_event_driver); + +MODULE_DESCRIPTION("GPIO Event Driver"); +MODULE_LICENSE("GPL"); + diff --git a/drivers/input/misc/gpio_input.c b/drivers/input/misc/gpio_input.c new file mode 100644 index 00000000000000..eefd02725aff40 --- /dev/null +++ b/drivers/input/misc/gpio_input.c @@ -0,0 +1,390 @@ +/* drivers/input/misc/gpio_input.c + * + * Copyright (C) 2007 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +enum { + DEBOUNCE_UNSTABLE = BIT(0), /* Got irq, while debouncing */ + DEBOUNCE_PRESSED = BIT(1), + DEBOUNCE_NOTPRESSED = BIT(2), + DEBOUNCE_WAIT_IRQ = BIT(3), /* Stable irq state */ + DEBOUNCE_POLL = BIT(4), /* Stable polling state */ + + DEBOUNCE_UNKNOWN = + DEBOUNCE_PRESSED | DEBOUNCE_NOTPRESSED, +}; + +struct gpio_key_state { + struct gpio_input_state *ds; + uint8_t debounce; +}; + +struct gpio_input_state { + struct gpio_event_input_devs *input_devs; + const struct gpio_event_input_info *info; + struct hrtimer timer; + int use_irq; + int debounce_count; + spinlock_t irq_lock; + struct wakeup_source *ws; + struct gpio_key_state key_state[0]; +}; + +static enum hrtimer_restart gpio_event_input_timer_func(struct hrtimer *timer) +{ + int i; + int pressed; + struct gpio_input_state *ds = + container_of(timer, struct gpio_input_state, timer); + unsigned gpio_flags = ds->info->flags; + unsigned npolarity; + int nkeys = ds->info->keymap_size; + const struct gpio_event_direct_entry *key_entry; + struct gpio_key_state *key_state; + unsigned long irqflags; + uint8_t debounce; + bool sync_needed; + +#if 0 + key_entry = kp->keys_info->keymap; + key_state = kp->key_state; + for (i = 0; i < nkeys; i++, key_entry++, key_state++) + pr_info("gpio_read_detect_status %d %d\n", key_entry->gpio, + gpio_read_detect_status(key_entry->gpio)); +#endif + key_entry = ds->info->keymap; + key_state = ds->key_state; + sync_needed = false; + spin_lock_irqsave(&ds->irq_lock, irqflags); + for (i = 0; i < nkeys; i++, key_entry++, key_state++) { + debounce = key_state->debounce; + if (debounce & DEBOUNCE_WAIT_IRQ) + continue; + if (key_state->debounce & DEBOUNCE_UNSTABLE) { + debounce = key_state->debounce = DEBOUNCE_UNKNOWN; + enable_irq(gpio_to_irq(key_entry->gpio)); + if (gpio_flags & GPIOEDF_PRINT_KEY_UNSTABLE) + pr_info("gpio_keys_scan_keys: key %x-%x, %d " + "(%d) continue debounce\n", + ds->info->type, key_entry->code, + i, key_entry->gpio); + } + npolarity = !(gpio_flags & GPIOEDF_ACTIVE_HIGH); + pressed = gpio_get_value(key_entry->gpio) ^ npolarity; + if (debounce & DEBOUNCE_POLL) { + if (pressed == !(debounce & DEBOUNCE_PRESSED)) { + ds->debounce_count++; + key_state->debounce = DEBOUNCE_UNKNOWN; + if (gpio_flags & GPIOEDF_PRINT_KEY_DEBOUNCE) + pr_info("gpio_keys_scan_keys: key %x-" + "%x, %d (%d) start debounce\n", + ds->info->type, key_entry->code, + i, key_entry->gpio); + } + continue; + } + if (pressed && (debounce & DEBOUNCE_NOTPRESSED)) { + if (gpio_flags & GPIOEDF_PRINT_KEY_DEBOUNCE) + pr_info("gpio_keys_scan_keys: key %x-%x, %d " + "(%d) debounce pressed 1\n", + ds->info->type, key_entry->code, + i, key_entry->gpio); + key_state->debounce = DEBOUNCE_PRESSED; + continue; + } + if (!pressed && (debounce & DEBOUNCE_PRESSED)) { + if (gpio_flags & GPIOEDF_PRINT_KEY_DEBOUNCE) + pr_info("gpio_keys_scan_keys: key %x-%x, %d " + "(%d) debounce pressed 0\n", + ds->info->type, key_entry->code, + i, key_entry->gpio); + key_state->debounce = DEBOUNCE_NOTPRESSED; + continue; + } + /* key is stable */ + ds->debounce_count--; + if (ds->use_irq) + key_state->debounce |= DEBOUNCE_WAIT_IRQ; + else + key_state->debounce |= DEBOUNCE_POLL; + if (gpio_flags & GPIOEDF_PRINT_KEYS) + pr_info("gpio_keys_scan_keys: key %x-%x, %d (%d) " + "changed to %d\n", ds->info->type, + key_entry->code, i, key_entry->gpio, pressed); + input_event(ds->input_devs->dev[key_entry->dev], ds->info->type, + key_entry->code, pressed); + sync_needed = true; + } + if (sync_needed) { + for (i = 0; i < ds->input_devs->count; i++) + input_sync(ds->input_devs->dev[i]); + } + +#if 0 + key_entry = kp->keys_info->keymap; + key_state = kp->key_state; + for (i = 0; i < nkeys; i++, key_entry++, key_state++) { + pr_info("gpio_read_detect_status %d %d\n", key_entry->gpio, + gpio_read_detect_status(key_entry->gpio)); + } +#endif + + if (ds->debounce_count) + hrtimer_start(timer, ds->info->debounce_time, HRTIMER_MODE_REL); + else if (!ds->use_irq) + hrtimer_start(timer, ds->info->poll_time, HRTIMER_MODE_REL); + else + __pm_relax(ds->ws); + + spin_unlock_irqrestore(&ds->irq_lock, irqflags); + + return HRTIMER_NORESTART; +} + +static irqreturn_t gpio_event_input_irq_handler(int irq, void *dev_id) +{ + struct gpio_key_state *ks = dev_id; + struct gpio_input_state *ds = ks->ds; + int keymap_index = ks - ds->key_state; + const struct gpio_event_direct_entry *key_entry; + unsigned long irqflags; + int pressed; + + if (!ds->use_irq) + return IRQ_HANDLED; + + key_entry = &ds->info->keymap[keymap_index]; + + if (ds->info->debounce_time.tv64) { + spin_lock_irqsave(&ds->irq_lock, irqflags); + if (ks->debounce & DEBOUNCE_WAIT_IRQ) { + ks->debounce = DEBOUNCE_UNKNOWN; + if (ds->debounce_count++ == 0) { + __pm_stay_awake(ds->ws); + hrtimer_start( + &ds->timer, ds->info->debounce_time, + HRTIMER_MODE_REL); + } + if (ds->info->flags & GPIOEDF_PRINT_KEY_DEBOUNCE) + pr_info("gpio_event_input_irq_handler: " + "key %x-%x, %d (%d) start debounce\n", + ds->info->type, key_entry->code, + keymap_index, key_entry->gpio); + } else { + disable_irq_nosync(irq); + ks->debounce = DEBOUNCE_UNSTABLE; + } + spin_unlock_irqrestore(&ds->irq_lock, irqflags); + } else { + pressed = gpio_get_value(key_entry->gpio) ^ + !(ds->info->flags & GPIOEDF_ACTIVE_HIGH); + if (ds->info->flags & GPIOEDF_PRINT_KEYS) + pr_info("gpio_event_input_irq_handler: key %x-%x, %d " + "(%d) changed to %d\n", + ds->info->type, key_entry->code, keymap_index, + key_entry->gpio, pressed); + input_event(ds->input_devs->dev[key_entry->dev], ds->info->type, + key_entry->code, pressed); + input_sync(ds->input_devs->dev[key_entry->dev]); + } + return IRQ_HANDLED; +} + +static int gpio_event_input_request_irqs(struct gpio_input_state *ds) +{ + int i; + int err; + unsigned int irq; + unsigned long req_flags = IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING; + + for (i = 0; i < ds->info->keymap_size; i++) { + err = irq = gpio_to_irq(ds->info->keymap[i].gpio); + if (err < 0) + goto err_gpio_get_irq_num_failed; + err = request_irq(irq, gpio_event_input_irq_handler, + req_flags, "gpio_keys", &ds->key_state[i]); + if (err) { + pr_err("gpio_event_input_request_irqs: request_irq " + "failed for input %d, irq %d\n", + ds->info->keymap[i].gpio, irq); + goto err_request_irq_failed; + } + if (ds->info->info.no_suspend) { + err = enable_irq_wake(irq); + if (err) { + pr_err("gpio_event_input_request_irqs: " + "enable_irq_wake failed for input %d, " + "irq %d\n", + ds->info->keymap[i].gpio, irq); + goto err_enable_irq_wake_failed; + } + } + } + return 0; + + for (i = ds->info->keymap_size - 1; i >= 0; i--) { + irq = gpio_to_irq(ds->info->keymap[i].gpio); + if (ds->info->info.no_suspend) + disable_irq_wake(irq); +err_enable_irq_wake_failed: + free_irq(irq, &ds->key_state[i]); +err_request_irq_failed: +err_gpio_get_irq_num_failed: + ; + } + return err; +} + +int gpio_event_input_func(struct gpio_event_input_devs *input_devs, + struct gpio_event_info *info, void **data, int func) +{ + int ret; + int i; + unsigned long irqflags; + struct gpio_event_input_info *di; + struct gpio_input_state *ds = *data; + char *wlname; + + di = container_of(info, struct gpio_event_input_info, info); + + if (func == GPIO_EVENT_FUNC_SUSPEND) { + if (ds->use_irq) + for (i = 0; i < di->keymap_size; i++) + disable_irq(gpio_to_irq(di->keymap[i].gpio)); + hrtimer_cancel(&ds->timer); + return 0; + } + if (func == GPIO_EVENT_FUNC_RESUME) { + spin_lock_irqsave(&ds->irq_lock, irqflags); + if (ds->use_irq) + for (i = 0; i < di->keymap_size; i++) + enable_irq(gpio_to_irq(di->keymap[i].gpio)); + hrtimer_start(&ds->timer, ktime_set(0, 0), HRTIMER_MODE_REL); + spin_unlock_irqrestore(&ds->irq_lock, irqflags); + return 0; + } + + if (func == GPIO_EVENT_FUNC_INIT) { + if (ktime_to_ns(di->poll_time) <= 0) + di->poll_time = ktime_set(0, 20 * NSEC_PER_MSEC); + + *data = ds = kzalloc(sizeof(*ds) + sizeof(ds->key_state[0]) * + di->keymap_size, GFP_KERNEL); + if (ds == NULL) { + ret = -ENOMEM; + pr_err("gpio_event_input_func: " + "Failed to allocate private data\n"); + goto err_ds_alloc_failed; + } + ds->debounce_count = di->keymap_size; + ds->input_devs = input_devs; + ds->info = di; + wlname = kasprintf(GFP_KERNEL, "gpio_input:%s%s", + input_devs->dev[0]->name, + (input_devs->count > 1) ? "..." : ""); + + ds->ws = wakeup_source_register(wlname); + kfree(wlname); + if (!ds->ws) { + ret = -ENOMEM; + pr_err("gpio_event_input_func: " + "Failed to allocate wakeup source\n"); + goto err_ws_failed; + } + + spin_lock_init(&ds->irq_lock); + + for (i = 0; i < di->keymap_size; i++) { + int dev = di->keymap[i].dev; + if (dev >= input_devs->count) { + pr_err("gpio_event_input_func: bad device " + "index %d >= %d for key code %d\n", + dev, input_devs->count, + di->keymap[i].code); + ret = -EINVAL; + goto err_bad_keymap; + } + input_set_capability(input_devs->dev[dev], di->type, + di->keymap[i].code); + ds->key_state[i].ds = ds; + ds->key_state[i].debounce = DEBOUNCE_UNKNOWN; + } + + for (i = 0; i < di->keymap_size; i++) { + ret = gpio_request(di->keymap[i].gpio, "gpio_kp_in"); + if (ret) { + pr_err("gpio_event_input_func: gpio_request " + "failed for %d\n", di->keymap[i].gpio); + goto err_gpio_request_failed; + } + ret = gpio_direction_input(di->keymap[i].gpio); + if (ret) { + pr_err("gpio_event_input_func: " + "gpio_direction_input failed for %d\n", + di->keymap[i].gpio); + goto err_gpio_configure_failed; + } + } + + ret = gpio_event_input_request_irqs(ds); + + spin_lock_irqsave(&ds->irq_lock, irqflags); + ds->use_irq = ret == 0; + + pr_info("GPIO Input Driver: Start gpio inputs for %s%s in %s " + "mode\n", input_devs->dev[0]->name, + (input_devs->count > 1) ? "..." : "", + ret == 0 ? "interrupt" : "polling"); + + hrtimer_init(&ds->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + ds->timer.function = gpio_event_input_timer_func; + hrtimer_start(&ds->timer, ktime_set(0, 0), HRTIMER_MODE_REL); + spin_unlock_irqrestore(&ds->irq_lock, irqflags); + return 0; + } + + ret = 0; + spin_lock_irqsave(&ds->irq_lock, irqflags); + hrtimer_cancel(&ds->timer); + if (ds->use_irq) { + for (i = di->keymap_size - 1; i >= 0; i--) { + int irq = gpio_to_irq(di->keymap[i].gpio); + if (ds->info->info.no_suspend) + disable_irq_wake(irq); + free_irq(irq, &ds->key_state[i]); + } + } + spin_unlock_irqrestore(&ds->irq_lock, irqflags); + + for (i = di->keymap_size - 1; i >= 0; i--) { +err_gpio_configure_failed: + gpio_free(di->keymap[i].gpio); +err_gpio_request_failed: + ; + } +err_bad_keymap: + wakeup_source_unregister(ds->ws); +err_ws_failed: + kfree(ds); +err_ds_alloc_failed: + return ret; +} diff --git a/drivers/input/misc/gpio_matrix.c b/drivers/input/misc/gpio_matrix.c new file mode 100644 index 00000000000000..08769dd88f56a9 --- /dev/null +++ b/drivers/input/misc/gpio_matrix.c @@ -0,0 +1,440 @@ +/* drivers/input/misc/gpio_matrix.c + * + * Copyright (C) 2007 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include +#include +#include +#include +#include + +struct gpio_kp { + struct gpio_event_input_devs *input_devs; + struct gpio_event_matrix_info *keypad_info; + struct hrtimer timer; + struct wakeup_source wake_src; + int current_output; + unsigned int use_irq:1; + unsigned int key_state_changed:1; + unsigned int last_key_state_changed:1; + unsigned int some_keys_pressed:2; + unsigned int disabled_irq:1; + unsigned long keys_pressed[0]; +}; + +static void clear_phantom_key(struct gpio_kp *kp, int out, int in) +{ + struct gpio_event_matrix_info *mi = kp->keypad_info; + int key_index = out * mi->ninputs + in; + unsigned short keyentry = mi->keymap[key_index]; + unsigned short keycode = keyentry & MATRIX_KEY_MASK; + unsigned short dev = keyentry >> MATRIX_CODE_BITS; + + if (!test_bit(keycode, kp->input_devs->dev[dev]->key)) { + if (mi->flags & GPIOKPF_PRINT_PHANTOM_KEYS) + pr_info("gpiomatrix: phantom key %x, %d-%d (%d-%d) " + "cleared\n", keycode, out, in, + mi->output_gpios[out], mi->input_gpios[in]); + __clear_bit(key_index, kp->keys_pressed); + } else { + if (mi->flags & GPIOKPF_PRINT_PHANTOM_KEYS) + pr_info("gpiomatrix: phantom key %x, %d-%d (%d-%d) " + "not cleared\n", keycode, out, in, + mi->output_gpios[out], mi->input_gpios[in]); + } +} + +static int restore_keys_for_input(struct gpio_kp *kp, int out, int in) +{ + int rv = 0; + int key_index; + + key_index = out * kp->keypad_info->ninputs + in; + while (out < kp->keypad_info->noutputs) { + if (test_bit(key_index, kp->keys_pressed)) { + rv = 1; + clear_phantom_key(kp, out, in); + } + key_index += kp->keypad_info->ninputs; + out++; + } + return rv; +} + +static void remove_phantom_keys(struct gpio_kp *kp) +{ + int out, in, inp; + int key_index; + + if (kp->some_keys_pressed < 3) + return; + + for (out = 0; out < kp->keypad_info->noutputs; out++) { + inp = -1; + key_index = out * kp->keypad_info->ninputs; + for (in = 0; in < kp->keypad_info->ninputs; in++, key_index++) { + if (test_bit(key_index, kp->keys_pressed)) { + if (inp == -1) { + inp = in; + continue; + } + if (inp >= 0) { + if (!restore_keys_for_input(kp, out + 1, + inp)) + break; + clear_phantom_key(kp, out, inp); + inp = -2; + } + restore_keys_for_input(kp, out, in); + } + } + } +} + +static void report_key(struct gpio_kp *kp, int key_index, int out, int in) +{ + struct gpio_event_matrix_info *mi = kp->keypad_info; + int pressed = test_bit(key_index, kp->keys_pressed); + unsigned short keyentry = mi->keymap[key_index]; + unsigned short keycode = keyentry & MATRIX_KEY_MASK; + unsigned short dev = keyentry >> MATRIX_CODE_BITS; + + if (pressed != test_bit(keycode, kp->input_devs->dev[dev]->key)) { + if (keycode == KEY_RESERVED) { + if (mi->flags & GPIOKPF_PRINT_UNMAPPED_KEYS) + pr_info("gpiomatrix: unmapped key, %d-%d " + "(%d-%d) changed to %d\n", + out, in, mi->output_gpios[out], + mi->input_gpios[in], pressed); + } else { + if (mi->flags & GPIOKPF_PRINT_MAPPED_KEYS) + pr_info("gpiomatrix: key %x, %d-%d (%d-%d) " + "changed to %d\n", keycode, + out, in, mi->output_gpios[out], + mi->input_gpios[in], pressed); + input_report_key(kp->input_devs->dev[dev], keycode, pressed); + } + } +} + +static void report_sync(struct gpio_kp *kp) +{ + int i; + + for (i = 0; i < kp->input_devs->count; i++) + input_sync(kp->input_devs->dev[i]); +} + +static enum hrtimer_restart gpio_keypad_timer_func(struct hrtimer *timer) +{ + int out, in; + int key_index; + int gpio; + struct gpio_kp *kp = container_of(timer, struct gpio_kp, timer); + struct gpio_event_matrix_info *mi = kp->keypad_info; + unsigned gpio_keypad_flags = mi->flags; + unsigned polarity = !!(gpio_keypad_flags & GPIOKPF_ACTIVE_HIGH); + + out = kp->current_output; + if (out == mi->noutputs) { + out = 0; + kp->last_key_state_changed = kp->key_state_changed; + kp->key_state_changed = 0; + kp->some_keys_pressed = 0; + } else { + key_index = out * mi->ninputs; + for (in = 0; in < mi->ninputs; in++, key_index++) { + gpio = mi->input_gpios[in]; + if (gpio_get_value(gpio) ^ !polarity) { + if (kp->some_keys_pressed < 3) + kp->some_keys_pressed++; + kp->key_state_changed |= !__test_and_set_bit( + key_index, kp->keys_pressed); + } else + kp->key_state_changed |= __test_and_clear_bit( + key_index, kp->keys_pressed); + } + gpio = mi->output_gpios[out]; + if (gpio_keypad_flags & GPIOKPF_DRIVE_INACTIVE) + gpio_set_value(gpio, !polarity); + else + gpio_direction_input(gpio); + out++; + } + kp->current_output = out; + if (out < mi->noutputs) { + gpio = mi->output_gpios[out]; + if (gpio_keypad_flags & GPIOKPF_DRIVE_INACTIVE) + gpio_set_value(gpio, polarity); + else + gpio_direction_output(gpio, polarity); + hrtimer_start(timer, mi->settle_time, HRTIMER_MODE_REL); + return HRTIMER_NORESTART; + } + if (gpio_keypad_flags & GPIOKPF_DEBOUNCE) { + if (kp->key_state_changed) { + hrtimer_start(&kp->timer, mi->debounce_delay, + HRTIMER_MODE_REL); + return HRTIMER_NORESTART; + } + kp->key_state_changed = kp->last_key_state_changed; + } + if (kp->key_state_changed) { + if (gpio_keypad_flags & GPIOKPF_REMOVE_SOME_PHANTOM_KEYS) + remove_phantom_keys(kp); + key_index = 0; + for (out = 0; out < mi->noutputs; out++) + for (in = 0; in < mi->ninputs; in++, key_index++) + report_key(kp, key_index, out, in); + report_sync(kp); + } + if (!kp->use_irq || kp->some_keys_pressed) { + hrtimer_start(timer, mi->poll_time, HRTIMER_MODE_REL); + return HRTIMER_NORESTART; + } + + /* No keys are pressed, reenable interrupt */ + for (out = 0; out < mi->noutputs; out++) { + if (gpio_keypad_flags & GPIOKPF_DRIVE_INACTIVE) + gpio_set_value(mi->output_gpios[out], polarity); + else + gpio_direction_output(mi->output_gpios[out], polarity); + } + for (in = 0; in < mi->ninputs; in++) + enable_irq(gpio_to_irq(mi->input_gpios[in])); + __pm_relax(&kp->wake_src); + return HRTIMER_NORESTART; +} + +static irqreturn_t gpio_keypad_irq_handler(int irq_in, void *dev_id) +{ + int i; + struct gpio_kp *kp = dev_id; + struct gpio_event_matrix_info *mi = kp->keypad_info; + unsigned gpio_keypad_flags = mi->flags; + + if (!kp->use_irq) { + /* ignore interrupt while registering the handler */ + kp->disabled_irq = 1; + disable_irq_nosync(irq_in); + return IRQ_HANDLED; + } + + for (i = 0; i < mi->ninputs; i++) + disable_irq_nosync(gpio_to_irq(mi->input_gpios[i])); + for (i = 0; i < mi->noutputs; i++) { + if (gpio_keypad_flags & GPIOKPF_DRIVE_INACTIVE) + gpio_set_value(mi->output_gpios[i], + !(gpio_keypad_flags & GPIOKPF_ACTIVE_HIGH)); + else + gpio_direction_input(mi->output_gpios[i]); + } + __pm_stay_awake(&kp->wake_src); + hrtimer_start(&kp->timer, ktime_set(0, 0), HRTIMER_MODE_REL); + return IRQ_HANDLED; +} + +static int gpio_keypad_request_irqs(struct gpio_kp *kp) +{ + int i; + int err; + unsigned int irq; + unsigned long request_flags; + struct gpio_event_matrix_info *mi = kp->keypad_info; + + switch (mi->flags & (GPIOKPF_ACTIVE_HIGH|GPIOKPF_LEVEL_TRIGGERED_IRQ)) { + default: + request_flags = IRQF_TRIGGER_FALLING; + break; + case GPIOKPF_ACTIVE_HIGH: + request_flags = IRQF_TRIGGER_RISING; + break; + case GPIOKPF_LEVEL_TRIGGERED_IRQ: + request_flags = IRQF_TRIGGER_LOW; + break; + case GPIOKPF_LEVEL_TRIGGERED_IRQ | GPIOKPF_ACTIVE_HIGH: + request_flags = IRQF_TRIGGER_HIGH; + break; + } + + for (i = 0; i < mi->ninputs; i++) { + err = irq = gpio_to_irq(mi->input_gpios[i]); + if (err < 0) + goto err_gpio_get_irq_num_failed; + err = request_irq(irq, gpio_keypad_irq_handler, request_flags, + "gpio_kp", kp); + if (err) { + pr_err("gpiomatrix: request_irq failed for input %d, " + "irq %d\n", mi->input_gpios[i], irq); + goto err_request_irq_failed; + } + err = enable_irq_wake(irq); + if (err) { + pr_err("gpiomatrix: set_irq_wake failed for input %d, " + "irq %d\n", mi->input_gpios[i], irq); + } + disable_irq(irq); + if (kp->disabled_irq) { + kp->disabled_irq = 0; + enable_irq(irq); + } + } + return 0; + + for (i = mi->noutputs - 1; i >= 0; i--) { + free_irq(gpio_to_irq(mi->input_gpios[i]), kp); +err_request_irq_failed: +err_gpio_get_irq_num_failed: + ; + } + return err; +} + +int gpio_event_matrix_func(struct gpio_event_input_devs *input_devs, + struct gpio_event_info *info, void **data, int func) +{ + int i; + int err; + int key_count; + struct gpio_kp *kp; + struct gpio_event_matrix_info *mi; + + mi = container_of(info, struct gpio_event_matrix_info, info); + if (func == GPIO_EVENT_FUNC_SUSPEND || func == GPIO_EVENT_FUNC_RESUME) { + /* TODO: disable scanning */ + return 0; + } + + if (func == GPIO_EVENT_FUNC_INIT) { + if (mi->keymap == NULL || + mi->input_gpios == NULL || + mi->output_gpios == NULL) { + err = -ENODEV; + pr_err("gpiomatrix: Incomplete pdata\n"); + goto err_invalid_platform_data; + } + key_count = mi->ninputs * mi->noutputs; + + *data = kp = kzalloc(sizeof(*kp) + sizeof(kp->keys_pressed[0]) * + BITS_TO_LONGS(key_count), GFP_KERNEL); + if (kp == NULL) { + err = -ENOMEM; + pr_err("gpiomatrix: Failed to allocate private data\n"); + goto err_kp_alloc_failed; + } + kp->input_devs = input_devs; + kp->keypad_info = mi; + for (i = 0; i < key_count; i++) { + unsigned short keyentry = mi->keymap[i]; + unsigned short keycode = keyentry & MATRIX_KEY_MASK; + unsigned short dev = keyentry >> MATRIX_CODE_BITS; + if (dev >= input_devs->count) { + pr_err("gpiomatrix: bad device index %d >= " + "%d for key code %d\n", + dev, input_devs->count, keycode); + err = -EINVAL; + goto err_bad_keymap; + } + if (keycode && keycode <= KEY_MAX) + input_set_capability(input_devs->dev[dev], + EV_KEY, keycode); + } + + for (i = 0; i < mi->noutputs; i++) { + err = gpio_request(mi->output_gpios[i], "gpio_kp_out"); + if (err) { + pr_err("gpiomatrix: gpio_request failed for " + "output %d\n", mi->output_gpios[i]); + goto err_request_output_gpio_failed; + } + if (gpio_cansleep(mi->output_gpios[i])) { + pr_err("gpiomatrix: unsupported output gpio %d," + " can sleep\n", mi->output_gpios[i]); + err = -EINVAL; + goto err_output_gpio_configure_failed; + } + if (mi->flags & GPIOKPF_DRIVE_INACTIVE) + err = gpio_direction_output(mi->output_gpios[i], + !(mi->flags & GPIOKPF_ACTIVE_HIGH)); + else + err = gpio_direction_input(mi->output_gpios[i]); + if (err) { + pr_err("gpiomatrix: gpio_configure failed for " + "output %d\n", mi->output_gpios[i]); + goto err_output_gpio_configure_failed; + } + } + for (i = 0; i < mi->ninputs; i++) { + err = gpio_request(mi->input_gpios[i], "gpio_kp_in"); + if (err) { + pr_err("gpiomatrix: gpio_request failed for " + "input %d\n", mi->input_gpios[i]); + goto err_request_input_gpio_failed; + } + err = gpio_direction_input(mi->input_gpios[i]); + if (err) { + pr_err("gpiomatrix: gpio_direction_input failed" + " for input %d\n", mi->input_gpios[i]); + goto err_gpio_direction_input_failed; + } + } + kp->current_output = mi->noutputs; + kp->key_state_changed = 1; + + hrtimer_init(&kp->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + kp->timer.function = gpio_keypad_timer_func; + wakeup_source_init(&kp->wake_src, "gpio_kp"); + err = gpio_keypad_request_irqs(kp); + kp->use_irq = err == 0; + + pr_info("GPIO Matrix Keypad Driver: Start keypad matrix for " + "%s%s in %s mode\n", input_devs->dev[0]->name, + (input_devs->count > 1) ? "..." : "", + kp->use_irq ? "interrupt" : "polling"); + + if (kp->use_irq) + __pm_stay_awake(&kp->wake_src); + hrtimer_start(&kp->timer, ktime_set(0, 0), HRTIMER_MODE_REL); + + return 0; + } + + err = 0; + kp = *data; + + if (kp->use_irq) + for (i = mi->noutputs - 1; i >= 0; i--) + free_irq(gpio_to_irq(mi->input_gpios[i]), kp); + + hrtimer_cancel(&kp->timer); + wakeup_source_trash(&kp->wake_src); + for (i = mi->noutputs - 1; i >= 0; i--) { +err_gpio_direction_input_failed: + gpio_free(mi->input_gpios[i]); +err_request_input_gpio_failed: + ; + } + for (i = mi->noutputs - 1; i >= 0; i--) { +err_output_gpio_configure_failed: + gpio_free(mi->output_gpios[i]); +err_request_output_gpio_failed: + ; + } +err_bad_keymap: + kfree(kp); +err_kp_alloc_failed: +err_invalid_platform_data: + return err; +} diff --git a/drivers/input/misc/gpio_output.c b/drivers/input/misc/gpio_output.c new file mode 100644 index 00000000000000..2aac2fad0a17bf --- /dev/null +++ b/drivers/input/misc/gpio_output.c @@ -0,0 +1,97 @@ +/* drivers/input/misc/gpio_output.c + * + * Copyright (C) 2007 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include +#include + +int gpio_event_output_event( + struct gpio_event_input_devs *input_devs, struct gpio_event_info *info, + void **data, unsigned int dev, unsigned int type, + unsigned int code, int value) +{ + int i; + struct gpio_event_output_info *oi; + oi = container_of(info, struct gpio_event_output_info, info); + if (type != oi->type) + return 0; + if (!(oi->flags & GPIOEDF_ACTIVE_HIGH)) + value = !value; + for (i = 0; i < oi->keymap_size; i++) + if (dev == oi->keymap[i].dev && code == oi->keymap[i].code) + gpio_set_value(oi->keymap[i].gpio, value); + return 0; +} + +int gpio_event_output_func( + struct gpio_event_input_devs *input_devs, struct gpio_event_info *info, + void **data, int func) +{ + int ret; + int i; + struct gpio_event_output_info *oi; + oi = container_of(info, struct gpio_event_output_info, info); + + if (func == GPIO_EVENT_FUNC_SUSPEND || func == GPIO_EVENT_FUNC_RESUME) + return 0; + + if (func == GPIO_EVENT_FUNC_INIT) { + int output_level = !(oi->flags & GPIOEDF_ACTIVE_HIGH); + + for (i = 0; i < oi->keymap_size; i++) { + int dev = oi->keymap[i].dev; + if (dev >= input_devs->count) { + pr_err("gpio_event_output_func: bad device " + "index %d >= %d for key code %d\n", + dev, input_devs->count, + oi->keymap[i].code); + ret = -EINVAL; + goto err_bad_keymap; + } + input_set_capability(input_devs->dev[dev], oi->type, + oi->keymap[i].code); + } + + for (i = 0; i < oi->keymap_size; i++) { + ret = gpio_request(oi->keymap[i].gpio, + "gpio_event_output"); + if (ret) { + pr_err("gpio_event_output_func: gpio_request " + "failed for %d\n", oi->keymap[i].gpio); + goto err_gpio_request_failed; + } + ret = gpio_direction_output(oi->keymap[i].gpio, + output_level); + if (ret) { + pr_err("gpio_event_output_func: " + "gpio_direction_output failed for %d\n", + oi->keymap[i].gpio); + goto err_gpio_direction_output_failed; + } + } + return 0; + } + + ret = 0; + for (i = oi->keymap_size - 1; i >= 0; i--) { +err_gpio_direction_output_failed: + gpio_free(oi->keymap[i].gpio); +err_gpio_request_failed: + ; + } +err_bad_keymap: + return ret; +} + diff --git a/drivers/input/misc/keychord.c b/drivers/input/misc/keychord.c new file mode 100644 index 00000000000000..a5ea27ad0e16c6 --- /dev/null +++ b/drivers/input/misc/keychord.c @@ -0,0 +1,391 @@ +/* + * drivers/input/misc/keychord.c + * + * Copyright (C) 2008 Google, Inc. + * Author: Mike Lockwood + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define KEYCHORD_NAME "keychord" +#define BUFFER_SIZE 16 + +MODULE_AUTHOR("Mike Lockwood "); +MODULE_DESCRIPTION("Key chord input driver"); +MODULE_SUPPORTED_DEVICE("keychord"); +MODULE_LICENSE("GPL"); + +#define NEXT_KEYCHORD(kc) ((struct input_keychord *) \ + ((char *)kc + sizeof(struct input_keychord) + \ + kc->count * sizeof(kc->keycodes[0]))) + +struct keychord_device { + struct input_handler input_handler; + int registered; + + /* list of keychords to monitor */ + struct input_keychord *keychords; + int keychord_count; + + /* bitmask of keys contained in our keychords */ + unsigned long keybit[BITS_TO_LONGS(KEY_CNT)]; + /* current state of the keys */ + unsigned long keystate[BITS_TO_LONGS(KEY_CNT)]; + /* number of keys that are currently pressed */ + int key_down; + + /* second input_device_id is needed for null termination */ + struct input_device_id device_ids[2]; + + spinlock_t lock; + wait_queue_head_t waitq; + unsigned char head; + unsigned char tail; + __u16 buff[BUFFER_SIZE]; +}; + +static int check_keychord(struct keychord_device *kdev, + struct input_keychord *keychord) +{ + int i; + + if (keychord->count != kdev->key_down) + return 0; + + for (i = 0; i < keychord->count; i++) { + if (!test_bit(keychord->keycodes[i], kdev->keystate)) + return 0; + } + + /* we have a match */ + return 1; +} + +static void keychord_event(struct input_handle *handle, unsigned int type, + unsigned int code, int value) +{ + struct keychord_device *kdev = handle->private; + struct input_keychord *keychord; + unsigned long flags; + int i, got_chord = 0; + + if (type != EV_KEY || code >= KEY_MAX) + return; + + spin_lock_irqsave(&kdev->lock, flags); + /* do nothing if key state did not change */ + if (!test_bit(code, kdev->keystate) == !value) + goto done; + __change_bit(code, kdev->keystate); + if (value) + kdev->key_down++; + else + kdev->key_down--; + + /* don't notify on key up */ + if (!value) + goto done; + /* ignore this event if it is not one of the keys we are monitoring */ + if (!test_bit(code, kdev->keybit)) + goto done; + + keychord = kdev->keychords; + if (!keychord) + goto done; + + /* check to see if the keyboard state matches any keychords */ + for (i = 0; i < kdev->keychord_count; i++) { + if (check_keychord(kdev, keychord)) { + kdev->buff[kdev->head] = keychord->id; + kdev->head = (kdev->head + 1) % BUFFER_SIZE; + got_chord = 1; + break; + } + /* skip to next keychord */ + keychord = NEXT_KEYCHORD(keychord); + } + +done: + spin_unlock_irqrestore(&kdev->lock, flags); + + if (got_chord) { + pr_info("keychord: got keychord id %d. Any tasks: %d\n", + keychord->id, + !list_empty_careful(&kdev->waitq.task_list)); + wake_up_interruptible(&kdev->waitq); + } +} + +static int keychord_connect(struct input_handler *handler, + struct input_dev *dev, + const struct input_device_id *id) +{ + int i, ret; + struct input_handle *handle; + struct keychord_device *kdev = + container_of(handler, struct keychord_device, input_handler); + + /* + * ignore this input device if it does not contain any keycodes + * that we are monitoring + */ + for (i = 0; i < KEY_MAX; i++) { + if (test_bit(i, kdev->keybit) && test_bit(i, dev->keybit)) + break; + } + if (i == KEY_MAX) + return -ENODEV; + + handle = kzalloc(sizeof(*handle), GFP_KERNEL); + if (!handle) + return -ENOMEM; + + handle->dev = dev; + handle->handler = handler; + handle->name = KEYCHORD_NAME; + handle->private = kdev; + + ret = input_register_handle(handle); + if (ret) + goto err_input_register_handle; + + ret = input_open_device(handle); + if (ret) + goto err_input_open_device; + + pr_info("keychord: using input dev %s for fevent\n", dev->name); + + return 0; + +err_input_open_device: + input_unregister_handle(handle); +err_input_register_handle: + kfree(handle); + return ret; +} + +static void keychord_disconnect(struct input_handle *handle) +{ + input_close_device(handle); + input_unregister_handle(handle); + kfree(handle); +} + +/* + * keychord_read is used to read keychord events from the driver + */ +static ssize_t keychord_read(struct file *file, char __user *buffer, + size_t count, loff_t *ppos) +{ + struct keychord_device *kdev = file->private_data; + __u16 id; + int retval; + unsigned long flags; + + if (count < sizeof(id)) + return -EINVAL; + count = sizeof(id); + + if (kdev->head == kdev->tail && (file->f_flags & O_NONBLOCK)) + return -EAGAIN; + + retval = wait_event_interruptible(kdev->waitq, + kdev->head != kdev->tail); + if (retval) + return retval; + + spin_lock_irqsave(&kdev->lock, flags); + /* pop a keychord ID off the queue */ + id = kdev->buff[kdev->tail]; + kdev->tail = (kdev->tail + 1) % BUFFER_SIZE; + spin_unlock_irqrestore(&kdev->lock, flags); + + if (copy_to_user(buffer, &id, count)) + return -EFAULT; + + return count; +} + +/* + * keychord_write is used to configure the driver + */ +static ssize_t keychord_write(struct file *file, const char __user *buffer, + size_t count, loff_t *ppos) +{ + struct keychord_device *kdev = file->private_data; + struct input_keychord *keychords = 0; + struct input_keychord *keychord, *next, *end; + int ret, i, key; + unsigned long flags; + + if (count < sizeof(struct input_keychord)) + return -EINVAL; + keychords = kzalloc(count, GFP_KERNEL); + if (!keychords) + return -ENOMEM; + + /* read list of keychords from userspace */ + if (copy_from_user(keychords, buffer, count)) { + kfree(keychords); + return -EFAULT; + } + + /* unregister handler before changing configuration */ + if (kdev->registered) { + input_unregister_handler(&kdev->input_handler); + kdev->registered = 0; + } + + spin_lock_irqsave(&kdev->lock, flags); + /* clear any existing configuration */ + kfree(kdev->keychords); + kdev->keychords = 0; + kdev->keychord_count = 0; + kdev->key_down = 0; + memset(kdev->keybit, 0, sizeof(kdev->keybit)); + memset(kdev->keystate, 0, sizeof(kdev->keystate)); + kdev->head = kdev->tail = 0; + + keychord = keychords; + end = (struct input_keychord *)((char *)keychord + count); + + while (keychord < end) { + next = NEXT_KEYCHORD(keychord); + if (keychord->count <= 0 || next > end) { + pr_err("keychord: invalid keycode count %d\n", + keychord->count); + goto err_unlock_return; + } + if (keychord->version != KEYCHORD_VERSION) { + pr_err("keychord: unsupported version %d\n", + keychord->version); + goto err_unlock_return; + } + + /* keep track of the keys we are monitoring in keybit */ + for (i = 0; i < keychord->count; i++) { + key = keychord->keycodes[i]; + if (key < 0 || key >= KEY_CNT) { + pr_err("keychord: keycode %d out of range\n", + key); + goto err_unlock_return; + } + __set_bit(key, kdev->keybit); + } + + kdev->keychord_count++; + keychord = next; + } + + kdev->keychords = keychords; + spin_unlock_irqrestore(&kdev->lock, flags); + + ret = input_register_handler(&kdev->input_handler); + if (ret) { + kfree(keychords); + kdev->keychords = 0; + return ret; + } + kdev->registered = 1; + + return count; + +err_unlock_return: + spin_unlock_irqrestore(&kdev->lock, flags); + kfree(keychords); + return -EINVAL; +} + +static unsigned int keychord_poll(struct file *file, poll_table *wait) +{ + struct keychord_device *kdev = file->private_data; + + poll_wait(file, &kdev->waitq, wait); + + if (kdev->head != kdev->tail) + return POLLIN | POLLRDNORM; + + return 0; +} + +static int keychord_open(struct inode *inode, struct file *file) +{ + struct keychord_device *kdev; + + kdev = kzalloc(sizeof(struct keychord_device), GFP_KERNEL); + if (!kdev) + return -ENOMEM; + + spin_lock_init(&kdev->lock); + init_waitqueue_head(&kdev->waitq); + + kdev->input_handler.event = keychord_event; + kdev->input_handler.connect = keychord_connect; + kdev->input_handler.disconnect = keychord_disconnect; + kdev->input_handler.name = KEYCHORD_NAME; + kdev->input_handler.id_table = kdev->device_ids; + + kdev->device_ids[0].flags = INPUT_DEVICE_ID_MATCH_EVBIT; + __set_bit(EV_KEY, kdev->device_ids[0].evbit); + + file->private_data = kdev; + + return 0; +} + +static int keychord_release(struct inode *inode, struct file *file) +{ + struct keychord_device *kdev = file->private_data; + + if (kdev->registered) + input_unregister_handler(&kdev->input_handler); + kfree(kdev); + + return 0; +} + +static const struct file_operations keychord_fops = { + .owner = THIS_MODULE, + .open = keychord_open, + .release = keychord_release, + .read = keychord_read, + .write = keychord_write, + .poll = keychord_poll, +}; + +static struct miscdevice keychord_misc = { + .fops = &keychord_fops, + .name = KEYCHORD_NAME, + .minor = MISC_DYNAMIC_MINOR, +}; + +static int __init keychord_init(void) +{ + return misc_register(&keychord_misc); +} + +static void __exit keychord_exit(void) +{ + misc_deregister(&keychord_misc); +} + +module_init(keychord_init); +module_exit(keychord_exit); diff --git a/drivers/input/misc/uinput.c b/drivers/input/misc/uinput.c index 92595b98e7ede7..022be0e22eba97 100644 --- a/drivers/input/misc/uinput.c +++ b/drivers/input/misc/uinput.c @@ -263,13 +263,21 @@ static int uinput_create_device(struct uinput_device *udev) return -EINVAL; } - if (test_bit(ABS_MT_SLOT, dev->absbit)) { - nslot = input_abs_get_max(dev, ABS_MT_SLOT) + 1; - error = input_mt_init_slots(dev, nslot, 0); - if (error) + if (test_bit(EV_ABS, dev->evbit)) { + input_alloc_absinfo(dev); + if (!dev->absinfo) { + error = -EINVAL; goto fail1; - } else if (test_bit(ABS_MT_POSITION_X, dev->absbit)) { - input_set_events_per_packet(dev, 60); + } + + if (test_bit(ABS_MT_SLOT, dev->absbit)) { + nslot = input_abs_get_max(dev, ABS_MT_SLOT) + 1; + error = input_mt_init_slots(dev, nslot, 0); + if (error) + goto fail1; + } else if (test_bit(ABS_MT_POSITION_X, dev->absbit)) { + input_set_events_per_packet(dev, 60); + } } if (test_bit(EV_FF, dev->evbit) && !udev->ff_effects_max) { diff --git a/drivers/input/mouse/alps.c b/drivers/input/mouse/alps.c index 6d7de9bfed9a7f..b93fe83a0b63f3 100644 --- a/drivers/input/mouse/alps.c +++ b/drivers/input/mouse/alps.c @@ -1346,6 +1346,18 @@ static void alps_process_packet_ss4_v2(struct psmouse *psmouse) priv->multi_packet = 0; + /* Report trackstick */ + if (alps_get_pkt_id_ss4_v2(packet) == SS4_PACKET_ID_STICK) { + if (priv->flags & ALPS_DUALPOINT) { + input_report_key(dev2, BTN_LEFT, f->ts_left); + input_report_key(dev2, BTN_RIGHT, f->ts_right); + input_report_key(dev2, BTN_MIDDLE, f->ts_middle); + input_sync(dev2); + } + return; + } + + /* Report touchpad */ alps_report_mt_data(psmouse, (f->fingers <= 4) ? f->fingers : 4); input_mt_report_finger_count(dev, f->fingers); @@ -1356,13 +1368,6 @@ static void alps_process_packet_ss4_v2(struct psmouse *psmouse) input_report_abs(dev, ABS_PRESSURE, f->pressure); input_sync(dev); - - if (priv->flags & ALPS_DUALPOINT) { - input_report_key(dev2, BTN_LEFT, f->ts_left); - input_report_key(dev2, BTN_RIGHT, f->ts_right); - input_report_key(dev2, BTN_MIDDLE, f->ts_middle); - input_sync(dev2); - } } static bool alps_is_valid_package_ss4_v2(struct psmouse *psmouse) diff --git a/drivers/input/rmi4/rmi_f54.c b/drivers/input/rmi4/rmi_f54.c index cf805b96086621..2e934aef3d2a8b 100644 --- a/drivers/input/rmi4/rmi_f54.c +++ b/drivers/input/rmi4/rmi_f54.c @@ -200,7 +200,7 @@ static int rmi_f54_request_report(struct rmi_function *fn, u8 report_type) error = rmi_write(rmi_dev, fn->fd.command_base_addr, F54_GET_REPORT); if (error < 0) - return error; + goto unlock; init_completion(&f54->cmd_done); @@ -209,9 +209,10 @@ static int rmi_f54_request_report(struct rmi_function *fn, u8 report_type) queue_delayed_work(f54->workqueue, &f54->work, 0); +unlock: mutex_unlock(&f54->data_mutex); - return 0; + return error; } static size_t rmi_f54_get_report_size(struct f54_data *f54) diff --git a/drivers/input/serio/i8042-x86ia64io.h b/drivers/input/serio/i8042-x86ia64io.h index 073246c7d1634e..0cdd95801a2570 100644 --- a/drivers/input/serio/i8042-x86ia64io.h +++ b/drivers/input/serio/i8042-x86ia64io.h @@ -211,6 +211,12 @@ static const struct dmi_system_id __initconst i8042_dmi_noloop_table[] = { DMI_MATCH(DMI_PRODUCT_VERSION, "Rev 1"), }, }, + { + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "PEGATRON CORPORATION"), + DMI_MATCH(DMI_PRODUCT_NAME, "C15B"), + }, + }, { } }; diff --git a/drivers/input/touchscreen/elants_i2c.c b/drivers/input/touchscreen/elants_i2c.c index 02aec284decac3..3e6003d32e565c 100644 --- a/drivers/input/touchscreen/elants_i2c.c +++ b/drivers/input/touchscreen/elants_i2c.c @@ -914,9 +914,9 @@ static irqreturn_t elants_i2c_irq(int irq, void *_dev) case QUEUE_HEADER_NORMAL: report_count = ts->buf[FW_HDR_COUNT]; - if (report_count > 3) { + if (report_count == 0 || report_count > 3) { dev_err(&client->dev, - "too large report count: %*ph\n", + "bad report count: %*ph\n", HEADER_SIZE, ts->buf); break; } diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 754595ee11b68a..11a13b5be73a38 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -1021,7 +1021,7 @@ static int __iommu_queue_command_sync(struct amd_iommu *iommu, next_tail = (tail + sizeof(*cmd)) % CMD_BUFFER_SIZE; left = (head - next_tail) % CMD_BUFFER_SIZE; - if (left <= 2) { + if (left <= 0x20) { struct iommu_cmd sync_cmd; int ret; diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd_iommu_v2.c index 594849a3a9be63..f8ed8c95b68537 100644 --- a/drivers/iommu/amd_iommu_v2.c +++ b/drivers/iommu/amd_iommu_v2.c @@ -805,8 +805,10 @@ int amd_iommu_init_device(struct pci_dev *pdev, int pasids) goto out_free_domain; group = iommu_group_get(&pdev->dev); - if (!group) + if (!group) { + ret = -EINVAL; goto out_free_domain; + } ret = iommu_attach_group(dev_state->domain, group); if (ret != 0) diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index d8376c2d18b3bf..d82637ab09fd49 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -2037,6 +2037,25 @@ static int domain_context_mapping_one(struct dmar_domain *domain, if (context_present(context)) goto out_unlock; + /* + * For kdump cases, old valid entries may be cached due to the + * in-flight DMA and copied pgtable, but there is no unmapping + * behaviour for them, thus we need an explicit cache flush for + * the newly-mapped device. For kdump, at this point, the device + * is supposed to finish reset at its driver probe stage, so no + * in-flight DMA will exist, and we don't need to worry anymore + * hereafter. + */ + if (context_copied(context)) { + u16 did_old = context_domain_id(context); + + if (did_old >= 0 && did_old < cap_ndoms(iommu->cap)) + iommu->flush.flush_context(iommu, did_old, + (((u16)bus) << 8) | devfn, + DMA_CCMD_MASK_NOBIT, + DMA_CCMD_DEVICE_INVL); + } + pgd = domain->pgd; context_clear_entry(context); @@ -5197,6 +5216,25 @@ static void intel_iommu_remove_device(struct device *dev) } #ifdef CONFIG_INTEL_IOMMU_SVM +#define MAX_NR_PASID_BITS (20) +static inline unsigned long intel_iommu_get_pts(struct intel_iommu *iommu) +{ + /* + * Convert ecap_pss to extend context entry pts encoding, also + * respect the soft pasid_max value set by the iommu. + * - number of PASID bits = ecap_pss + 1 + * - number of PASID table entries = 2^(pts + 5) + * Therefore, pts = ecap_pss - 4 + * e.g. KBL ecap_pss = 0x13, PASID has 20 bits, pts = 15 + */ + if (ecap_pss(iommu->ecap) < 5) + return 0; + + /* pasid_max is encoded as actual number of entries not the bits */ + return find_first_bit((unsigned long *)&iommu->pasid_max, + MAX_NR_PASID_BITS) - 5; +} + int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev) { struct device_domain_info *info; @@ -5229,7 +5267,9 @@ int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sd if (!(ctx_lo & CONTEXT_PASIDE)) { context[1].hi = (u64)virt_to_phys(iommu->pasid_state_table); - context[1].lo = (u64)virt_to_phys(iommu->pasid_table) | ecap_pss(iommu->ecap); + context[1].lo = (u64)virt_to_phys(iommu->pasid_table) | + intel_iommu_get_pts(iommu); + wmb(); /* CONTEXT_TT_MULTI_LEVEL and CONTEXT_TT_DEV_IOTLB are both * extended to permit requests-with-PASID if the PASIDE bit diff --git a/drivers/irqchip/irq-bcm7038-l1.c b/drivers/irqchip/irq-bcm7038-l1.c index 353c5498621180..c2662a1bfdd374 100644 --- a/drivers/irqchip/irq-bcm7038-l1.c +++ b/drivers/irqchip/irq-bcm7038-l1.c @@ -215,6 +215,31 @@ static int bcm7038_l1_set_affinity(struct irq_data *d, return 0; } +static void bcm7038_l1_cpu_offline(struct irq_data *d) +{ + struct cpumask *mask = irq_data_get_affinity_mask(d); + int cpu = smp_processor_id(); + cpumask_t new_affinity; + + /* This CPU was not on the affinity mask */ + if (!cpumask_test_cpu(cpu, mask)) + return; + + if (cpumask_weight(mask) > 1) { + /* + * Multiple CPU affinity, remove this CPU from the affinity + * mask + */ + cpumask_copy(&new_affinity, mask); + cpumask_clear_cpu(cpu, &new_affinity); + } else { + /* Only CPU, put on the lowest online CPU */ + cpumask_clear(&new_affinity); + cpumask_set_cpu(cpumask_first(cpu_online_mask), &new_affinity); + } + irq_set_affinity_locked(d, &new_affinity, false); +} + static int __init bcm7038_l1_init_one(struct device_node *dn, unsigned int idx, struct bcm7038_l1_chip *intc) @@ -266,6 +291,7 @@ static struct irq_chip bcm7038_l1_irq_chip = { .irq_mask = bcm7038_l1_mask, .irq_unmask = bcm7038_l1_unmask, .irq_set_affinity = bcm7038_l1_set_affinity, + .irq_cpu_offline = bcm7038_l1_cpu_offline, }; static int bcm7038_l1_map(struct irq_domain *d, unsigned int virq, diff --git a/drivers/isdn/hardware/eicon/message.c b/drivers/isdn/hardware/eicon/message.c index 1a1d99704fe694..296f1411fe8420 100644 --- a/drivers/isdn/hardware/eicon/message.c +++ b/drivers/isdn/hardware/eicon/message.c @@ -11297,7 +11297,8 @@ static void mixer_notify_update(PLCI *plci, byte others) ((CAPI_MSG *) msg)->header.ncci = 0; ((CAPI_MSG *) msg)->info.facility_req.Selector = SELECTOR_LINE_INTERCONNECT; ((CAPI_MSG *) msg)->info.facility_req.structs[0] = 3; - PUT_WORD(&(((CAPI_MSG *) msg)->info.facility_req.structs[1]), LI_REQ_SILENT_UPDATE); + ((CAPI_MSG *) msg)->info.facility_req.structs[1] = LI_REQ_SILENT_UPDATE & 0xff; + ((CAPI_MSG *) msg)->info.facility_req.structs[2] = LI_REQ_SILENT_UPDATE >> 8; ((CAPI_MSG *) msg)->info.facility_req.structs[3] = 0; w = api_put(notify_plci->appl, (CAPI_MSG *) msg); if (w != _QUEUE_FULL) diff --git a/drivers/md/Kconfig b/drivers/md/Kconfig index 02a5345a44a60a..1e66909af4bcd8 100644 --- a/drivers/md/Kconfig +++ b/drivers/md/Kconfig @@ -458,6 +458,21 @@ config DM_VERITY If unsure, say N. +config DM_VERITY_HASH_PREFETCH_MIN_SIZE_128 + bool "Prefetch size 128" + +config DM_VERITY_HASH_PREFETCH_MIN_SIZE + int "Verity hash prefetch minimum size" + depends on DM_VERITY + range 1 4096 + default 128 if DM_VERITY_HASH_PREFETCH_MIN_SIZE_128 + default 1 + ---help--- + This sets minimum number of hash blocks to prefetch for dm-verity. + For devices like eMMC, having larger prefetch size like 128 can improve + performance with increased memory consumption for keeping more hashes + in RAM. + config DM_VERITY_FEC bool "Verity forward error correction support" depends on DM_VERITY @@ -500,4 +515,22 @@ config DM_LOG_WRITES If unsure, say N. +config DM_ANDROID_VERITY + tristate "Android verity target support" + depends on DM_VERITY + depends on X509_CERTIFICATE_PARSER + depends on SYSTEM_TRUSTED_KEYRING + depends on PUBLIC_KEY_ALGO_RSA + depends on KEYS + depends on ASYMMETRIC_KEY_TYPE + depends on ASYMMETRIC_PUBLIC_KEY_SUBTYPE + depends on MD_LINEAR + select DM_VERITY_HASH_PREFETCH_MIN_SIZE_128 + ---help--- + This device-mapper target is virtually a VERITY target. This + target is setup by reading the metadata contents piggybacked + to the actual data blocks in the block device. The signature + of the metadata contents are verified against the key included + in the system keyring. Upon success, the underlying verity + target is setup. endif # MD diff --git a/drivers/md/Makefile b/drivers/md/Makefile index 3cbda1af87a0d6..f26ce41af389a7 100644 --- a/drivers/md/Makefile +++ b/drivers/md/Makefile @@ -59,6 +59,7 @@ obj-$(CONFIG_DM_CACHE_SMQ) += dm-cache-smq.o obj-$(CONFIG_DM_CACHE_CLEANER) += dm-cache-cleaner.o obj-$(CONFIG_DM_ERA) += dm-era.o obj-$(CONFIG_DM_LOG_WRITES) += dm-log-writes.o +obj-$(CONFIG_DM_ANDROID_VERITY) += dm-android-verity.o ifeq ($(CONFIG_DM_UEVENT),y) dm-mod-objs += dm-uevent.o diff --git a/drivers/md/dm-android-verity.c b/drivers/md/dm-android-verity.c new file mode 100644 index 00000000000000..881e7099d40125 --- /dev/null +++ b/drivers/md/dm-android-verity.c @@ -0,0 +1,926 @@ +/* + * Copyright (C) 2015 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "dm-verity.h" +#include "dm-android-verity.h" + +static char verifiedbootstate[VERITY_COMMANDLINE_PARAM_LENGTH]; +static char veritymode[VERITY_COMMANDLINE_PARAM_LENGTH]; +static char veritykeyid[VERITY_DEFAULT_KEY_ID_LENGTH]; +static char buildvariant[BUILD_VARIANT]; + +static bool target_added; +static bool verity_enabled = true; +struct dentry *debug_dir; +static int android_verity_ctr(struct dm_target *ti, unsigned argc, char **argv); + +static struct target_type android_verity_target = { + .name = "android-verity", + .version = {1, 0, 0}, + .module = THIS_MODULE, + .ctr = android_verity_ctr, + .dtr = verity_dtr, + .map = verity_map, + .status = verity_status, + .prepare_ioctl = verity_prepare_ioctl, + .iterate_devices = verity_iterate_devices, + .io_hints = verity_io_hints, +}; + +static int __init verified_boot_state_param(char *line) +{ + strlcpy(verifiedbootstate, line, sizeof(verifiedbootstate)); + return 1; +} + +__setup("androidboot.verifiedbootstate=", verified_boot_state_param); + +static int __init verity_mode_param(char *line) +{ + strlcpy(veritymode, line, sizeof(veritymode)); + return 1; +} + +__setup("androidboot.veritymode=", verity_mode_param); + +static int __init verity_keyid_param(char *line) +{ + strlcpy(veritykeyid, line, sizeof(veritykeyid)); + return 1; +} + +__setup("veritykeyid=", verity_keyid_param); + +static int __init verity_buildvariant(char *line) +{ + strlcpy(buildvariant, line, sizeof(buildvariant)); + return 1; +} + +__setup("buildvariant=", verity_buildvariant); + +static inline bool default_verity_key_id(void) +{ + return veritykeyid[0] != '\0'; +} + +static inline bool is_eng(void) +{ + static const char typeeng[] = "eng"; + + return !strncmp(buildvariant, typeeng, sizeof(typeeng)); +} + +static inline bool is_userdebug(void) +{ + static const char typeuserdebug[] = "userdebug"; + + return !strncmp(buildvariant, typeuserdebug, sizeof(typeuserdebug)); +} + + +static int table_extract_mpi_array(struct public_key_signature *pks, + const void *data, size_t len) +{ + MPI mpi = mpi_read_raw_data(data, len); + + if (!mpi) { + DMERR("Error while allocating mpi array"); + return -ENOMEM; + } + + pks->mpi[0] = mpi; + pks->nr_mpi = 1; + return 0; +} + +static struct public_key_signature *table_make_digest( + enum hash_algo hash, + const void *table, + unsigned long table_len) +{ + struct public_key_signature *pks = NULL; + struct crypto_shash *tfm; + struct shash_desc *desc; + size_t digest_size, desc_size; + int ret; + + /* Allocate the hashing algorithm we're going to need and find out how + * big the hash operational data will be. + */ + tfm = crypto_alloc_shash(hash_algo_name[hash], 0, 0); + if (IS_ERR(tfm)) + return ERR_CAST(tfm); + + desc_size = crypto_shash_descsize(tfm) + sizeof(*desc); + digest_size = crypto_shash_digestsize(tfm); + + /* We allocate the hash operational data storage on the end of out + * context data and the digest output buffer on the end of that. + */ + ret = -ENOMEM; + pks = kzalloc(digest_size + sizeof(*pks) + desc_size, GFP_KERNEL); + if (!pks) + goto error; + + pks->pkey_hash_algo = hash; + pks->digest = (u8 *)pks + sizeof(*pks) + desc_size; + pks->digest_size = digest_size; + + desc = (struct shash_desc *)(pks + 1); + desc->tfm = tfm; + desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; + + ret = crypto_shash_init(desc); + if (ret < 0) + goto error; + + ret = crypto_shash_finup(desc, table, table_len, pks->digest); + if (ret < 0) + goto error; + + crypto_free_shash(tfm); + return pks; + +error: + kfree(pks); + crypto_free_shash(tfm); + return ERR_PTR(ret); +} + +static int read_block_dev(struct bio_read *payload, struct block_device *bdev, + sector_t offset, int length) +{ + struct bio *bio; + int err = 0, i; + + payload->number_of_pages = DIV_ROUND_UP(length, PAGE_SIZE); + + bio = bio_alloc(GFP_KERNEL, payload->number_of_pages); + if (!bio) { + DMERR("Error while allocating bio"); + return -ENOMEM; + } + + bio->bi_bdev = bdev; + bio->bi_iter.bi_sector = offset; + + payload->page_io = kzalloc(sizeof(struct page *) * + payload->number_of_pages, GFP_KERNEL); + if (!payload->page_io) { + DMERR("page_io array alloc failed"); + err = -ENOMEM; + goto free_bio; + } + + for (i = 0; i < payload->number_of_pages; i++) { + payload->page_io[i] = alloc_page(GFP_KERNEL); + if (!payload->page_io[i]) { + DMERR("alloc_page failed"); + err = -ENOMEM; + goto free_pages; + } + if (!bio_add_page(bio, payload->page_io[i], PAGE_SIZE, 0)) { + DMERR("bio_add_page error"); + err = -EIO; + goto free_pages; + } + } + + if (!submit_bio_wait(READ, bio)) + /* success */ + goto free_bio; + DMERR("bio read failed"); + err = -EIO; + +free_pages: + for (i = 0; i < payload->number_of_pages; i++) + if (payload->page_io[i]) + __free_page(payload->page_io[i]); + kfree(payload->page_io); +free_bio: + bio_put(bio); + return err; +} + +static inline u64 fec_div_round_up(u64 x, u64 y) +{ + u64 remainder; + + return div64_u64_rem(x, y, &remainder) + + (remainder > 0 ? 1 : 0); +} + +static inline void populate_fec_metadata(struct fec_header *header, + struct fec_ecc_metadata *ecc) +{ + ecc->blocks = fec_div_round_up(le64_to_cpu(header->inp_size), + FEC_BLOCK_SIZE); + ecc->roots = le32_to_cpu(header->roots); + ecc->start = le64_to_cpu(header->inp_size); +} + +static inline int validate_fec_header(struct fec_header *header, u64 offset) +{ + /* move offset to make the sanity check work for backup header + * as well. */ + offset -= offset % FEC_BLOCK_SIZE; + if (le32_to_cpu(header->magic) != FEC_MAGIC || + le32_to_cpu(header->version) != FEC_VERSION || + le32_to_cpu(header->size) != sizeof(struct fec_header) || + le32_to_cpu(header->roots) == 0 || + le32_to_cpu(header->roots) >= FEC_RSM) + return -EINVAL; + + return 0; +} + +static int extract_fec_header(dev_t dev, struct fec_header *fec, + struct fec_ecc_metadata *ecc) +{ + u64 device_size; + struct bio_read payload; + int i, err = 0; + struct block_device *bdev; + + bdev = blkdev_get_by_dev(dev, FMODE_READ, NULL); + + if (IS_ERR_OR_NULL(bdev)) { + DMERR("bdev get error"); + return PTR_ERR(bdev); + } + + device_size = i_size_read(bdev->bd_inode); + + /* fec metadata size is a power of 2 and PAGE_SIZE + * is a power of 2 as well. + */ + BUG_ON(FEC_BLOCK_SIZE > PAGE_SIZE); + /* 512 byte sector alignment */ + BUG_ON(((device_size - FEC_BLOCK_SIZE) % (1 << SECTOR_SHIFT)) != 0); + + err = read_block_dev(&payload, bdev, (device_size - + FEC_BLOCK_SIZE) / (1 << SECTOR_SHIFT), FEC_BLOCK_SIZE); + if (err) { + DMERR("Error while reading verity metadata"); + goto error; + } + + BUG_ON(sizeof(struct fec_header) > PAGE_SIZE); + memcpy(fec, page_address(payload.page_io[0]), + sizeof(*fec)); + + ecc->valid = true; + if (validate_fec_header(fec, device_size - FEC_BLOCK_SIZE)) { + /* Try the backup header */ + memcpy(fec, page_address(payload.page_io[0]) + FEC_BLOCK_SIZE + - sizeof(*fec) , + sizeof(*fec)); + if (validate_fec_header(fec, device_size - + sizeof(struct fec_header))) + ecc->valid = false; + } + + if (ecc->valid) + populate_fec_metadata(fec, ecc); + + for (i = 0; i < payload.number_of_pages; i++) + __free_page(payload.page_io[i]); + kfree(payload.page_io); + +error: + blkdev_put(bdev, FMODE_READ); + return err; +} +static void find_metadata_offset(struct fec_header *fec, + struct block_device *bdev, u64 *metadata_offset) +{ + u64 device_size; + + device_size = i_size_read(bdev->bd_inode); + + if (le32_to_cpu(fec->magic) == FEC_MAGIC) + *metadata_offset = le64_to_cpu(fec->inp_size) - + VERITY_METADATA_SIZE; + else + *metadata_offset = device_size - VERITY_METADATA_SIZE; +} + +static int find_size(dev_t dev, u64 *device_size) +{ + struct block_device *bdev; + + bdev = blkdev_get_by_dev(dev, FMODE_READ, NULL); + if (IS_ERR_OR_NULL(bdev)) { + DMERR("blkdev_get_by_dev failed"); + return PTR_ERR(bdev); + } + + *device_size = i_size_read(bdev->bd_inode); + *device_size >>= SECTOR_SHIFT; + + DMINFO("blkdev size in sectors: %llu", *device_size); + blkdev_put(bdev, FMODE_READ); + return 0; +} + +static int verify_header(struct android_metadata_header *header) +{ + int retval = -EINVAL; + + if (is_userdebug() && le32_to_cpu(header->magic_number) == + VERITY_METADATA_MAGIC_DISABLE) + return VERITY_STATE_DISABLE; + + if (!(le32_to_cpu(header->magic_number) == + VERITY_METADATA_MAGIC_NUMBER) || + (le32_to_cpu(header->magic_number) == + VERITY_METADATA_MAGIC_DISABLE)) { + DMERR("Incorrect magic number"); + return retval; + } + + if (le32_to_cpu(header->protocol_version) != + VERITY_METADATA_VERSION) { + DMERR("Unsupported version %u", + le32_to_cpu(header->protocol_version)); + return retval; + } + + return 0; +} + +static int extract_metadata(dev_t dev, struct fec_header *fec, + struct android_metadata **metadata, + bool *verity_enabled) +{ + struct block_device *bdev; + struct android_metadata_header *header; + int i; + u32 table_length, copy_length, offset; + u64 metadata_offset; + struct bio_read payload; + int err = 0; + + bdev = blkdev_get_by_dev(dev, FMODE_READ, NULL); + + if (IS_ERR_OR_NULL(bdev)) { + DMERR("blkdev_get_by_dev failed"); + return -ENODEV; + } + + find_metadata_offset(fec, bdev, &metadata_offset); + + /* Verity metadata size is a power of 2 and PAGE_SIZE + * is a power of 2 as well. + * PAGE_SIZE is also a multiple of 512 bytes. + */ + if (VERITY_METADATA_SIZE > PAGE_SIZE) + BUG_ON(VERITY_METADATA_SIZE % PAGE_SIZE != 0); + /* 512 byte sector alignment */ + BUG_ON(metadata_offset % (1 << SECTOR_SHIFT) != 0); + + err = read_block_dev(&payload, bdev, metadata_offset / + (1 << SECTOR_SHIFT), VERITY_METADATA_SIZE); + if (err) { + DMERR("Error while reading verity metadata"); + goto blkdev_release; + } + + header = kzalloc(sizeof(*header), GFP_KERNEL); + if (!header) { + DMERR("kzalloc failed for header"); + err = -ENOMEM; + goto free_payload; + } + + memcpy(header, page_address(payload.page_io[0]), + sizeof(*header)); + + DMINFO("bio magic_number:%u protocol_version:%d table_length:%u", + le32_to_cpu(header->magic_number), + le32_to_cpu(header->protocol_version), + le32_to_cpu(header->table_length)); + + err = verify_header(header); + + if (err == VERITY_STATE_DISABLE) { + DMERR("Mounting root with verity disabled"); + *verity_enabled = false; + /* we would still have to read the metadata to figure out + * the data blocks size. Or may be could map the entire + * partition similar to mounting the device. + * + * Reset error as well as the verity_enabled flag is changed. + */ + err = 0; + } else if (err) + goto free_header; + + *metadata = kzalloc(sizeof(**metadata), GFP_KERNEL); + if (!*metadata) { + DMERR("kzalloc for metadata failed"); + err = -ENOMEM; + goto free_header; + } + + (*metadata)->header = header; + table_length = le32_to_cpu(header->table_length); + + if (table_length == 0 || + table_length > (VERITY_METADATA_SIZE - + sizeof(struct android_metadata_header))) { + DMERR("table_length too long"); + err = -EINVAL; + goto free_metadata; + } + + (*metadata)->verity_table = kzalloc(table_length + 1, GFP_KERNEL); + + if (!(*metadata)->verity_table) { + DMERR("kzalloc verity_table failed"); + err = -ENOMEM; + goto free_metadata; + } + + if (sizeof(struct android_metadata_header) + + table_length <= PAGE_SIZE) { + memcpy((*metadata)->verity_table, + page_address(payload.page_io[0]) + + sizeof(struct android_metadata_header), + table_length); + } else { + copy_length = PAGE_SIZE - + sizeof(struct android_metadata_header); + memcpy((*metadata)->verity_table, + page_address(payload.page_io[0]) + + sizeof(struct android_metadata_header), + copy_length); + table_length -= copy_length; + offset = copy_length; + i = 1; + while (table_length != 0) { + if (table_length > PAGE_SIZE) { + memcpy((*metadata)->verity_table + offset, + page_address(payload.page_io[i]), + PAGE_SIZE); + offset += PAGE_SIZE; + table_length -= PAGE_SIZE; + } else { + memcpy((*metadata)->verity_table + offset, + page_address(payload.page_io[i]), + table_length); + table_length = 0; + } + i++; + } + } + (*metadata)->verity_table[table_length] = '\0'; + + DMINFO("verity_table: %s", (*metadata)->verity_table); + goto free_payload; + +free_metadata: + kfree(*metadata); +free_header: + kfree(header); +free_payload: + for (i = 0; i < payload.number_of_pages; i++) + if (payload.page_io[i]) + __free_page(payload.page_io[i]); + kfree(payload.page_io); +blkdev_release: + blkdev_put(bdev, FMODE_READ); + return err; +} + +/* helper functions to extract properties from dts */ +const char *find_dt_value(const char *name) +{ + struct device_node *firmware; + const char *value; + + firmware = of_find_node_by_path("/firmware/android"); + if (!firmware) + return NULL; + value = of_get_property(firmware, name, NULL); + of_node_put(firmware); + + return value; +} + +static int verity_mode(void) +{ + static const char enforcing[] = "enforcing"; + static const char verified_mode_prop[] = "veritymode"; + const char *value; + + value = find_dt_value(verified_mode_prop); + if (!value) + value = veritymode; + if (!strncmp(value, enforcing, sizeof(enforcing) - 1)) + return DM_VERITY_MODE_RESTART; + + return DM_VERITY_MODE_EIO; +} + +static int verify_verity_signature(char *key_id, + struct android_metadata *metadata) +{ + key_ref_t key_ref; + struct key *key; + struct public_key_signature *pks = NULL; + int retval = -EINVAL; + + key_ref = keyring_search(make_key_ref(system_trusted_keyring, 1), + &key_type_asymmetric, key_id); + + if (IS_ERR(key_ref)) { + DMERR("keyring: key not found"); + return -ENOKEY; + } + + key = key_ref_to_ptr(key_ref); + + pks = table_make_digest(HASH_ALGO_SHA256, + (const void *)metadata->verity_table, + le32_to_cpu(metadata->header->table_length)); + + if (IS_ERR(pks)) { + DMERR("hashing failed"); + goto error; + } + + retval = table_extract_mpi_array(pks, &metadata->header->signature[0], + RSANUMBYTES); + if (retval < 0) { + DMERR("Error extracting mpi %d", retval); + goto error; + } + + retval = verify_signature(key, pks); + mpi_free(pks->rsa.s); +error: + kfree(pks); + key_put(key); + + return retval; +} + +static void handle_error(void) +{ + int mode = verity_mode(); + if (mode == DM_VERITY_MODE_RESTART) { + DMERR("triggering restart"); + kernel_restart("dm-verity device corrupted"); + } else { + DMERR("Mounting verity root failed"); + } +} + +static inline bool test_mult_overflow(sector_t a, u32 b) +{ + sector_t r = (sector_t)~0ULL; + + sector_div(r, b); + return a > r; +} + +static int add_as_linear_device(struct dm_target *ti, char *dev) +{ + /*Move to linear mapping defines*/ + char *linear_table_args[DM_LINEAR_ARGS] = {dev, + DM_LINEAR_TARGET_OFFSET}; + int err = 0; + + android_verity_target.dtr = dm_linear_dtr, + android_verity_target.map = dm_linear_map, + android_verity_target.status = dm_linear_status, + android_verity_target.prepare_ioctl = dm_linear_prepare_ioctl, + android_verity_target.iterate_devices = dm_linear_iterate_devices, + android_verity_target.direct_access = dm_linear_direct_access, + android_verity_target.io_hints = NULL; + + err = dm_linear_ctr(ti, DM_LINEAR_ARGS, linear_table_args); + + if (!err) { + DMINFO("Added android-verity as a linear target"); + target_added = true; + } else + DMERR("Failed to add android-verity as linear target"); + + return err; +} + +/* + * Target parameters: + * Key id of the public key in the system keyring. + * Verity metadata's signature would be verified against + * this. If the key id contains spaces, replace them + * with '#'. + * The block device for which dm-verity is being setup. + */ +static int android_verity_ctr(struct dm_target *ti, unsigned argc, char **argv) +{ + dev_t uninitialized_var(dev); + struct android_metadata *metadata = NULL; + int err = 0, i, mode; + char *key_id, *table_ptr, dummy, *target_device, + *verity_table_args[VERITY_TABLE_ARGS + 2 + VERITY_TABLE_OPT_FEC_ARGS]; + /* One for specifying number of opt args and one for mode */ + sector_t data_sectors; + u32 data_block_size; + unsigned int no_of_args = VERITY_TABLE_ARGS + 2 + VERITY_TABLE_OPT_FEC_ARGS; + struct fec_header uninitialized_var(fec); + struct fec_ecc_metadata uninitialized_var(ecc); + char buf[FEC_ARG_LENGTH], *buf_ptr; + unsigned long long tmpll; + u64 uninitialized_var(device_size); + + if (argc == 1) { + /* Use the default keyid */ + if (default_verity_key_id()) + key_id = veritykeyid; + else if (!is_eng()) { + DMERR("veritykeyid= is not set"); + handle_error(); + return -EINVAL; + } + } else if (argc == 2) + key_id = argv[1]; + else { + DMERR("Incorrect number of arguments"); + handle_error(); + return -EINVAL; + } + + target_device = argv[0]; + + dev = name_to_dev_t(target_device); + if (!dev) { + DMERR("no dev found for %s", target_device); + handle_error(); + return -EINVAL; + } + + if (is_eng()) { + err = find_size(dev, &device_size); + if (err) { + DMERR("error finding bdev size"); + handle_error(); + return err; + } + + ti->len = device_size; + err = add_as_linear_device(ti, target_device); + if (err) { + handle_error(); + return err; + } + verity_enabled = false; + return 0; + } + + strreplace(key_id, '#', ' '); + + DMINFO("key:%s dev:%s", key_id, target_device); + + if (extract_fec_header(dev, &fec, &ecc)) { + DMERR("Error while extracting fec header"); + handle_error(); + return -EINVAL; + } + + err = extract_metadata(dev, &fec, &metadata, &verity_enabled); + + if (err) { + DMERR("Error while extracting metadata"); + handle_error(); + goto free_metadata; + } + + if (verity_enabled) { + err = verify_verity_signature(key_id, metadata); + + if (err) { + DMERR("Signature verification failed"); + handle_error(); + goto free_metadata; + } else + DMINFO("Signature verification success"); + } + + table_ptr = metadata->verity_table; + + for (i = 0; i < VERITY_TABLE_ARGS; i++) { + verity_table_args[i] = strsep(&table_ptr, " "); + if (verity_table_args[i] == NULL) + break; + } + + if (i != VERITY_TABLE_ARGS) { + DMERR("Verity table not in the expected format"); + err = -EINVAL; + handle_error(); + goto free_metadata; + } + + if (sscanf(verity_table_args[5], "%llu%c", &tmpll, &dummy) + != 1) { + DMERR("Verity table not in the expected format"); + handle_error(); + err = -EINVAL; + goto free_metadata; + } + + if (tmpll > ULONG_MAX) { + DMERR(" too large. Forgot to turn on CONFIG_LBDAF?"); + handle_error(); + err = -EINVAL; + goto free_metadata; + } + + data_sectors = tmpll; + + if (sscanf(verity_table_args[3], "%u%c", &data_block_size, &dummy) + != 1) { + DMERR("Verity table not in the expected format"); + handle_error(); + err = -EINVAL; + goto free_metadata; + } + + if (test_mult_overflow(data_sectors, data_block_size >> + SECTOR_SHIFT)) { + DMERR("data_sectors too large"); + handle_error(); + err = -EOVERFLOW; + goto free_metadata; + } + + data_sectors *= data_block_size >> SECTOR_SHIFT; + DMINFO("Data sectors %llu", (unsigned long long)data_sectors); + + /* update target length */ + ti->len = data_sectors; + + /* Setup linear target and free */ + if (!verity_enabled) { + err = add_as_linear_device(ti, target_device); + goto free_metadata; + } + + /*substitute data_dev and hash_dev*/ + verity_table_args[1] = target_device; + verity_table_args[2] = target_device; + + mode = verity_mode(); + + if (ecc.valid && IS_BUILTIN(CONFIG_DM_VERITY_FEC)) { + if (mode) { + err = snprintf(buf, FEC_ARG_LENGTH, + "%u %s " VERITY_TABLE_OPT_FEC_FORMAT, + 1 + VERITY_TABLE_OPT_FEC_ARGS, + mode == DM_VERITY_MODE_RESTART ? + VERITY_TABLE_OPT_RESTART : + VERITY_TABLE_OPT_LOGGING, + target_device, + ecc.start / FEC_BLOCK_SIZE, ecc.blocks, + ecc.roots); + } else { + err = snprintf(buf, FEC_ARG_LENGTH, + "%u " VERITY_TABLE_OPT_FEC_FORMAT, + VERITY_TABLE_OPT_FEC_ARGS, target_device, + ecc.start / FEC_BLOCK_SIZE, ecc.blocks, + ecc.roots); + } + } else if (mode) { + err = snprintf(buf, FEC_ARG_LENGTH, + "2 " VERITY_TABLE_OPT_IGNZERO " %s", + mode == DM_VERITY_MODE_RESTART ? + VERITY_TABLE_OPT_RESTART : VERITY_TABLE_OPT_LOGGING); + } else { + err = snprintf(buf, FEC_ARG_LENGTH, "1 %s", + "ignore_zero_blocks"); + } + + if (err < 0 || err >= FEC_ARG_LENGTH) + goto free_metadata; + + buf_ptr = buf; + + for (i = VERITY_TABLE_ARGS; i < (VERITY_TABLE_ARGS + + VERITY_TABLE_OPT_FEC_ARGS + 2); i++) { + verity_table_args[i] = strsep(&buf_ptr, " "); + if (verity_table_args[i] == NULL) { + no_of_args = i; + break; + } + } + + err = verity_ctr(ti, no_of_args, verity_table_args); + + if (err) + DMERR("android-verity failed to mount as verity target"); + else { + target_added = true; + DMINFO("android-verity mounted as verity target"); + } + +free_metadata: + if (metadata) { + kfree(metadata->header); + kfree(metadata->verity_table); + } + kfree(metadata); + return err; +} + +static int __init dm_android_verity_init(void) +{ + int r; + struct dentry *file; + + r = dm_register_target(&android_verity_target); + if (r < 0) + DMERR("register failed %d", r); + + /* Tracks the status of the last added target */ + debug_dir = debugfs_create_dir("android_verity", NULL); + + if (IS_ERR_OR_NULL(debug_dir)) { + DMERR("Cannot create android_verity debugfs directory: %ld", + PTR_ERR(debug_dir)); + goto end; + } + + file = debugfs_create_bool("target_added", S_IRUGO, debug_dir, + &target_added); + + if (IS_ERR_OR_NULL(file)) { + DMERR("Cannot create android_verity debugfs directory: %ld", + PTR_ERR(debug_dir)); + debugfs_remove_recursive(debug_dir); + goto end; + } + + file = debugfs_create_bool("verity_enabled", S_IRUGO, debug_dir, + &verity_enabled); + + if (IS_ERR_OR_NULL(file)) { + DMERR("Cannot create android_verity debugfs directory: %ld", + PTR_ERR(debug_dir)); + debugfs_remove_recursive(debug_dir); + } + +end: + return r; +} + +static void __exit dm_android_verity_exit(void) +{ + if (!IS_ERR_OR_NULL(debug_dir)) + debugfs_remove_recursive(debug_dir); + + dm_unregister_target(&android_verity_target); +} + +module_init(dm_android_verity_init); +module_exit(dm_android_verity_exit); diff --git a/drivers/md/dm-android-verity.h b/drivers/md/dm-android-verity.h new file mode 100644 index 00000000000000..c8d7ab642780d4 --- /dev/null +++ b/drivers/md/dm-android-verity.h @@ -0,0 +1,123 @@ +/* + * Copyright (C) 2015 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#ifndef DM_ANDROID_VERITY_H +#define DM_ANDROID_VERITY_H + +#include + +#define RSANUMBYTES 256 +#define VERITY_METADATA_MAGIC_NUMBER 0xb001b001 +#define VERITY_METADATA_MAGIC_DISABLE 0x46464f56 +#define VERITY_METADATA_VERSION 0 +#define VERITY_STATE_DISABLE 1 +#define DATA_BLOCK_SIZE (4 * 1024) +#define VERITY_METADATA_SIZE (8 * DATA_BLOCK_SIZE) +#define VERITY_TABLE_ARGS 10 +#define VERITY_COMMANDLINE_PARAM_LENGTH 20 +#define BUILD_VARIANT 20 + +/* + * : is the format for the identifier. + * subject can either be the Common Name(CN) + Organization Name(O) or + * just the CN if the it is prefixed with O + * From https://tools.ietf.org/html/rfc5280#appendix-A + * ub-organization-name-length INTEGER ::= 64 + * ub-common-name-length INTEGER ::= 64 + * + * http://lxr.free-electrons.com/source/crypto/asymmetric_keys/x509_cert_parser.c?v=3.9#L278 + * ctx->o_size + 2 + ctx->cn_size + 1 + * + 41 characters for ":" and sha1 id + * 64 + 2 + 64 + 1 + 1 + 40 (172) + * setting VERITY_DEFAULT_KEY_ID_LENGTH to 200 characters. + */ +#define VERITY_DEFAULT_KEY_ID_LENGTH 200 + +#define FEC_MAGIC 0xFECFECFE +#define FEC_BLOCK_SIZE (4 * 1024) +#define FEC_VERSION 0 +#define FEC_RSM 255 +#define FEC_ARG_LENGTH 300 + +#define VERITY_TABLE_OPT_RESTART "restart_on_corruption" +#define VERITY_TABLE_OPT_LOGGING "ignore_corruption" +#define VERITY_TABLE_OPT_IGNZERO "ignore_zero_blocks" + +#define VERITY_TABLE_OPT_FEC_FORMAT \ + "use_fec_from_device %s fec_start %llu fec_blocks %llu fec_roots %u ignore_zero_blocks" +#define VERITY_TABLE_OPT_FEC_ARGS 9 + +#define VERITY_DEBUG 0 + +#define DM_MSG_PREFIX "android-verity" + +#define DM_LINEAR_ARGS 2 +#define DM_LINEAR_TARGET_OFFSET "0" + +/* + * There can be two formats. + * if fec is present + * + * if fec is not present + * + */ +struct fec_header { + __le32 magic; + __le32 version; + __le32 size; + __le32 roots; + __le32 fec_size; + __le64 inp_size; + u8 hash[SHA256_DIGEST_SIZE]; +} __attribute__((packed)); + +struct android_metadata_header { + __le32 magic_number; + __le32 protocol_version; + char signature[RSANUMBYTES]; + __le32 table_length; +}; + +struct android_metadata { + struct android_metadata_header *header; + char *verity_table; +}; + +struct fec_ecc_metadata { + bool valid; + u32 roots; + u64 blocks; + u64 rounds; + u64 start; +}; + +struct bio_read { + struct page **page_io; + int number_of_pages; +}; + +extern struct target_type linear_target; + +extern void dm_linear_dtr(struct dm_target *ti); +extern int dm_linear_map(struct dm_target *ti, struct bio *bio); +extern void dm_linear_status(struct dm_target *ti, status_type_t type, + unsigned status_flags, char *result, unsigned maxlen); +extern int dm_linear_prepare_ioctl(struct dm_target *ti, + struct block_device **bdev, fmode_t *mode); +extern int dm_linear_iterate_devices(struct dm_target *ti, + iterate_devices_callout_fn fn, void *data); +extern int dm_linear_ctr(struct dm_target *ti, unsigned int argc, char **argv); +extern long dm_linear_direct_access(struct dm_target *ti, sector_t sector, + void **kaddr, pfn_t *pfn, long size); +#endif /* DM_ANDROID_VERITY_H */ diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c index a2768835d3948a..7a5b75fd39d6f9 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c @@ -1503,12 +1503,15 @@ static int crypt_set_key(struct crypt_config *cc, char *key) if (!cc->key_size && strcmp(key, "-")) goto out; + /* clear the flag since following operations may invalidate previously valid key */ + clear_bit(DM_CRYPT_KEY_VALID, &cc->flags); + if (cc->key_size && crypt_decode_key(cc->key, key, cc->key_size) < 0) goto out; - set_bit(DM_CRYPT_KEY_VALID, &cc->flags); - r = crypt_setkey_allcpus(cc); + if (!r) + set_bit(DM_CRYPT_KEY_VALID, &cc->flags); out: /* Hex key string not needed after here, so wipe it. */ @@ -1863,16 +1866,24 @@ static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv) } ret = -ENOMEM; - cc->io_queue = alloc_workqueue("kcryptd_io", WQ_MEM_RECLAIM, 1); + cc->io_queue = alloc_workqueue("kcryptd_io", + WQ_HIGHPRI | + WQ_MEM_RECLAIM, + 1); if (!cc->io_queue) { ti->error = "Couldn't create kcryptd io queue"; goto bad; } if (test_bit(DM_CRYPT_SAME_CPU, &cc->flags)) - cc->crypt_queue = alloc_workqueue("kcryptd", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 1); + cc->crypt_queue = alloc_workqueue("kcryptd", + WQ_HIGHPRI | + WQ_MEM_RECLAIM, 1); else - cc->crypt_queue = alloc_workqueue("kcryptd", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM | WQ_UNBOUND, + cc->crypt_queue = alloc_workqueue("kcryptd", + WQ_HIGHPRI | + WQ_MEM_RECLAIM | + WQ_UNBOUND, num_online_cpus()); if (!cc->crypt_queue) { ti->error = "Couldn't create kcryptd queue"; diff --git a/drivers/md/dm-flakey.c b/drivers/md/dm-flakey.c index 6a2e8dd44a1b83..3643cba713518e 100644 --- a/drivers/md/dm-flakey.c +++ b/drivers/md/dm-flakey.c @@ -200,11 +200,13 @@ static int flakey_ctr(struct dm_target *ti, unsigned int argc, char **argv) if (!(fc->up_interval + fc->down_interval)) { ti->error = "Total (up + down) interval is zero"; + r = -EINVAL; goto bad; } if (fc->up_interval + fc->down_interval < fc->up_interval) { ti->error = "Interval overflow"; + r = -EINVAL; goto bad; } diff --git a/drivers/md/dm-ioctl.c b/drivers/md/dm-ioctl.c index 966eb4b61aed56..89ec6d26881ba0 100644 --- a/drivers/md/dm-ioctl.c +++ b/drivers/md/dm-ioctl.c @@ -1927,6 +1927,45 @@ void dm_interface_exit(void) dm_hash_exit(); } + +/** + * dm_ioctl_export - Permanently export a mapped device via the ioctl interface + * @md: Pointer to mapped_device + * @name: Buffer (size DM_NAME_LEN) for name + * @uuid: Buffer (size DM_UUID_LEN) for uuid or NULL if not desired + */ +int dm_ioctl_export(struct mapped_device *md, const char *name, + const char *uuid) +{ + int r = 0; + struct hash_cell *hc; + + if (!md) { + r = -ENXIO; + goto out; + } + + /* The name and uuid can only be set once. */ + mutex_lock(&dm_hash_cells_mutex); + hc = dm_get_mdptr(md); + mutex_unlock(&dm_hash_cells_mutex); + if (hc) { + DMERR("%s: already exported", dm_device_name(md)); + r = -ENXIO; + goto out; + } + + r = dm_hash_insert(name, uuid, md); + if (r) { + DMERR("%s: could not bind to '%s'", dm_device_name(md), name); + goto out; + } + + /* Let udev know we've changed. */ + dm_kobject_uevent(md, KOBJ_CHANGE, dm_get_event_nr(md)); +out: + return r; +} /** * dm_copy_name_and_uuid - Copy mapped device name & uuid into supplied buffers * @md: Pointer to mapped_device diff --git a/drivers/md/dm-linear.c b/drivers/md/dm-linear.c index 4788b0b989a9ba..4ad62d680547ab 100644 --- a/drivers/md/dm-linear.c +++ b/drivers/md/dm-linear.c @@ -25,7 +25,7 @@ struct linear_c { /* * Construct a linear mapping: */ -static int linear_ctr(struct dm_target *ti, unsigned int argc, char **argv) +int dm_linear_ctr(struct dm_target *ti, unsigned int argc, char **argv) { struct linear_c *lc; unsigned long long tmp; @@ -66,14 +66,16 @@ static int linear_ctr(struct dm_target *ti, unsigned int argc, char **argv) kfree(lc); return ret; } +EXPORT_SYMBOL_GPL(dm_linear_ctr); -static void linear_dtr(struct dm_target *ti) +void dm_linear_dtr(struct dm_target *ti) { struct linear_c *lc = (struct linear_c *) ti->private; dm_put_device(ti, lc->dev); kfree(lc); } +EXPORT_SYMBOL_GPL(dm_linear_dtr); static sector_t linear_map_sector(struct dm_target *ti, sector_t bi_sector) { @@ -92,14 +94,15 @@ static void linear_map_bio(struct dm_target *ti, struct bio *bio) linear_map_sector(ti, bio->bi_iter.bi_sector); } -static int linear_map(struct dm_target *ti, struct bio *bio) +int dm_linear_map(struct dm_target *ti, struct bio *bio) { linear_map_bio(ti, bio); return DM_MAPIO_REMAPPED; } +EXPORT_SYMBOL_GPL(dm_linear_map); -static void linear_status(struct dm_target *ti, status_type_t type, +void dm_linear_status(struct dm_target *ti, status_type_t type, unsigned status_flags, char *result, unsigned maxlen) { struct linear_c *lc = (struct linear_c *) ti->private; @@ -115,8 +118,9 @@ static void linear_status(struct dm_target *ti, status_type_t type, break; } } +EXPORT_SYMBOL_GPL(dm_linear_status); -static int linear_prepare_ioctl(struct dm_target *ti, +int dm_linear_prepare_ioctl(struct dm_target *ti, struct block_device **bdev, fmode_t *mode) { struct linear_c *lc = (struct linear_c *) ti->private; @@ -132,16 +136,18 @@ static int linear_prepare_ioctl(struct dm_target *ti, return 1; return 0; } +EXPORT_SYMBOL_GPL(dm_linear_prepare_ioctl); -static int linear_iterate_devices(struct dm_target *ti, +int dm_linear_iterate_devices(struct dm_target *ti, iterate_devices_callout_fn fn, void *data) { struct linear_c *lc = ti->private; return fn(ti, lc->dev, lc->start, ti->len, data); } +EXPORT_SYMBOL_GPL(dm_linear_iterate_devices); -static long linear_direct_access(struct dm_target *ti, sector_t sector, +long dm_linear_direct_access(struct dm_target *ti, sector_t sector, void **kaddr, pfn_t *pfn, long size) { struct linear_c *lc = ti->private; @@ -158,18 +164,19 @@ static long linear_direct_access(struct dm_target *ti, sector_t sector, return ret; } +EXPORT_SYMBOL_GPL(dm_linear_direct_access); static struct target_type linear_target = { .name = "linear", .version = {1, 3, 0}, .module = THIS_MODULE, - .ctr = linear_ctr, - .dtr = linear_dtr, - .map = linear_map, - .status = linear_status, - .prepare_ioctl = linear_prepare_ioctl, - .iterate_devices = linear_iterate_devices, - .direct_access = linear_direct_access, + .ctr = dm_linear_ctr, + .dtr = dm_linear_dtr, + .map = dm_linear_map, + .status = dm_linear_status, + .prepare_ioctl = dm_linear_prepare_ioctl, + .iterate_devices = dm_linear_iterate_devices, + .direct_access = dm_linear_direct_access, }; int __init dm_linear_init(void) diff --git a/drivers/md/dm-raid.c b/drivers/md/dm-raid.c index 6d53810963f753..af2d79b52484b6 100644 --- a/drivers/md/dm-raid.c +++ b/drivers/md/dm-raid.c @@ -2994,6 +2994,9 @@ static int raid_ctr(struct dm_target *ti, unsigned int argc, char **argv) } } + /* Disable/enable discard support on raid set. */ + configure_discard_support(rs); + mddev_unlock(&rs->md); return 0; @@ -3580,12 +3583,6 @@ static int raid_preresume(struct dm_target *ti) if (test_bit(RT_FLAG_UPDATE_SBS, &rs->runtime_flags)) rs_update_sbs(rs); - /* - * Disable/enable discard support on raid set after any - * conversion, because devices can have been added - */ - configure_discard_support(rs); - /* Load the bitmap from disk unless raid0 */ r = __load_dirty_region_bitmap(rs); if (r) diff --git a/drivers/md/dm-rq.c b/drivers/md/dm-rq.c index 1d0d2adc050a55..2c965424d383fc 100644 --- a/drivers/md/dm-rq.c +++ b/drivers/md/dm-rq.c @@ -226,6 +226,9 @@ static void rq_end_stats(struct mapped_device *md, struct request *orig) */ static void rq_completed(struct mapped_device *md, int rw, bool run_queue) { + struct request_queue *q = md->queue; + unsigned long flags; + atomic_dec(&md->pending[rw]); /* nudge anyone waiting on suspend queue */ @@ -238,8 +241,11 @@ static void rq_completed(struct mapped_device *md, int rw, bool run_queue) * back into ->request_fn() could deadlock attempting to grab the * queue lock again. */ - if (!md->queue->mq_ops && run_queue) - blk_run_queue_async(md->queue); + if (!q->mq_ops && run_queue) { + spin_lock_irqsave(q->queue_lock, flags); + blk_run_queue_async(q); + spin_unlock_irqrestore(q->queue_lock, flags); + } /* * dm_put() must be at the end of this function. See the comment above @@ -798,6 +804,10 @@ static void dm_old_request_fn(struct request_queue *q) int srcu_idx; struct dm_table *map = dm_get_live_table(md, &srcu_idx); + if (unlikely(!map)) { + dm_put_live_table(md, srcu_idx); + return; + } ti = dm_table_find_target(map, pos); dm_put_live_table(md, srcu_idx); } diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c index c4b53b332607be..d837a289429ac3 100644 --- a/drivers/md/dm-table.c +++ b/drivers/md/dm-table.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -924,12 +925,6 @@ static int dm_table_determine_type(struct dm_table *t) BUG_ON(!request_based); /* No targets in this table */ - if (list_empty(devices) && __table_type_request_based(live_md_type)) { - /* inherit live MD type */ - t->type = live_md_type; - return 0; - } - /* * The only way to establish DM_TYPE_MQ_REQUEST_BASED is by * having a compatible target use dm_table_set_type. @@ -948,6 +943,19 @@ static int dm_table_determine_type(struct dm_table *t) return -EINVAL; } + if (list_empty(devices)) { + int srcu_idx; + struct dm_table *live_table = dm_get_live_table(t->md, &srcu_idx); + + /* inherit live table's type and all_blk_mq */ + if (live_table) { + t->type = live_table->type; + t->all_blk_mq = live_table->all_blk_mq; + } + dm_put_live_table(t->md, srcu_idx); + return 0; + } + /* Non-request-stackable devices can't be used for request-based dm */ list_for_each_entry(dd, devices, list) { struct request_queue *q = bdev_get_queue(dd->dm_dev->bdev); @@ -974,6 +982,11 @@ static int dm_table_determine_type(struct dm_table *t) t->all_blk_mq = true; } + if (t->type == DM_TYPE_MQ_REQUEST_BASED && !t->all_blk_mq) { + DMERR("table load rejected: all devices are not blk-mq request-stackable"); + return -EINVAL; + } + return 0; } diff --git a/drivers/md/dm-verity-fec.c b/drivers/md/dm-verity-fec.c index 0f0eb8a3d922a2..a8d4d2fd88e776 100644 --- a/drivers/md/dm-verity-fec.c +++ b/drivers/md/dm-verity-fec.c @@ -11,6 +11,7 @@ #include "dm-verity-fec.h" #include +#include #define DM_MSG_PREFIX "verity-fec" @@ -175,9 +176,11 @@ static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio, if (r < 0 && neras) DMERR_LIMIT("%s: FEC %llu: failed to correct: %d", v->data_dev->name, (unsigned long long)rsb, r); - else if (r > 0) + else if (r > 0) { DMWARN_LIMIT("%s: FEC %llu: corrected %d errors", v->data_dev->name, (unsigned long long)rsb, r); + atomic_add_unless(&v->fec->corrected, 1, INT_MAX); + } return r; } @@ -439,6 +442,13 @@ int verity_fec_decode(struct dm_verity *v, struct dm_verity_io *io, if (!verity_fec_is_enabled(v)) return -EOPNOTSUPP; + if (fio->level >= DM_VERITY_FEC_MAX_RECURSION) { + DMWARN_LIMIT("%s: FEC: recursion too deep", v->data_dev->name); + return -EIO; + } + + fio->level++; + if (type == DM_VERITY_BLOCK_TYPE_METADATA) block += v->data_blocks; @@ -470,7 +480,7 @@ int verity_fec_decode(struct dm_verity *v, struct dm_verity_io *io, if (r < 0) { r = fec_decode_rsb(v, io, fio, rsb, offset, true); if (r < 0) - return r; + goto done; } if (dest) @@ -480,6 +490,8 @@ int verity_fec_decode(struct dm_verity *v, struct dm_verity_io *io, r = verity_for_bv_block(v, io, iter, fec_bv_copy); } +done: + fio->level--; return r; } @@ -520,6 +532,7 @@ void verity_fec_init_io(struct dm_verity_io *io) memset(fio->bufs, 0, sizeof(fio->bufs)); fio->nbufs = 0; fio->output = NULL; + fio->level = 0; } /* @@ -546,6 +559,7 @@ unsigned verity_fec_status_table(struct dm_verity *v, unsigned sz, void verity_fec_dtr(struct dm_verity *v) { struct dm_verity_fec *f = v->fec; + struct kobject *kobj = &f->kobj_holder.kobj; if (!verity_fec_is_enabled(v)) goto out; @@ -562,6 +576,12 @@ void verity_fec_dtr(struct dm_verity *v) if (f->dev) dm_put_device(v->ti, f->dev); + + if (kobj->state_initialized) { + kobject_put(kobj); + wait_for_completion(dm_get_completion_from_kobject(kobj)); + } + out: kfree(f); v->fec = NULL; @@ -650,6 +670,28 @@ int verity_fec_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v, return 0; } +static ssize_t corrected_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + struct dm_verity_fec *f = container_of(kobj, struct dm_verity_fec, + kobj_holder.kobj); + + return sprintf(buf, "%d\n", atomic_read(&f->corrected)); +} + +static struct kobj_attribute attr_corrected = __ATTR_RO(corrected); + +static struct attribute *fec_attrs[] = { + &attr_corrected.attr, + NULL +}; + +static struct kobj_type fec_ktype = { + .sysfs_ops = &kobj_sysfs_ops, + .default_attrs = fec_attrs, + .release = dm_kobject_release +}; + /* * Allocate dm_verity_fec for v->fec. Must be called before verity_fec_ctr. */ @@ -673,8 +715,10 @@ int verity_fec_ctr_alloc(struct dm_verity *v) */ int verity_fec_ctr(struct dm_verity *v) { + int r; struct dm_verity_fec *f = v->fec; struct dm_target *ti = v->ti; + struct mapped_device *md = dm_table_get_md(ti->table); u64 hash_blocks; if (!verity_fec_is_enabled(v)) { @@ -682,6 +726,16 @@ int verity_fec_ctr(struct dm_verity *v) return 0; } + /* Create a kobject and sysfs attributes */ + init_completion(&f->kobj_holder.completion); + + r = kobject_init_and_add(&f->kobj_holder.kobj, &fec_ktype, + &disk_to_dev(dm_disk(md))->kobj, "%s", "fec"); + if (r) { + ti->error = "Cannot create kobject"; + return r; + } + /* * FEC is computed over data blocks, possible metadata, and * hash blocks. In other words, FEC covers total of fec_blocks diff --git a/drivers/md/dm-verity-fec.h b/drivers/md/dm-verity-fec.h index 7fa0298b995e9e..4db0cae262eb7c 100644 --- a/drivers/md/dm-verity-fec.h +++ b/drivers/md/dm-verity-fec.h @@ -12,6 +12,8 @@ #ifndef DM_VERITY_FEC_H #define DM_VERITY_FEC_H +#include "dm.h" +#include "dm-core.h" #include "dm-verity.h" #include @@ -27,6 +29,9 @@ #define DM_VERITY_FEC_BUF_MAX \ (1 << (PAGE_SHIFT - DM_VERITY_FEC_BUF_RS_BITS)) +/* maximum recursion level for verity_fec_decode */ +#define DM_VERITY_FEC_MAX_RECURSION 4 + #define DM_VERITY_OPT_FEC_DEV "use_fec_from_device" #define DM_VERITY_OPT_FEC_BLOCKS "fec_blocks" #define DM_VERITY_OPT_FEC_START "fec_start" @@ -48,6 +53,8 @@ struct dm_verity_fec { mempool_t *extra_pool; /* mempool for extra buffers */ mempool_t *output_pool; /* mempool for output */ struct kmem_cache *cache; /* cache for buffers */ + atomic_t corrected; /* corrected errors */ + struct dm_kobject_holder kobj_holder; /* for sysfs attributes */ }; /* per-bio data */ @@ -58,6 +65,7 @@ struct dm_verity_fec_io { unsigned nbufs; /* number of buffers allocated */ u8 *output; /* buffer for corrected output */ size_t output_pos; + unsigned level; /* recursion level */ }; #ifdef CONFIG_DM_VERITY_FEC diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c index 0aba34a7b3b3fe..5d0a9963b10855 100644 --- a/drivers/md/dm-verity-target.c +++ b/drivers/md/dm-verity-target.c @@ -501,6 +501,7 @@ static void verity_prefetch_io(struct work_struct *work) container_of(work, struct dm_verity_prefetch_work, work); struct dm_verity *v = pw->v; int i; + sector_t prefetch_size; for (i = v->levels - 2; i >= 0; i--) { sector_t hash_block_start; @@ -523,8 +524,14 @@ static void verity_prefetch_io(struct work_struct *work) hash_block_end = v->hash_blocks - 1; } no_prefetch_cluster: + // for emmc, it is more efficient to send bigger read + prefetch_size = max((sector_t)CONFIG_DM_VERITY_HASH_PREFETCH_MIN_SIZE, + hash_block_end - hash_block_start + 1); + if ((hash_block_start + prefetch_size) >= (v->hash_start + v->hash_blocks)) { + prefetch_size = hash_block_end - hash_block_start + 1; + } dm_bufio_prefetch(v->bufio, hash_block_start, - hash_block_end - hash_block_start + 1); + prefetch_size); } kfree(pw); @@ -551,7 +558,7 @@ static void verity_submit_prefetch(struct dm_verity *v, struct dm_verity_io *io) * Bio map function. It allocates dm_verity_io structure and bio vector and * fills them. Then it issues prefetches and the I/O. */ -static int verity_map(struct dm_target *ti, struct bio *bio) +int verity_map(struct dm_target *ti, struct bio *bio) { struct dm_verity *v = ti->private; struct dm_verity_io *io; @@ -592,11 +599,12 @@ static int verity_map(struct dm_target *ti, struct bio *bio) return DM_MAPIO_SUBMITTED; } +EXPORT_SYMBOL_GPL(verity_map); /* * Status: V (valid) or C (corruption found) */ -static void verity_status(struct dm_target *ti, status_type_t type, +void verity_status(struct dm_target *ti, status_type_t type, unsigned status_flags, char *result, unsigned maxlen) { struct dm_verity *v = ti->private; @@ -655,8 +663,9 @@ static void verity_status(struct dm_target *ti, status_type_t type, break; } } +EXPORT_SYMBOL_GPL(verity_status); -static int verity_prepare_ioctl(struct dm_target *ti, +int verity_prepare_ioctl(struct dm_target *ti, struct block_device **bdev, fmode_t *mode) { struct dm_verity *v = ti->private; @@ -668,16 +677,18 @@ static int verity_prepare_ioctl(struct dm_target *ti, return 1; return 0; } +EXPORT_SYMBOL_GPL(verity_prepare_ioctl); -static int verity_iterate_devices(struct dm_target *ti, +int verity_iterate_devices(struct dm_target *ti, iterate_devices_callout_fn fn, void *data) { struct dm_verity *v = ti->private; return fn(ti, v->data_dev, v->data_start, ti->len, data); } +EXPORT_SYMBOL_GPL(verity_iterate_devices); -static void verity_io_hints(struct dm_target *ti, struct queue_limits *limits) +void verity_io_hints(struct dm_target *ti, struct queue_limits *limits) { struct dm_verity *v = ti->private; @@ -689,8 +700,9 @@ static void verity_io_hints(struct dm_target *ti, struct queue_limits *limits) blk_limits_io_min(limits, limits->logical_block_size); } +EXPORT_SYMBOL_GPL(verity_io_hints); -static void verity_dtr(struct dm_target *ti) +void verity_dtr(struct dm_target *ti) { struct dm_verity *v = ti->private; @@ -719,6 +731,7 @@ static void verity_dtr(struct dm_target *ti) kfree(v); } +EXPORT_SYMBOL_GPL(verity_dtr); static int verity_alloc_zero_digest(struct dm_verity *v) { @@ -817,7 +830,7 @@ static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v) * * Hex string or "-" if no salt. */ -static int verity_ctr(struct dm_target *ti, unsigned argc, char **argv) +int verity_ctr(struct dm_target *ti, unsigned argc, char **argv) { struct dm_verity *v; struct dm_arg_set as; @@ -1053,6 +1066,7 @@ static int verity_ctr(struct dm_target *ti, unsigned argc, char **argv) return r; } +EXPORT_SYMBOL_GPL(verity_ctr); static struct target_type verity_target = { .name = "verity", diff --git a/drivers/md/dm-verity.h b/drivers/md/dm-verity.h index fb419f422d7316..75effca400a3b1 100644 --- a/drivers/md/dm-verity.h +++ b/drivers/md/dm-verity.h @@ -126,4 +126,14 @@ extern int verity_hash(struct dm_verity *v, struct shash_desc *desc, extern int verity_hash_for_block(struct dm_verity *v, struct dm_verity_io *io, sector_t block, u8 *digest, bool *is_zero); +extern void verity_status(struct dm_target *ti, status_type_t type, + unsigned status_flags, char *result, unsigned maxlen); +extern int verity_prepare_ioctl(struct dm_target *ti, + struct block_device **bdev, fmode_t *mode); +extern int verity_iterate_devices(struct dm_target *ti, + iterate_devices_callout_fn fn, void *data); +extern void verity_io_hints(struct dm_target *ti, struct queue_limits *limits); +extern void verity_dtr(struct dm_target *ti); +extern int verity_ctr(struct dm_target *ti, unsigned argc, char **argv); +extern int verity_map(struct dm_target *ti, struct bio *bio); #endif /* DM_VERITY_H */ diff --git a/drivers/md/md.c b/drivers/md/md.c index 2089d46b0eb89c..24925f2aa23539 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -6829,7 +6829,7 @@ static int md_ioctl(struct block_device *bdev, fmode_t mode, /* need to ensure recovery thread has run */ wait_event_interruptible_timeout(mddev->sb_wait, !test_bit(MD_RECOVERY_NEEDED, - &mddev->flags), + &mddev->recovery), msecs_to_jiffies(5000)); if (cmd == STOP_ARRAY || cmd == STOP_ARRAY_RO) { /* Need to flush page cache, and ensure no-one else opens @@ -7092,7 +7092,8 @@ static int md_open(struct block_device *bdev, fmode_t mode) if (test_bit(MD_CLOSING, &mddev->flags)) { mutex_unlock(&mddev->open_mutex); - return -ENODEV; + err = -ENODEV; + goto out; } err = 0; @@ -7101,6 +7102,8 @@ static int md_open(struct block_device *bdev, fmode_t mode) check_disk_change(bdev); out: + if (err) + mddev_put(mddev); return err; } diff --git a/drivers/md/persistent-data/dm-space-map-metadata.c b/drivers/md/persistent-data/dm-space-map-metadata.c index 7e44005595c1e7..20557e2c60c623 100644 --- a/drivers/md/persistent-data/dm-space-map-metadata.c +++ b/drivers/md/persistent-data/dm-space-map-metadata.c @@ -775,17 +775,15 @@ int dm_sm_metadata_create(struct dm_space_map *sm, memcpy(&smm->sm, &bootstrap_ops, sizeof(smm->sm)); r = sm_ll_new_metadata(&smm->ll, tm); + if (!r) { + if (nr_blocks > DM_SM_METADATA_MAX_BLOCKS) + nr_blocks = DM_SM_METADATA_MAX_BLOCKS; + r = sm_ll_extend(&smm->ll, nr_blocks); + } + memcpy(&smm->sm, &ops, sizeof(smm->sm)); if (r) return r; - if (nr_blocks > DM_SM_METADATA_MAX_BLOCKS) - nr_blocks = DM_SM_METADATA_MAX_BLOCKS; - r = sm_ll_extend(&smm->ll, nr_blocks); - if (r) - return r; - - memcpy(&smm->sm, &ops, sizeof(smm->sm)); - /* * Now we need to update the newly created data structures with the * allocated blocks that they were built from. diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index 92ac251e91e62e..cce6057b9aca5b 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -6984,6 +6984,15 @@ static int raid5_run(struct mddev *mddev) stripe = (stripe | (stripe-1)) + 1; mddev->queue->limits.discard_alignment = stripe; mddev->queue->limits.discard_granularity = stripe; + + /* + * We use 16-bit counter of active stripes in bi_phys_segments + * (minus one for over-loaded initialization) + */ + blk_queue_max_hw_sectors(mddev->queue, 0xfffe * STRIPE_SECTORS); + blk_queue_max_discard_sectors(mddev->queue, + 0xfffe * STRIPE_SECTORS); + /* * unaligned part of discard request will be ignored, so can't * guarantee discard_zeroes_data diff --git a/drivers/media/dvb-frontends/mn88472.c b/drivers/media/dvb-frontends/mn88472.c index 18fb2df1e2bd47..72650116732cb7 100644 --- a/drivers/media/dvb-frontends/mn88472.c +++ b/drivers/media/dvb-frontends/mn88472.c @@ -488,18 +488,6 @@ static int mn88472_probe(struct i2c_client *client, goto err_kfree; } - /* Check demod answers with correct chip id */ - ret = regmap_read(dev->regmap[0], 0xff, &utmp); - if (ret) - goto err_regmap_0_regmap_exit; - - dev_dbg(&client->dev, "chip id=%02x\n", utmp); - - if (utmp != 0x02) { - ret = -ENODEV; - goto err_regmap_0_regmap_exit; - } - /* * Chip has three I2C addresses for different register banks. Used * addresses are 0x18, 0x1a and 0x1c. We register two dummy clients, @@ -536,6 +524,18 @@ static int mn88472_probe(struct i2c_client *client, } i2c_set_clientdata(dev->client[2], dev); + /* Check demod answers with correct chip id */ + ret = regmap_read(dev->regmap[2], 0xff, &utmp); + if (ret) + goto err_regmap_2_regmap_exit; + + dev_dbg(&client->dev, "chip id=%02x\n", utmp); + + if (utmp != 0x02) { + ret = -ENODEV; + goto err_regmap_2_regmap_exit; + } + /* Sleep because chip is active by default */ ret = regmap_write(dev->regmap[2], 0x05, 0x3e); if (ret) diff --git a/drivers/media/dvb-frontends/mn88473.c b/drivers/media/dvb-frontends/mn88473.c index 451974a1d7ed46..2932bdc8fa94b8 100644 --- a/drivers/media/dvb-frontends/mn88473.c +++ b/drivers/media/dvb-frontends/mn88473.c @@ -485,18 +485,6 @@ static int mn88473_probe(struct i2c_client *client, goto err_kfree; } - /* Check demod answers with correct chip id */ - ret = regmap_read(dev->regmap[0], 0xff, &uitmp); - if (ret) - goto err_regmap_0_regmap_exit; - - dev_dbg(&client->dev, "chip id=%02x\n", uitmp); - - if (uitmp != 0x03) { - ret = -ENODEV; - goto err_regmap_0_regmap_exit; - } - /* * Chip has three I2C addresses for different register banks. Used * addresses are 0x18, 0x1a and 0x1c. We register two dummy clients, @@ -533,6 +521,18 @@ static int mn88473_probe(struct i2c_client *client, } i2c_set_clientdata(dev->client[2], dev); + /* Check demod answers with correct chip id */ + ret = regmap_read(dev->regmap[2], 0xff, &uitmp); + if (ret) + goto err_regmap_2_regmap_exit; + + dev_dbg(&client->dev, "chip id=%02x\n", uitmp); + + if (uitmp != 0x03) { + ret = -ENODEV; + goto err_regmap_2_regmap_exit; + } + /* Sleep because chip is active by default */ ret = regmap_write(dev->regmap[2], 0x05, 0x3e); if (ret) diff --git a/drivers/media/i2c/Kconfig b/drivers/media/i2c/Kconfig index 2669b4bad91068..5a27bffa02fb5a 100644 --- a/drivers/media/i2c/Kconfig +++ b/drivers/media/i2c/Kconfig @@ -655,6 +655,7 @@ config VIDEO_S5K6A3 config VIDEO_S5K4ECGX tristate "Samsung S5K4ECGX sensor support" depends on I2C && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API + select CRC32 ---help--- This is a V4L2 sensor-level driver for Samsung S5K4ECGX 5M camera sensor with an embedded SoC image signal processor. diff --git a/drivers/media/i2c/tvp5150.c b/drivers/media/i2c/tvp5150.c index 4740da39d6988a..59aa4dafb60b0d 100644 --- a/drivers/media/i2c/tvp5150.c +++ b/drivers/media/i2c/tvp5150.c @@ -288,8 +288,12 @@ static inline void tvp5150_selmux(struct v4l2_subdev *sd) tvp5150_write(sd, TVP5150_OP_MODE_CTL, opmode); tvp5150_write(sd, TVP5150_VD_IN_SRC_SEL_1, input); - /* Svideo should enable YCrCb output and disable GPCL output - * For Composite and TV, it should be the reverse + /* + * Setup the FID/GLCO/VLK/HVLK and INTREQ/GPCL/VBLK output signals. For + * S-Video we output the vertical lock (VLK) signal on FID/GLCO/VLK/HVLK + * and set INTREQ/GPCL/VBLK to logic 0. For composite we output the + * field indicator (FID) signal on FID/GLCO/VLK/HVLK and set + * INTREQ/GPCL/VBLK to logic 1. */ val = tvp5150_read(sd, TVP5150_MISC_CTL); if (val < 0) { @@ -298,9 +302,9 @@ static inline void tvp5150_selmux(struct v4l2_subdev *sd) } if (decoder->input == TVP5150_SVIDEO) - val = (val & ~0x40) | 0x10; + val = (val & ~TVP5150_MISC_CTL_GPCL) | TVP5150_MISC_CTL_HVLK; else - val = (val & ~0x10) | 0x40; + val = (val & ~TVP5150_MISC_CTL_HVLK) | TVP5150_MISC_CTL_GPCL; tvp5150_write(sd, TVP5150_MISC_CTL, val); }; @@ -452,7 +456,12 @@ static const struct i2c_reg_value tvp5150_init_enable[] = { },{ /* Automatic offset and AGC enabled */ TVP5150_ANAL_CHL_CTL, 0x15 },{ /* Activate YCrCb output 0x9 or 0xd ? */ - TVP5150_MISC_CTL, 0x6f + TVP5150_MISC_CTL, TVP5150_MISC_CTL_GPCL | + TVP5150_MISC_CTL_INTREQ_OE | + TVP5150_MISC_CTL_YCBCR_OE | + TVP5150_MISC_CTL_SYNC_OE | + TVP5150_MISC_CTL_VBLANK | + TVP5150_MISC_CTL_CLOCK_OE, },{ /* Activates video std autodetection for all standards */ TVP5150_AUTOSW_MSK, 0x0 },{ /* Default format: 0x47. For 4:2:2: 0x40 */ @@ -815,6 +824,7 @@ static int tvp5150_s_ctrl(struct v4l2_ctrl *ctrl) return 0; case V4L2_CID_HUE: tvp5150_write(sd, TVP5150_HUE_CTL, ctrl->val); + break; case V4L2_CID_TEST_PATTERN: decoder->enable = ctrl->val ? false : true; tvp5150_selmux(sd); @@ -857,8 +867,6 @@ static int tvp5150_fill_fmt(struct v4l2_subdev *sd, f = &format->format; - tvp5150_reset(sd, 0); - f->width = decoder->rect.width; f->height = decoder->rect.height / 2; @@ -1047,21 +1055,27 @@ static const struct media_entity_operations tvp5150_sd_media_ops = { static int tvp5150_s_stream(struct v4l2_subdev *sd, int enable) { struct tvp5150 *decoder = to_tvp5150(sd); - /* Output format: 8-bit ITU-R BT.656 with embedded syncs */ - int val = 0x09; - - /* Output format: 8-bit 4:2:2 YUV with discrete sync */ - if (decoder->mbus_type == V4L2_MBUS_PARALLEL) - val = 0x0d; + int val; - /* Initializes TVP5150 to its default values */ - /* # set PCLK (27MHz) */ - tvp5150_write(sd, TVP5150_CONF_SHARED_PIN, 0x00); + /* Enable or disable the video output signals. */ + val = tvp5150_read(sd, TVP5150_MISC_CTL); + if (val < 0) + return val; + + val &= ~(TVP5150_MISC_CTL_YCBCR_OE | TVP5150_MISC_CTL_SYNC_OE | + TVP5150_MISC_CTL_CLOCK_OE); + + if (enable) { + /* + * Enable the YCbCr and clock outputs. In discrete sync mode + * (non-BT.656) additionally enable the the sync outputs. + */ + val |= TVP5150_MISC_CTL_YCBCR_OE | TVP5150_MISC_CTL_CLOCK_OE; + if (decoder->mbus_type == V4L2_MBUS_PARALLEL) + val |= TVP5150_MISC_CTL_SYNC_OE; + } - if (enable) - tvp5150_write(sd, TVP5150_MISC_CTL, val); - else - tvp5150_write(sd, TVP5150_MISC_CTL, 0x00); + tvp5150_write(sd, TVP5150_MISC_CTL, val); return 0; } @@ -1520,7 +1534,6 @@ static int tvp5150_probe(struct i2c_client *c, res = core->hdl.error; goto err; } - v4l2_ctrl_handler_setup(&core->hdl); /* Default is no cropping */ core->rect.top = 0; @@ -1531,6 +1544,8 @@ static int tvp5150_probe(struct i2c_client *c, core->rect.left = 0; core->rect.width = TVP5150_H_MAX; + tvp5150_reset(sd, 0); /* Calls v4l2_ctrl_handler_setup() */ + res = v4l2_async_register_subdev(sd); if (res < 0) goto err; diff --git a/drivers/media/i2c/tvp5150_reg.h b/drivers/media/i2c/tvp5150_reg.h index 25a99494491870..30a48c28d05ab5 100644 --- a/drivers/media/i2c/tvp5150_reg.h +++ b/drivers/media/i2c/tvp5150_reg.h @@ -9,6 +9,15 @@ #define TVP5150_ANAL_CHL_CTL 0x01 /* Analog channel controls */ #define TVP5150_OP_MODE_CTL 0x02 /* Operation mode controls */ #define TVP5150_MISC_CTL 0x03 /* Miscellaneous controls */ +#define TVP5150_MISC_CTL_VBLK_GPCL BIT(7) +#define TVP5150_MISC_CTL_GPCL BIT(6) +#define TVP5150_MISC_CTL_INTREQ_OE BIT(5) +#define TVP5150_MISC_CTL_HVLK BIT(4) +#define TVP5150_MISC_CTL_YCBCR_OE BIT(3) +#define TVP5150_MISC_CTL_SYNC_OE BIT(2) +#define TVP5150_MISC_CTL_VBLANK BIT(1) +#define TVP5150_MISC_CTL_CLOCK_OE BIT(0) + #define TVP5150_AUTOSW_MSK 0x04 /* Autoswitch mask: TVP5150A / TVP5150AM */ /* Reserved 05h */ diff --git a/drivers/media/pci/solo6x10/solo6x10.h b/drivers/media/pci/solo6x10/solo6x10.h index 5bd498735a6655..3f8da5e8c43019 100644 --- a/drivers/media/pci/solo6x10/solo6x10.h +++ b/drivers/media/pci/solo6x10/solo6x10.h @@ -284,7 +284,10 @@ static inline u32 solo_reg_read(struct solo_dev *solo_dev, int reg) static inline void solo_reg_write(struct solo_dev *solo_dev, int reg, u32 data) { + u16 val; + writel(data, solo_dev->reg_base + reg); + pci_read_config_word(solo_dev->pdev, PCI_STATUS, &val); } static inline void solo_irq_on(struct solo_dev *dev, u32 mask) diff --git a/drivers/media/platform/Kconfig b/drivers/media/platform/Kconfig index ce4a96fccc433c..5ff803efdc03da 100644 --- a/drivers/media/platform/Kconfig +++ b/drivers/media/platform/Kconfig @@ -93,7 +93,7 @@ config VIDEO_OMAP3_DEBUG config VIDEO_PXA27x tristate "PXA27x Quick Capture Interface driver" - depends on VIDEO_DEV && HAS_DMA + depends on VIDEO_DEV && VIDEO_V4L2 && HAS_DMA depends on PXA27x || COMPILE_TEST select VIDEOBUF2_DMA_SG select SG_SPLIT diff --git a/drivers/media/platform/blackfin/ppi.c b/drivers/media/platform/blackfin/ppi.c index cff63e511e6d9a..b8f3d9fa66e907 100644 --- a/drivers/media/platform/blackfin/ppi.c +++ b/drivers/media/platform/blackfin/ppi.c @@ -214,6 +214,8 @@ static int ppi_set_params(struct ppi_if *ppi, struct ppi_params *params) if (params->dlen > 24 || params->dlen <= 0) return -EINVAL; pctrl = devm_pinctrl_get(ppi->dev); + if (IS_ERR(pctrl)) + return PTR_ERR(pctrl); pstate = pinctrl_lookup_state(pctrl, pin_state[(params->dlen + 7) / 8 - 1]); if (pinctrl_select_state(pctrl, pstate)) diff --git a/drivers/media/platform/s5p-mfc/s5p_mfc.c b/drivers/media/platform/s5p-mfc/s5p_mfc.c index 0a5b8f5e011e7e..27e7cf65c2a77a 100644 --- a/drivers/media/platform/s5p-mfc/s5p_mfc.c +++ b/drivers/media/platform/s5p-mfc/s5p_mfc.c @@ -926,10 +926,11 @@ static int s5p_mfc_release(struct file *file) mfc_debug_enter(); if (dev) mutex_lock(&dev->mfc_mutex); - s5p_mfc_clock_on(); vb2_queue_release(&ctx->vq_src); vb2_queue_release(&ctx->vq_dst); if (dev) { + s5p_mfc_clock_on(); + /* Mark context as idle */ clear_work_bit_irqsave(ctx); /* @@ -951,9 +952,9 @@ static int s5p_mfc_release(struct file *file) if (s5p_mfc_power_off() < 0) mfc_err("Power off failed\n"); } + mfc_debug(2, "Shutting down clock\n"); + s5p_mfc_clock_off(); } - mfc_debug(2, "Shutting down clock\n"); - s5p_mfc_clock_off(); if (dev) dev->ctx[ctx->num] = NULL; s5p_mfc_dec_ctrls_delete(ctx); @@ -1082,6 +1083,7 @@ static struct device *s5p_mfc_alloc_memdev(struct device *dev, idx); if (ret == 0) return child; + device_del(child); } put_device(child); diff --git a/drivers/media/platform/sti/hva/hva-hw.c b/drivers/media/platform/sti/hva/hva-hw.c index d341d499452899..cf2a8d88453672 100644 --- a/drivers/media/platform/sti/hva/hva-hw.c +++ b/drivers/media/platform/sti/hva/hva-hw.c @@ -305,16 +305,16 @@ int hva_hw_probe(struct platform_device *pdev, struct hva_dev *hva) /* get memory for registers */ regs = platform_get_resource(pdev, IORESOURCE_MEM, 0); hva->regs = devm_ioremap_resource(dev, regs); - if (IS_ERR_OR_NULL(hva->regs)) { + if (IS_ERR(hva->regs)) { dev_err(dev, "%s failed to get regs\n", HVA_PREFIX); return PTR_ERR(hva->regs); } /* get memory for esram */ esram = platform_get_resource(pdev, IORESOURCE_MEM, 1); - if (IS_ERR_OR_NULL(esram)) { + if (!esram) { dev_err(dev, "%s failed to get esram\n", HVA_PREFIX); - return PTR_ERR(esram); + return -ENODEV; } hva->esram_addr = esram->start; hva->esram_size = resource_size(esram); diff --git a/drivers/media/rc/ite-cir.c b/drivers/media/rc/ite-cir.c index 0f301903aa6ffd..63165d324fffdb 100644 --- a/drivers/media/rc/ite-cir.c +++ b/drivers/media/rc/ite-cir.c @@ -263,6 +263,8 @@ static void ite_set_carrier_params(struct ite_dev *dev) if (allowance > ITE_RXDCR_MAX) allowance = ITE_RXDCR_MAX; + + use_demodulator = true; } } diff --git a/drivers/media/spi/gs1662.c b/drivers/media/spi/gs1662.c index d76f36233f4305..5143a90219c084 100644 --- a/drivers/media/spi/gs1662.c +++ b/drivers/media/spi/gs1662.c @@ -453,10 +453,9 @@ static int gs_probe(struct spi_device *spi) static int gs_remove(struct spi_device *spi) { struct v4l2_subdev *sd = spi_get_drvdata(spi); - struct gs *gs = to_gs(sd); v4l2_device_unregister_subdev(sd); - kfree(gs); + return 0; } diff --git a/drivers/media/usb/dvb-usb/dibusb-common.c b/drivers/media/usb/dvb-usb/dibusb-common.c index de3ee254747942..8207e6900656be 100644 --- a/drivers/media/usb/dvb-usb/dibusb-common.c +++ b/drivers/media/usb/dvb-usb/dibusb-common.c @@ -382,9 +382,9 @@ int dibusb_rc_query(struct dvb_usb_device *d, u32 *event, int *state) if (buf[0] != 0) deb_info("key: %*ph\n", 5, buf); +ret: kfree(buf); -ret: return ret; } EXPORT_SYMBOL(dibusb_rc_query); diff --git a/drivers/media/usb/dvb-usb/pctv452e.c b/drivers/media/usb/dvb-usb/pctv452e.c index 07fa08be9e994a..d54ebe7e02150f 100644 --- a/drivers/media/usb/dvb-usb/pctv452e.c +++ b/drivers/media/usb/dvb-usb/pctv452e.c @@ -97,14 +97,13 @@ struct pctv452e_state { u8 c; /* transaction counter, wraps around... */ u8 initialized; /* set to 1 if 0x15 has been sent */ u16 last_rc_key; - - unsigned char data[80]; }; static int tt3650_ci_msg(struct dvb_usb_device *d, u8 cmd, u8 *data, unsigned int write_len, unsigned int read_len) { struct pctv452e_state *state = (struct pctv452e_state *)d->priv; + u8 *buf; u8 id; unsigned int rlen; int ret; @@ -114,36 +113,39 @@ static int tt3650_ci_msg(struct dvb_usb_device *d, u8 cmd, u8 *data, return -EIO; } - mutex_lock(&state->ca_mutex); + buf = kmalloc(64, GFP_KERNEL); + if (!buf) + return -ENOMEM; + id = state->c++; - state->data[0] = SYNC_BYTE_OUT; - state->data[1] = id; - state->data[2] = cmd; - state->data[3] = write_len; + buf[0] = SYNC_BYTE_OUT; + buf[1] = id; + buf[2] = cmd; + buf[3] = write_len; - memcpy(state->data + 4, data, write_len); + memcpy(buf + 4, data, write_len); rlen = (read_len > 0) ? 64 : 0; - ret = dvb_usb_generic_rw(d, state->data, 4 + write_len, - state->data, rlen, /* delay_ms */ 0); + ret = dvb_usb_generic_rw(d, buf, 4 + write_len, + buf, rlen, /* delay_ms */ 0); if (0 != ret) goto failed; ret = -EIO; - if (SYNC_BYTE_IN != state->data[0] || id != state->data[1]) + if (SYNC_BYTE_IN != buf[0] || id != buf[1]) goto failed; - memcpy(data, state->data + 4, read_len); + memcpy(data, buf + 4, read_len); - mutex_unlock(&state->ca_mutex); + kfree(buf); return 0; failed: err("CI error %d; %02X %02X %02X -> %*ph.", - ret, SYNC_BYTE_OUT, id, cmd, 3, state->data); + ret, SYNC_BYTE_OUT, id, cmd, 3, buf); - mutex_unlock(&state->ca_mutex); + kfree(buf); return ret; } @@ -410,53 +412,57 @@ static int pctv452e_i2c_msg(struct dvb_usb_device *d, u8 addr, u8 *rcv_buf, u8 rcv_len) { struct pctv452e_state *state = (struct pctv452e_state *)d->priv; + u8 *buf; u8 id; int ret; - mutex_lock(&state->ca_mutex); + buf = kmalloc(64, GFP_KERNEL); + if (!buf) + return -ENOMEM; + id = state->c++; ret = -EINVAL; if (snd_len > 64 - 7 || rcv_len > 64 - 7) goto failed; - state->data[0] = SYNC_BYTE_OUT; - state->data[1] = id; - state->data[2] = PCTV_CMD_I2C; - state->data[3] = snd_len + 3; - state->data[4] = addr << 1; - state->data[5] = snd_len; - state->data[6] = rcv_len; + buf[0] = SYNC_BYTE_OUT; + buf[1] = id; + buf[2] = PCTV_CMD_I2C; + buf[3] = snd_len + 3; + buf[4] = addr << 1; + buf[5] = snd_len; + buf[6] = rcv_len; - memcpy(state->data + 7, snd_buf, snd_len); + memcpy(buf + 7, snd_buf, snd_len); - ret = dvb_usb_generic_rw(d, state->data, 7 + snd_len, - state->data, /* rcv_len */ 64, + ret = dvb_usb_generic_rw(d, buf, 7 + snd_len, + buf, /* rcv_len */ 64, /* delay_ms */ 0); if (ret < 0) goto failed; /* TT USB protocol error. */ ret = -EIO; - if (SYNC_BYTE_IN != state->data[0] || id != state->data[1]) + if (SYNC_BYTE_IN != buf[0] || id != buf[1]) goto failed; /* I2C device didn't respond as expected. */ ret = -EREMOTEIO; - if (state->data[5] < snd_len || state->data[6] < rcv_len) + if (buf[5] < snd_len || buf[6] < rcv_len) goto failed; - memcpy(rcv_buf, state->data + 7, rcv_len); - mutex_unlock(&state->ca_mutex); + memcpy(rcv_buf, buf + 7, rcv_len); + kfree(buf); return rcv_len; failed: err("I2C error %d; %02X %02X %02X %02X %02X -> %*ph", ret, SYNC_BYTE_OUT, id, addr << 1, snd_len, rcv_len, - 7, state->data); + 7, buf); - mutex_unlock(&state->ca_mutex); + kfree(buf); return ret; } @@ -505,7 +511,7 @@ static u32 pctv452e_i2c_func(struct i2c_adapter *adapter) static int pctv452e_power_ctrl(struct dvb_usb_device *d, int i) { struct pctv452e_state *state = (struct pctv452e_state *)d->priv; - u8 *rx; + u8 *b0, *rx; int ret; info("%s: %d\n", __func__, i); @@ -516,11 +522,12 @@ static int pctv452e_power_ctrl(struct dvb_usb_device *d, int i) if (state->initialized) return 0; - rx = kmalloc(PCTV_ANSWER_LEN, GFP_KERNEL); - if (!rx) + b0 = kmalloc(5 + PCTV_ANSWER_LEN, GFP_KERNEL); + if (!b0) return -ENOMEM; - mutex_lock(&state->ca_mutex); + rx = b0 + 5; + /* hmm where shoud this should go? */ ret = usb_set_interface(d->udev, 0, ISOC_INTERFACE_ALTERNATIVE); if (ret != 0) @@ -528,66 +535,70 @@ static int pctv452e_power_ctrl(struct dvb_usb_device *d, int i) __func__, ret); /* this is a one-time initialization, dont know where to put */ - state->data[0] = 0xaa; - state->data[1] = state->c++; - state->data[2] = PCTV_CMD_RESET; - state->data[3] = 1; - state->data[4] = 0; + b0[0] = 0xaa; + b0[1] = state->c++; + b0[2] = PCTV_CMD_RESET; + b0[3] = 1; + b0[4] = 0; /* reset board */ - ret = dvb_usb_generic_rw(d, state->data, 5, rx, PCTV_ANSWER_LEN, 0); + ret = dvb_usb_generic_rw(d, b0, 5, rx, PCTV_ANSWER_LEN, 0); if (ret) goto ret; - state->data[1] = state->c++; - state->data[4] = 1; + b0[1] = state->c++; + b0[4] = 1; /* reset board (again?) */ - ret = dvb_usb_generic_rw(d, state->data, 5, rx, PCTV_ANSWER_LEN, 0); + ret = dvb_usb_generic_rw(d, b0, 5, rx, PCTV_ANSWER_LEN, 0); if (ret) goto ret; state->initialized = 1; ret: - mutex_unlock(&state->ca_mutex); - kfree(rx); + kfree(b0); return ret; } static int pctv452e_rc_query(struct dvb_usb_device *d) { struct pctv452e_state *state = (struct pctv452e_state *)d->priv; + u8 *b, *rx; int ret, i; u8 id; - mutex_lock(&state->ca_mutex); + b = kmalloc(CMD_BUFFER_SIZE + PCTV_ANSWER_LEN, GFP_KERNEL); + if (!b) + return -ENOMEM; + + rx = b + CMD_BUFFER_SIZE; + id = state->c++; /* prepare command header */ - state->data[0] = SYNC_BYTE_OUT; - state->data[1] = id; - state->data[2] = PCTV_CMD_IR; - state->data[3] = 0; + b[0] = SYNC_BYTE_OUT; + b[1] = id; + b[2] = PCTV_CMD_IR; + b[3] = 0; /* send ir request */ - ret = dvb_usb_generic_rw(d, state->data, 4, - state->data, PCTV_ANSWER_LEN, 0); + ret = dvb_usb_generic_rw(d, b, 4, rx, PCTV_ANSWER_LEN, 0); if (ret != 0) goto ret; if (debug > 3) { - info("%s: read: %2d: %*ph: ", __func__, ret, 3, state->data); - for (i = 0; (i < state->data[3]) && ((i + 3) < PCTV_ANSWER_LEN); i++) - info(" %02x", state->data[i + 3]); + info("%s: read: %2d: %*ph: ", __func__, ret, 3, rx); + for (i = 0; (i < rx[3]) && ((i+3) < PCTV_ANSWER_LEN); i++) + info(" %02x", rx[i+3]); info("\n"); } - if ((state->data[3] == 9) && (state->data[12] & 0x01)) { + if ((rx[3] == 9) && (rx[12] & 0x01)) { /* got a "press" event */ - state->last_rc_key = RC_SCANCODE_RC5(state->data[7], state->data[6]); + state->last_rc_key = RC_SCANCODE_RC5(rx[7], rx[6]); if (debug > 2) info("%s: cmd=0x%02x sys=0x%02x\n", - __func__, state->data[6], state->data[7]); + __func__, rx[6], rx[7]); rc_keydown(d->rc_dev, RC_TYPE_RC5, state->last_rc_key, 0); } else if (state->last_rc_key) { @@ -595,7 +606,7 @@ static int pctv452e_rc_query(struct dvb_usb_device *d) state->last_rc_key = 0; } ret: - mutex_unlock(&state->ca_mutex); + kfree(b); return ret; } diff --git a/drivers/mfd/tps65217.c b/drivers/mfd/tps65217.c index 9a4d8684dd32bd..df2e7756927f20 100644 --- a/drivers/mfd/tps65217.c +++ b/drivers/mfd/tps65217.c @@ -424,6 +424,24 @@ static int tps65217_probe(struct i2c_client *client, return 0; } +static int tps65217_remove(struct i2c_client *client) +{ + struct tps65217 *tps = i2c_get_clientdata(client); + unsigned int virq; + int i; + + for (i = 0; i < ARRAY_SIZE(tps65217_irqs); i++) { + virq = irq_find_mapping(tps->irq_domain, i); + if (virq) + irq_dispose_mapping(virq); + } + + irq_domain_remove(tps->irq_domain); + tps->irq_domain = NULL; + + return 0; +} + static const struct i2c_device_id tps65217_id_table[] = { {"tps65217", TPS65217}, { /* sentinel */ } @@ -437,6 +455,7 @@ static struct i2c_driver tps65217_driver = { }, .id_table = tps65217_id_table, .probe = tps65217_probe, + .remove = tps65217_remove, }; static int __init tps65217_init(void) diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig index 64971baf11faeb..407cf290c03105 100644 --- a/drivers/misc/Kconfig +++ b/drivers/misc/Kconfig @@ -766,6 +766,18 @@ config PANEL_BOOT_MESSAGE An empty message will only clear the display at driver init time. Any other printf()-formatted message is valid with newline and escape codes. +config UID_CPUTIME + bool "Per-UID cpu time statistics" + depends on PROFILING + help + Per UID based cpu time statistics exported to /proc/uid_cputime + +config MEMORY_STATE_TIME + tristate "Memory freq/bandwidth time statistics" + depends on PROFILING + help + Memory time statistics exported to /sys/kernel/memory_state_time + source "drivers/misc/c2port/Kconfig" source "drivers/misc/eeprom/Kconfig" source "drivers/misc/cb710/Kconfig" diff --git a/drivers/misc/Makefile b/drivers/misc/Makefile index 31983366090a78..3f89debe458299 100644 --- a/drivers/misc/Makefile +++ b/drivers/misc/Makefile @@ -54,6 +54,9 @@ obj-$(CONFIG_VEXPRESS_SYSCFG) += vexpress-syscfg.o obj-$(CONFIG_CXL_BASE) += cxl/ obj-$(CONFIG_PANEL) += panel.o +obj-$(CONFIG_UID_CPUTIME) += uid_cputime.o +obj-$(CONFIG_MEMORY_STATE_TIME) += memory_state_time.o + lkdtm-$(CONFIG_LKDTM) += lkdtm_core.o lkdtm-$(CONFIG_LKDTM) += lkdtm_bugs.o lkdtm-$(CONFIG_LKDTM) += lkdtm_heap.o diff --git a/drivers/misc/mei/bus.c b/drivers/misc/mei/bus.c index 8cac7ef9ad0d98..dbe676de7a1944 100644 --- a/drivers/misc/mei/bus.c +++ b/drivers/misc/mei/bus.c @@ -408,7 +408,7 @@ bool mei_cldev_enabled(struct mei_cl_device *cldev) EXPORT_SYMBOL_GPL(mei_cldev_enabled); /** - * mei_cldev_enable_device - enable me client device + * mei_cldev_enable - enable me client device * create connection with me client * * @cldev: me client device diff --git a/drivers/misc/mei/client.c b/drivers/misc/mei/client.c index 6fe02350578dce..e2af61f7e3b659 100644 --- a/drivers/misc/mei/client.c +++ b/drivers/misc/mei/client.c @@ -425,7 +425,7 @@ static inline void mei_io_list_free(struct mei_cl_cb *list, struct mei_cl *cl) * * @cl: host client * @length: size of the buffer - * @type: operation type + * @fop_type: operation type * @fp: associated file pointer (might be NULL) * * Return: cb on success and NULL on failure @@ -459,7 +459,7 @@ struct mei_cl_cb *mei_cl_alloc_cb(struct mei_cl *cl, size_t length, * * @cl: host client * @length: size of the buffer - * @type: operation type + * @fop_type: operation type * @fp: associated file pointer (might be NULL) * * Return: cb on success and NULL on failure @@ -686,7 +686,7 @@ void mei_host_client_init(struct mei_device *dev) pm_runtime_mark_last_busy(dev->dev); dev_dbg(dev->dev, "rpm: autosuspend\n"); - pm_runtime_autosuspend(dev->dev); + pm_request_autosuspend(dev->dev); } /** @@ -1536,7 +1536,7 @@ int mei_cl_irq_write(struct mei_cl *cl, struct mei_cl_cb *cb, rets = first_chunk ? mei_cl_tx_flow_ctrl_creds(cl) : 1; if (rets < 0) - return rets; + goto err; if (rets == 0) { cl_dbg(dev, cl, "No flow control credentials: not sending.\n"); @@ -1570,11 +1570,8 @@ int mei_cl_irq_write(struct mei_cl *cl, struct mei_cl_cb *cb, cb->buf.size, cb->buf_idx); rets = mei_write_message(dev, &mei_hdr, buf->data + cb->buf_idx); - if (rets) { - cl->status = rets; - list_move_tail(&cb->list, &cmpl_list->list); - return rets; - } + if (rets) + goto err; cl->status = 0; cl->writing_state = MEI_WRITING; @@ -1582,14 +1579,21 @@ int mei_cl_irq_write(struct mei_cl *cl, struct mei_cl_cb *cb, cb->completed = mei_hdr.msg_complete == 1; if (first_chunk) { - if (mei_cl_tx_flow_ctrl_creds_reduce(cl)) - return -EIO; + if (mei_cl_tx_flow_ctrl_creds_reduce(cl)) { + rets = -EIO; + goto err; + } } if (mei_hdr.msg_complete) list_move_tail(&cb->list, &dev->write_waiting_list.list); return 0; + +err: + cl->status = rets; + list_move_tail(&cb->list, &cmpl_list->list); + return rets; } /** diff --git a/drivers/misc/mei/hw-me-regs.h b/drivers/misc/mei/hw-me-regs.h index 7ad15d678878cf..c8307e8b4c1639 100644 --- a/drivers/misc/mei/hw-me-regs.h +++ b/drivers/misc/mei/hw-me-regs.h @@ -122,6 +122,8 @@ #define MEI_DEV_ID_SPT_H 0xA13A /* Sunrise Point H */ #define MEI_DEV_ID_SPT_H_2 0xA13B /* Sunrise Point H 2 */ +#define MEI_DEV_ID_LBG 0xA1BA /* Lewisburg (SPT) */ + #define MEI_DEV_ID_BXT_M 0x1A9A /* Broxton M */ #define MEI_DEV_ID_APL_I 0x5A9A /* Apollo Lake I */ diff --git a/drivers/misc/mei/pci-me.c b/drivers/misc/mei/pci-me.c index f3ffd883b232ea..f9c6ec4b98ab7d 100644 --- a/drivers/misc/mei/pci-me.c +++ b/drivers/misc/mei/pci-me.c @@ -87,6 +87,7 @@ static const struct pci_device_id mei_me_pci_tbl[] = { {MEI_PCI_DEVICE(MEI_DEV_ID_SPT_2, mei_me_pch8_cfg)}, {MEI_PCI_DEVICE(MEI_DEV_ID_SPT_H, mei_me_pch8_sps_cfg)}, {MEI_PCI_DEVICE(MEI_DEV_ID_SPT_H_2, mei_me_pch8_sps_cfg)}, + {MEI_PCI_DEVICE(MEI_DEV_ID_LBG, mei_me_pch8_cfg)}, {MEI_PCI_DEVICE(MEI_DEV_ID_BXT_M, mei_me_pch8_cfg)}, {MEI_PCI_DEVICE(MEI_DEV_ID_APL_I, mei_me_pch8_cfg)}, diff --git a/drivers/misc/memory_state_time.c b/drivers/misc/memory_state_time.c new file mode 100644 index 00000000000000..34c797a06a31aa --- /dev/null +++ b/drivers/misc/memory_state_time.c @@ -0,0 +1,454 @@ +/* drivers/misc/memory_state_time.c + * + * Copyright (C) 2016 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define KERNEL_ATTR_RO(_name) \ +static struct kobj_attribute _name##_attr = __ATTR_RO(_name) + +#define KERNEL_ATTR_RW(_name) \ +static struct kobj_attribute _name##_attr = \ + __ATTR(_name, 0644, _name##_show, _name##_store) + +#define FREQ_HASH_BITS 4 +DECLARE_HASHTABLE(freq_hash_table, FREQ_HASH_BITS); + +static DEFINE_MUTEX(mem_lock); + +#define TAG "memory_state_time" +#define BW_NODE "/soc/memory-state-time" +#define FREQ_TBL "freq-tbl" +#define BW_TBL "bw-buckets" +#define NUM_SOURCES "num-sources" + +#define LOWEST_FREQ 2 + +static int curr_bw; +static int curr_freq; +static u32 *bw_buckets; +static u32 *freq_buckets; +static int num_freqs; +static int num_buckets; +static int registered_bw_sources; +static u64 last_update; +static bool init_success; +static struct workqueue_struct *memory_wq; +static u32 num_sources = 10; +static int *bandwidths; + +struct freq_entry { + int freq; + u64 *buckets; /* Bandwidth buckets. */ + struct hlist_node hash; +}; + +struct queue_container { + struct work_struct update_state; + int value; + u64 time_now; + int id; + struct mutex *lock; +}; + +static int find_bucket(int bw) +{ + int i; + + if (bw_buckets != NULL) { + for (i = 0; i < num_buckets; i++) { + if (bw_buckets[i] > bw) { + pr_debug("Found bucket %d for bandwidth %d\n", + i, bw); + return i; + } + } + return num_buckets - 1; + } + return 0; +} + +static u64 get_time_diff(u64 time_now) +{ + u64 ms; + + ms = time_now - last_update; + last_update = time_now; + return ms; +} + +static ssize_t show_stat_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + int i, j; + int len = 0; + struct freq_entry *freq_entry; + + for (i = 0; i < num_freqs; i++) { + hash_for_each_possible(freq_hash_table, freq_entry, hash, + freq_buckets[i]) { + if (freq_entry->freq == freq_buckets[i]) { + len += scnprintf(buf + len, PAGE_SIZE - len, + "%d ", freq_buckets[i]); + if (len >= PAGE_SIZE) + break; + for (j = 0; j < num_buckets; j++) { + len += scnprintf(buf + len, + PAGE_SIZE - len, + "%llu ", + freq_entry->buckets[j]); + } + len += scnprintf(buf + len, PAGE_SIZE - len, + "\n"); + } + } + } + pr_debug("Current Time: %llu\n", ktime_get_boot_ns()); + return len; +} +KERNEL_ATTR_RO(show_stat); + +static void update_table(u64 time_now) +{ + struct freq_entry *freq_entry; + + pr_debug("Last known bw %d freq %d\n", curr_bw, curr_freq); + hash_for_each_possible(freq_hash_table, freq_entry, hash, curr_freq) { + if (curr_freq == freq_entry->freq) { + freq_entry->buckets[find_bucket(curr_bw)] + += get_time_diff(time_now); + break; + } + } +} + +static bool freq_exists(int freq) +{ + int i; + + for (i = 0; i < num_freqs; i++) { + if (freq == freq_buckets[i]) + return true; + } + return false; +} + +static int calculate_total_bw(int bw, int index) +{ + int i; + int total_bw = 0; + + pr_debug("memory_state_time New bw %d for id %d\n", bw, index); + bandwidths[index] = bw; + for (i = 0; i < registered_bw_sources; i++) + total_bw += bandwidths[i]; + return total_bw; +} + +static void freq_update_do_work(struct work_struct *work) +{ + struct queue_container *freq_state_update + = container_of(work, struct queue_container, + update_state); + if (freq_state_update) { + mutex_lock(&mem_lock); + update_table(freq_state_update->time_now); + curr_freq = freq_state_update->value; + mutex_unlock(&mem_lock); + kfree(freq_state_update); + } +} + +static void bw_update_do_work(struct work_struct *work) +{ + struct queue_container *bw_state_update + = container_of(work, struct queue_container, + update_state); + if (bw_state_update) { + mutex_lock(&mem_lock); + update_table(bw_state_update->time_now); + curr_bw = calculate_total_bw(bw_state_update->value, + bw_state_update->id); + mutex_unlock(&mem_lock); + kfree(bw_state_update); + } +} + +static void memory_state_freq_update(struct memory_state_update_block *ub, + int value) +{ + if (IS_ENABLED(CONFIG_MEMORY_STATE_TIME)) { + if (freq_exists(value) && init_success) { + struct queue_container *freq_container + = kmalloc(sizeof(struct queue_container), + GFP_KERNEL); + if (!freq_container) + return; + INIT_WORK(&freq_container->update_state, + freq_update_do_work); + freq_container->time_now = ktime_get_boot_ns(); + freq_container->value = value; + pr_debug("Scheduling freq update in work queue\n"); + queue_work(memory_wq, &freq_container->update_state); + } else { + pr_debug("Freq does not exist.\n"); + } + } +} + +static void memory_state_bw_update(struct memory_state_update_block *ub, + int value) +{ + if (IS_ENABLED(CONFIG_MEMORY_STATE_TIME)) { + if (init_success) { + struct queue_container *bw_container + = kmalloc(sizeof(struct queue_container), + GFP_KERNEL); + if (!bw_container) + return; + INIT_WORK(&bw_container->update_state, + bw_update_do_work); + bw_container->time_now = ktime_get_boot_ns(); + bw_container->value = value; + bw_container->id = ub->id; + pr_debug("Scheduling bandwidth update in work queue\n"); + queue_work(memory_wq, &bw_container->update_state); + } + } +} + +struct memory_state_update_block *memory_state_register_frequency_source(void) +{ + struct memory_state_update_block *block; + + if (IS_ENABLED(CONFIG_MEMORY_STATE_TIME)) { + pr_debug("Allocating frequency source\n"); + block = kmalloc(sizeof(struct memory_state_update_block), + GFP_KERNEL); + if (!block) + return NULL; + block->update_call = memory_state_freq_update; + return block; + } + pr_err("Config option disabled.\n"); + return NULL; +} +EXPORT_SYMBOL_GPL(memory_state_register_frequency_source); + +struct memory_state_update_block *memory_state_register_bandwidth_source(void) +{ + struct memory_state_update_block *block; + + if (IS_ENABLED(CONFIG_MEMORY_STATE_TIME)) { + pr_debug("Allocating bandwidth source %d\n", + registered_bw_sources); + block = kmalloc(sizeof(struct memory_state_update_block), + GFP_KERNEL); + if (!block) + return NULL; + block->update_call = memory_state_bw_update; + if (registered_bw_sources < num_sources) { + block->id = registered_bw_sources++; + } else { + pr_err("Unable to allocate source; max number reached\n"); + kfree(block); + return NULL; + } + return block; + } + pr_err("Config option disabled.\n"); + return NULL; +} +EXPORT_SYMBOL_GPL(memory_state_register_bandwidth_source); + +/* Buckets are designated by their maximum. + * Returns the buckets decided by the capability of the device. + */ +static int get_bw_buckets(struct device *dev) +{ + int ret, lenb; + struct device_node *node = dev->of_node; + + of_property_read_u32(node, NUM_SOURCES, &num_sources); + if (of_find_property(node, BW_TBL, &lenb)) { + bandwidths = devm_kzalloc(dev, + sizeof(*bandwidths) * num_sources, GFP_KERNEL); + if (!bandwidths) + return -ENOMEM; + lenb /= sizeof(*bw_buckets); + bw_buckets = devm_kzalloc(dev, lenb * sizeof(*bw_buckets), + GFP_KERNEL); + if (!bw_buckets) { + devm_kfree(dev, bandwidths); + return -ENOMEM; + } + ret = of_property_read_u32_array(node, BW_TBL, bw_buckets, + lenb); + if (ret < 0) { + devm_kfree(dev, bandwidths); + devm_kfree(dev, bw_buckets); + pr_err("Unable to read bandwidth table from device tree.\n"); + return ret; + } + } + curr_bw = 0; + num_buckets = lenb; + return 0; +} + +/* Adds struct freq_entry nodes to the hashtable for each compatible frequency. + * Returns the supported number of frequencies. + */ +static int freq_buckets_init(struct device *dev) +{ + struct freq_entry *freq_entry; + int i; + int ret, lenf; + struct device_node *node = dev->of_node; + + if (of_find_property(node, FREQ_TBL, &lenf)) { + lenf /= sizeof(*freq_buckets); + freq_buckets = devm_kzalloc(dev, lenf * sizeof(*freq_buckets), + GFP_KERNEL); + if (!freq_buckets) + return -ENOMEM; + pr_debug("freqs found len %d\n", lenf); + ret = of_property_read_u32_array(node, FREQ_TBL, freq_buckets, + lenf); + if (ret < 0) { + devm_kfree(dev, freq_buckets); + pr_err("Unable to read frequency table from device tree.\n"); + return ret; + } + pr_debug("ret freq %d\n", ret); + } + num_freqs = lenf; + curr_freq = freq_buckets[LOWEST_FREQ]; + + for (i = 0; i < num_freqs; i++) { + freq_entry = devm_kzalloc(dev, sizeof(struct freq_entry), + GFP_KERNEL); + if (!freq_entry) + return -ENOMEM; + freq_entry->buckets = devm_kzalloc(dev, sizeof(u64)*num_buckets, + GFP_KERNEL); + if (!freq_entry->buckets) { + devm_kfree(dev, freq_entry); + return -ENOMEM; + } + pr_debug("memory_state_time Adding freq to ht %d\n", + freq_buckets[i]); + freq_entry->freq = freq_buckets[i]; + hash_add(freq_hash_table, &freq_entry->hash, freq_buckets[i]); + } + return 0; +} + +struct kobject *memory_kobj; +EXPORT_SYMBOL_GPL(memory_kobj); + +static struct attribute *memory_attrs[] = { + &show_stat_attr.attr, + NULL +}; + +static struct attribute_group memory_attr_group = { + .attrs = memory_attrs, +}; + +static int memory_state_time_probe(struct platform_device *pdev) +{ + int error; + + error = get_bw_buckets(&pdev->dev); + if (error) + return error; + error = freq_buckets_init(&pdev->dev); + if (error) + return error; + last_update = ktime_get_boot_ns(); + init_success = true; + + pr_debug("memory_state_time initialized with num_freqs %d\n", + num_freqs); + return 0; +} + +static const struct of_device_id match_table[] = { + { .compatible = "memory-state-time" }, + {} +}; + +static struct platform_driver memory_state_time_driver = { + .probe = memory_state_time_probe, + .driver = { + .name = "memory-state-time", + .of_match_table = match_table, + .owner = THIS_MODULE, + }, +}; + +static int __init memory_state_time_init(void) +{ + int error; + + hash_init(freq_hash_table); + memory_wq = create_singlethread_workqueue("memory_wq"); + if (!memory_wq) { + pr_err("Unable to create workqueue.\n"); + return -EINVAL; + } + /* + * Create sys/kernel directory for memory_state_time. + */ + memory_kobj = kobject_create_and_add(TAG, kernel_kobj); + if (!memory_kobj) { + pr_err("Unable to allocate memory_kobj for sysfs directory.\n"); + error = -ENOMEM; + goto wq; + } + error = sysfs_create_group(memory_kobj, &memory_attr_group); + if (error) { + pr_err("Unable to create sysfs folder.\n"); + goto kobj; + } + + error = platform_driver_register(&memory_state_time_driver); + if (error) { + pr_err("Unable to register memory_state_time platform driver.\n"); + goto group; + } + return 0; + +group: sysfs_remove_group(memory_kobj, &memory_attr_group); +kobj: kobject_put(memory_kobj); +wq: destroy_workqueue(memory_wq); + return error; +} +module_init(memory_state_time_init); diff --git a/drivers/misc/ti-st/Kconfig b/drivers/misc/ti-st/Kconfig index f34dcc51473052..f2df2c7352e2f8 100644 --- a/drivers/misc/ti-st/Kconfig +++ b/drivers/misc/ti-st/Kconfig @@ -14,4 +14,12 @@ config TI_ST are returned to relevant protocol drivers based on their packet types. +config ST_HCI + tristate "HCI TTY emulation driver for Bluetooth" + depends on TI_ST + help + This enables the TTY device like emulation for HCI used by + user-space Bluetooth stacks. + It will provide a character device for user space Bluetooth stack to + send/receive data over shared transport. endmenu diff --git a/drivers/misc/ti-st/Makefile b/drivers/misc/ti-st/Makefile index 78d7ebb1474940..454621909fb702 100644 --- a/drivers/misc/ti-st/Makefile +++ b/drivers/misc/ti-st/Makefile @@ -4,3 +4,4 @@ # obj-$(CONFIG_TI_ST) += st_drv.o st_drv-objs := st_core.o st_kim.o st_ll.o +obj-$(CONFIG_ST_HCI) += tty_hci.o diff --git a/drivers/misc/ti-st/st_kim.c b/drivers/misc/ti-st/st_kim.c index bf0d7708beac59..a67d59b1d7090c 100644 --- a/drivers/misc/ti-st/st_kim.c +++ b/drivers/misc/ti-st/st_kim.c @@ -36,6 +36,8 @@ #include #include #include +#include +#include #define MAX_ST_DEVICES 3 /* Imagine 1 on each UART for now */ static struct platform_device *st_kim_devices[MAX_ST_DEVICES]; @@ -43,6 +45,9 @@ static struct platform_device *st_kim_devices[MAX_ST_DEVICES]; /**********************************************************************/ /* internal functions */ +struct ti_st_plat_data *dt_pdata; +static struct ti_st_plat_data *get_platform_data(struct device *dev); + /** * st_get_plat_device - * function which returns the reference to the platform device @@ -463,7 +468,12 @@ long st_kim_start(void *kim_data) struct kim_data_s *kim_gdata = (struct kim_data_s *)kim_data; pr_info(" %s", __func__); - pdata = kim_gdata->kim_pdev->dev.platform_data; + if (kim_gdata->kim_pdev->dev.of_node) { + pr_debug("use device tree data"); + pdata = dt_pdata; + } else { + pdata = kim_gdata->kim_pdev->dev.platform_data; + } do { /* platform specific enabling code here */ @@ -523,12 +533,18 @@ long st_kim_stop(void *kim_data) { long err = 0; struct kim_data_s *kim_gdata = (struct kim_data_s *)kim_data; - struct ti_st_plat_data *pdata = - kim_gdata->kim_pdev->dev.platform_data; + struct ti_st_plat_data *pdata; struct tty_struct *tty = kim_gdata->core_data->tty; reinit_completion(&kim_gdata->ldisc_installed); + if (kim_gdata->kim_pdev->dev.of_node) { + pr_debug("use device tree data"); + pdata = dt_pdata; + } else + pdata = kim_gdata->kim_pdev->dev.platform_data; + + if (tty) { /* can be called before ldisc is installed */ /* Flush any pending characters in the driver and discipline. */ tty_ldisc_flush(tty); @@ -720,13 +736,52 @@ static const struct file_operations list_debugfs_fops = { * board-*.c file */ +static const struct of_device_id kim_of_match[] = { +{ + .compatible = "kim", + }, + {} +}; +MODULE_DEVICE_TABLE(of, kim_of_match); + +static struct ti_st_plat_data *get_platform_data(struct device *dev) +{ + struct device_node *np = dev->of_node; + const u32 *dt_property; + int len; + + dt_pdata = kzalloc(sizeof(*dt_pdata), GFP_KERNEL); + if (!dt_pdata) + return NULL; + + dt_property = of_get_property(np, "dev_name", &len); + if (dt_property) + memcpy(&dt_pdata->dev_name, dt_property, len); + of_property_read_u32(np, "nshutdown_gpio", + &dt_pdata->nshutdown_gpio); + of_property_read_u32(np, "flow_cntrl", &dt_pdata->flow_cntrl); + of_property_read_u32(np, "baud_rate", &dt_pdata->baud_rate); + + return dt_pdata; +} + static struct dentry *kim_debugfs_dir; static int kim_probe(struct platform_device *pdev) { struct kim_data_s *kim_gdata; - struct ti_st_plat_data *pdata = pdev->dev.platform_data; + struct ti_st_plat_data *pdata; int err; + if (pdev->dev.of_node) + pdata = get_platform_data(&pdev->dev); + else + pdata = pdev->dev.platform_data; + + if (pdata == NULL) { + dev_err(&pdev->dev, "Platform Data is missing\n"); + return -ENXIO; + } + if ((pdev->id != -1) && (pdev->id < MAX_ST_DEVICES)) { /* multiple devices could exist */ st_kim_devices[pdev->id] = pdev; @@ -807,9 +862,16 @@ static int kim_probe(struct platform_device *pdev) static int kim_remove(struct platform_device *pdev) { /* free the GPIOs requested */ - struct ti_st_plat_data *pdata = pdev->dev.platform_data; + struct ti_st_plat_data *pdata; struct kim_data_s *kim_gdata; + if (pdev->dev.of_node) { + pr_debug("use device tree data"); + pdata = dt_pdata; + } else { + pdata = pdev->dev.platform_data; + } + kim_gdata = platform_get_drvdata(pdev); /* Free the Bluetooth/FM/GPIO @@ -827,12 +889,22 @@ static int kim_remove(struct platform_device *pdev) kfree(kim_gdata); kim_gdata = NULL; + kfree(dt_pdata); + dt_pdata = NULL; + return 0; } static int kim_suspend(struct platform_device *pdev, pm_message_t state) { - struct ti_st_plat_data *pdata = pdev->dev.platform_data; + struct ti_st_plat_data *pdata; + + if (pdev->dev.of_node) { + pr_debug("use device tree data"); + pdata = dt_pdata; + } else { + pdata = pdev->dev.platform_data; + } if (pdata->suspend) return pdata->suspend(pdev, state); @@ -842,7 +914,14 @@ static int kim_suspend(struct platform_device *pdev, pm_message_t state) static int kim_resume(struct platform_device *pdev) { - struct ti_st_plat_data *pdata = pdev->dev.platform_data; + struct ti_st_plat_data *pdata; + + if (pdev->dev.of_node) { + pr_debug("use device tree data"); + pdata = dt_pdata; + } else { + pdata = pdev->dev.platform_data; + } if (pdata->resume) return pdata->resume(pdev); @@ -859,6 +938,7 @@ static struct platform_driver kim_platform_driver = { .resume = kim_resume, .driver = { .name = "kim", + .of_match_table = of_match_ptr(kim_of_match), }, }; diff --git a/drivers/misc/ti-st/st_ll.c b/drivers/misc/ti-st/st_ll.c index 93b4d67cc4a3a6..518e1b7f2f95a5 100644 --- a/drivers/misc/ti-st/st_ll.c +++ b/drivers/misc/ti-st/st_ll.c @@ -26,6 +26,7 @@ #include /**********************************************************************/ + /* internal functions */ static void send_ll_cmd(struct st_data_s *st_data, unsigned char cmd) @@ -53,7 +54,13 @@ static void ll_device_want_to_sleep(struct st_data_s *st_data) /* communicate to platform about chip asleep */ kim_data = st_data->kim_data; - pdata = kim_data->kim_pdev->dev.platform_data; + if (kim_data->kim_pdev->dev.of_node) { + pr_debug("use device tree data"); + pdata = dt_pdata; + } else { + pdata = kim_data->kim_pdev->dev.platform_data; + } + if (pdata->chip_asleep) pdata->chip_asleep(NULL); } @@ -86,7 +93,13 @@ static void ll_device_want_to_wakeup(struct st_data_s *st_data) /* communicate to platform about chip wakeup */ kim_data = st_data->kim_data; - pdata = kim_data->kim_pdev->dev.platform_data; + if (kim_data->kim_pdev->dev.of_node) { + pr_debug("use device tree data"); + pdata = dt_pdata; + } else { + pdata = kim_data->kim_pdev->dev.platform_data; + } + if (pdata->chip_awake) pdata->chip_awake(NULL); } diff --git a/drivers/misc/ti-st/tty_hci.c b/drivers/misc/ti-st/tty_hci.c new file mode 100644 index 00000000000000..1f0986ae36d51e --- /dev/null +++ b/drivers/misc/ti-st/tty_hci.c @@ -0,0 +1,543 @@ +/* + * TTY emulation for user-space Bluetooth stacks over HCI-H4 + * Copyright (C) 2011-2012 Texas Instruments + * Author: Pavan Savoy + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +/** define one of the following for debugging +#define DEBUG +#define VERBOSE +*/ + +#define pr_fmt(fmt) "(hci_tty): " fmt +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +/* Number of seconds to wait for registration completion + * when ST returns PENDING status. + */ +#define BT_REGISTER_TIMEOUT 6000 /* 6 sec */ + +/** + * struct ti_st - driver operation structure + * @hdev: hci device pointer which binds to bt driver + * @reg_status: ST registration callback status + * @st_write: write function provided by the ST driver + * to be used by the driver during send_frame. + * @wait_reg_completion - completion sync between ti_st_open + * and st_reg_completion_cb. + */ +struct ti_st { + struct hci_dev *hdev; + char reg_status; + long (*st_write)(struct sk_buff *); + struct completion wait_reg_completion; + wait_queue_head_t data_q; + struct sk_buff_head rx_list; +}; + +#define DEVICE_NAME "hci_tty" + +/***********Functions called from ST driver**********************************/ +/* Called by Shared Transport layer when receive data is + * available */ +static long st_receive(void *priv_data, struct sk_buff *skb) +{ + struct ti_st *hst = (void *)priv_data; + + pr_debug("@ %s", __func__); +#ifdef VERBOSE + print_hex_dump(KERN_INFO, ">rx>", DUMP_PREFIX_NONE, + 16, 1, skb->data, skb->len, 0); +#endif + skb_queue_tail(&hst->rx_list, skb); + wake_up_interruptible(&hst->data_q); + return 0; +} + +/* Called by ST layer to indicate protocol registration completion + * status.ti_st_open() function will wait for signal from this + * API when st_register() function returns ST_PENDING. + */ +static void st_reg_completion_cb(void *priv_data, int data) +{ + struct ti_st *lhst = (void *)priv_data; + + pr_info("@ %s\n", __func__); + /* Save registration status for use in ti_st_open() */ + lhst->reg_status = data; + /* complete the wait in ti_st_open() */ + complete(&lhst->wait_reg_completion); +} + +/* protocol structure registered with shared transport */ +#define MAX_BT_CHNL_IDS 3 +static struct st_proto_s ti_st_proto[MAX_BT_CHNL_IDS] = { + { + .chnl_id = 0x04, /* HCI Events */ + .hdr_len = 2, + .offset_len_in_hdr = 1, + .len_size = 1, /* sizeof(plen) in struct hci_event_hdr */ + .reserve = 8, + }, + { + .chnl_id = 0x02, /* ACL */ + .hdr_len = 4, + .offset_len_in_hdr = 2, + .len_size = 2, /* sizeof(dlen) in struct hci_acl_hdr */ + .reserve = 8, + }, + { + .chnl_id = 0x03, /* SCO */ + .hdr_len = 3, + .offset_len_in_hdr = 2, + .len_size = 1, /* sizeof(dlen) in struct hci_sco_hdr */ + .reserve = 8, + }, +}; +/** hci_tty_open Function + * This function will perform an register on ST driver. + * + * Parameters : + * @file : File pointer for BT char driver + * @inod : + * Returns 0 - on success + * else suitable error code + */ +int hci_tty_open(struct inode *inod, struct file *file) +{ + int i = 0, err = 0; + unsigned long timeleft; + struct ti_st *hst; + + pr_info("inside %s (%p, %p)\n", __func__, inod, file); + + hst = kzalloc(sizeof(*hst), GFP_KERNEL); + file->private_data = hst; + hst = file->private_data; + + for (i = 0; i < MAX_BT_CHNL_IDS; i++) { + ti_st_proto[i].priv_data = hst; + ti_st_proto[i].max_frame_size = 1026; + ti_st_proto[i].recv = st_receive; + ti_st_proto[i].reg_complete_cb = st_reg_completion_cb; + + /* Prepare wait-for-completion handler */ + init_completion(&hst->wait_reg_completion); + /* Reset ST registration callback status flag, + * this value will be updated in + * st_reg_completion_cb() + * function whenever it called from ST driver. + */ + hst->reg_status = -EINPROGRESS; + + err = st_register(&ti_st_proto[i]); + if (!err) + goto done; + + if (err != -EINPROGRESS) { + pr_err("st_register failed %d", err); + goto error; + } + + /* ST is busy with either protocol + * registration or firmware download. + */ + pr_debug("waiting for registration completion signal from ST"); + timeleft = wait_for_completion_timeout + (&hst->wait_reg_completion, + msecs_to_jiffies(BT_REGISTER_TIMEOUT)); + if (!timeleft) { + pr_err("Timeout(%d sec),didn't get reg completion signal from ST", + BT_REGISTER_TIMEOUT / 1000); + err = -ETIMEDOUT; + goto error; + } + + /* Is ST registration callback + * called with ERROR status? */ + if (hst->reg_status != 0) { + pr_err("ST registration completed with invalid status %d", + hst->reg_status); + err = -EAGAIN; + goto error; + } + +done: + hst->st_write = ti_st_proto[i].write; + if (!hst->st_write) { + pr_err("undefined ST write function"); + for (i = 0; i < MAX_BT_CHNL_IDS; i++) { + /* Undo registration with ST */ + err = st_unregister(&ti_st_proto[i]); + if (err) + pr_err("st_unregister() failed with error %d", + err); + hst->st_write = NULL; + } + return -EIO; + } + } + + skb_queue_head_init(&hst->rx_list); + init_waitqueue_head(&hst->data_q); + + return 0; + +error: + kfree(hst); + return err; +} + +/** hci_tty_release Function + * This function will un-registers from the ST driver. + * + * Parameters : + * @file : File pointer for BT char driver + * @inod : + * Returns 0 - on success + * else suitable error code + */ +int hci_tty_release(struct inode *inod, struct file *file) +{ + int err, i; + struct ti_st *hst = file->private_data; + + pr_info("inside %s (%p, %p)\n", __func__, inod, file); + + for (i = 0; i < MAX_BT_CHNL_IDS; i++) { + err = st_unregister(&ti_st_proto[i]); + if (err) + pr_err("st_unregister(%d) failed with error %d", + ti_st_proto[i].chnl_id, err); + } + + hst->st_write = NULL; + skb_queue_purge(&hst->rx_list); + kfree(hst); + return err; +} + +/** hci_tty_read Function + * + * Parameters : + * @file : File pointer for BT char driver + * @data : Data which needs to be passed to APP + * @size : Length of the data passesd + * offset : + * Returns Size of packet received - on success + * else suitable error code + */ +ssize_t hci_tty_read(struct file *file, char __user *data, size_t size, + loff_t *offset) +{ + int len = 0, tout; + struct sk_buff *skb = NULL, *rskb = NULL; + struct ti_st *hst; + + pr_debug("inside %s (%p, %p, %zu, %p)\n", + __func__, file, data, size, offset); + + /* Validate input parameters */ + if ((NULL == file) || (((NULL == data) || (0 == size)))) { + pr_err("Invalid input params passed to %s", __func__); + return -EINVAL; + } + + hst = file->private_data; + + /* cannot come here if poll-ed before reading + * if not poll-ed wait on the same wait_q + */ + tout = wait_event_interruptible_timeout(hst->data_q, + !skb_queue_empty(&hst->rx_list), + msecs_to_jiffies(1000)); + /* Check for timed out condition */ + if (0 == tout) { + pr_err("Device Read timed out\n"); + return -ETIMEDOUT; + } + + /* hst->rx_list not empty skb already present */ + skb = skb_dequeue(&hst->rx_list); + if (!skb) { + pr_err("dequed skb is null?\n"); + return -EIO; + } + +#ifdef VERBOSE + print_hex_dump(KERN_INFO, ">in>", DUMP_PREFIX_NONE, + 16, 1, skb->data, skb->len, 0); +#endif + + /* Forward the data to the user */ + if (skb->len >= size) { + pr_err("FIONREAD not done before read\n"); + return -ENOMEM; + } else { + /* returning skb */ + rskb = alloc_skb(size, GFP_KERNEL); + if (!rskb) { + pr_err("alloc_skb error\n"); + return -ENOMEM; + } + + /* cb[0] has the pkt_type 0x04 or 0x02 or 0x03 */ + memcpy(skb_put(rskb, 1), &skb->cb[0], 1); + memcpy(skb_put(rskb, skb->len), skb->data, skb->len); + + if (copy_to_user(data, rskb->data, rskb->len)) { + pr_err("unable to copy to user space\n"); + /* Queue the skb back to head */ + skb_queue_head(&hst->rx_list, skb); + kfree_skb(rskb); + return -EIO; + } + } + + len = rskb->len; /* len of returning skb */ + kfree_skb(skb); + kfree_skb(rskb); + pr_debug("total size read= %d\n", len); + return len; +} + +/* hci_tty_write Function + * + * Parameters : + * @file : File pointer for BT char driver + * @data : packet data from BT application + * @size : Size of the packet data + * @offset : + * Returns Size of packet on success + * else suitable error code + */ +ssize_t hci_tty_write(struct file *file, const char __user *data, + size_t size, loff_t *offset) +{ + struct ti_st *hst = file->private_data; + struct sk_buff *skb; + + pr_debug("inside %s (%p, %p, %zu, %p)\n", + __func__, file, data, size, offset); + + if (!hst->st_write) { + pr_err(" Can't write to ST, hhci_tty->st_write null ?"); + return -EINVAL; + } + + skb = alloc_skb(size, GFP_KERNEL); + /* Validate Created SKB */ + if (NULL == skb) { + pr_err("Error aaloacting SKB"); + return -ENOMEM; + } + + /* Forward the data from the user space to ST core */ + if (copy_from_user(skb_put(skb, size), data, size)) { + pr_err(" Unable to copy from user space"); + kfree_skb(skb); + return -EIO; + } + +#ifdef VERBOSE + pr_debug("start data.."); + print_hex_dump(KERN_INFO, "data, size, 0); + pr_debug("\n..end data"); +#endif + + hst->st_write(skb); + return size; +} + +/** hci_tty_ioctl Function + * This will peform the functions as directed by the command and command + * argument. + * + * Parameters : + * @file : File pointer for BT char driver + * @cmd : IOCTL Command + * @arg : Command argument for IOCTL command + * Returns 0 on success + * else suitable error code + */ +static long hci_tty_ioctl(struct file *file, + unsigned int cmd, unsigned long arg) +{ + struct sk_buff *skb = NULL; + int retcode = 0; + struct ti_st *hst; + unsigned int value = 0; + + pr_debug("inside %s (%p, %u, %lx)", __func__, file, cmd, arg); + + /* Validate input parameters */ + if ((NULL == file) || (0 == cmd)) { + pr_err("invalid input parameters passed to %s", __func__); + return -EINVAL; + } + + hst = file->private_data; + + switch (cmd) { + case FIONREAD: + /* Deque the SKB from the head if rx_list is not empty + * update the argument with skb->len to provide amount of data + * available in the available SKB +1 for the PKT_TYPE + * field not provided in data by TI-ST. + */ + skb = skb_dequeue(&hst->rx_list); + if (skb != NULL) { + value = skb->len + 1; + /* Re-Store the SKB for furtur Read operations */ + skb_queue_head(&hst->rx_list, skb); + } + pr_debug("returning %d\n", value); + if (copy_to_user((void __user *)arg, &value, sizeof(value))) + return -EFAULT; + break; + default: + pr_debug("Un-Identified IOCTL %d", cmd); + retcode = 0; + break; + } + + return retcode; +} + +/** hci_tty_poll Function + * This function will wait till some data is received to the hci_tty driver from ST + * + * Parameters : + * @file : File pointer for BT char driver + * @wait : POLL wait information + * Returns status of POLL on success + * else suitable error code + */ +static unsigned int hci_tty_poll(struct file *file, poll_table *wait) +{ + struct ti_st *hst = file->private_data; + unsigned long mask = 0; + + pr_debug("@ %s\n", __func__); + + /* wait to be completed by st_receive */ + poll_wait(file, &hst->data_q, wait); + pr_debug("poll broke\n"); + + if (!skb_queue_empty(&hst->rx_list)) { + pr_debug("rx list que !empty\n"); + mask |= POLLIN; /* TODO: check app for mask */ + } + + return mask; +} + +/* BT Char driver function pointers + * These functions are called from USER space by pefroming File Operations + * on /dev/hci_tty node exposed by this driver during init + */ +const struct file_operations hci_tty_chrdev_ops = { + .owner = THIS_MODULE, + .open = hci_tty_open, + .read = hci_tty_read, + .write = hci_tty_write, + .unlocked_ioctl = hci_tty_ioctl, + .poll = hci_tty_poll, + .release = hci_tty_release, +}; + +/*********Functions called during insmod and delmod****************************/ + +static int hci_tty_major; /* major number */ +static struct class *hci_tty_class; /* class during class_create */ +static struct device *hci_tty_dev; /* dev during device_create */ +/** hci_tty_init Function + * This function Initializes the hci_tty driver parametes and exposes + * /dev/hci_tty node to user space + * + * Parameters : NULL + * Returns 0 on success + * else suitable error code + */ +static int __init hci_tty_init(void) +{ + pr_info("inside %s\n", __func__); + + /* Expose the device DEVICE_NAME to user space + * And obtain the major number for the device + */ + hci_tty_major = register_chrdev(0, DEVICE_NAME, &hci_tty_chrdev_ops); + + if (0 > hci_tty_major) { + pr_err("Error when registering to char dev"); + return hci_tty_major; + } + + /* udev */ + hci_tty_class = class_create(THIS_MODULE, DEVICE_NAME); + if (IS_ERR(hci_tty_class)) { + pr_err("Something went wrong in class_create"); + unregister_chrdev(hci_tty_major, DEVICE_NAME); + return -1; + } + + hci_tty_dev = + device_create(hci_tty_class, NULL, MKDEV(hci_tty_major, 0), + NULL, DEVICE_NAME); + if (IS_ERR(hci_tty_dev)) { + pr_err("Error in device create"); + unregister_chrdev(hci_tty_major, DEVICE_NAME); + class_destroy(hci_tty_class); + return -1; + } + pr_info("allocated %d, %d\n", hci_tty_major, 0); + return 0; +} + +/** hci_tty_exit Function + * This function Destroys the hci_tty driver parametes and /dev/hci_tty node + * + * Parameters : NULL + * Returns NULL + */ +static void __exit hci_tty_exit(void) +{ + pr_info("inside %s\n", __func__); + pr_info("bye.. freeing up %d\n", hci_tty_major); + + device_destroy(hci_tty_class, MKDEV(hci_tty_major, 0)); + class_destroy(hci_tty_class); + unregister_chrdev(hci_tty_major, DEVICE_NAME); +} + +module_init(hci_tty_init); +module_exit(hci_tty_exit); + +MODULE_AUTHOR("Pavan Savoy "); +MODULE_LICENSE("GPL"); diff --git a/drivers/misc/uid_cputime.c b/drivers/misc/uid_cputime.c new file mode 100644 index 00000000000000..c1ad5246f564e5 --- /dev/null +++ b/drivers/misc/uid_cputime.c @@ -0,0 +1,240 @@ +/* drivers/misc/uid_cputime.c + * + * Copyright (C) 2014 - 2015 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define UID_HASH_BITS 10 +DECLARE_HASHTABLE(hash_table, UID_HASH_BITS); + +static DEFINE_MUTEX(uid_lock); +static struct proc_dir_entry *parent; + +struct uid_entry { + uid_t uid; + cputime_t utime; + cputime_t stime; + cputime_t active_utime; + cputime_t active_stime; + struct hlist_node hash; +}; + +static struct uid_entry *find_uid_entry(uid_t uid) +{ + struct uid_entry *uid_entry; + hash_for_each_possible(hash_table, uid_entry, hash, uid) { + if (uid_entry->uid == uid) + return uid_entry; + } + return NULL; +} + +static struct uid_entry *find_or_register_uid(uid_t uid) +{ + struct uid_entry *uid_entry; + + uid_entry = find_uid_entry(uid); + if (uid_entry) + return uid_entry; + + uid_entry = kzalloc(sizeof(struct uid_entry), GFP_ATOMIC); + if (!uid_entry) + return NULL; + + uid_entry->uid = uid; + + hash_add(hash_table, &uid_entry->hash, uid); + + return uid_entry; +} + +static int uid_stat_show(struct seq_file *m, void *v) +{ + struct uid_entry *uid_entry; + struct task_struct *task, *temp; + cputime_t utime; + cputime_t stime; + unsigned long bkt; + + mutex_lock(&uid_lock); + + hash_for_each(hash_table, bkt, uid_entry, hash) { + uid_entry->active_stime = 0; + uid_entry->active_utime = 0; + } + + read_lock(&tasklist_lock); + do_each_thread(temp, task) { + uid_entry = find_or_register_uid(from_kuid_munged( + current_user_ns(), task_uid(task))); + if (!uid_entry) { + read_unlock(&tasklist_lock); + mutex_unlock(&uid_lock); + pr_err("%s: failed to find the uid_entry for uid %d\n", + __func__, from_kuid_munged(current_user_ns(), + task_uid(task))); + return -ENOMEM; + } + task_cputime_adjusted(task, &utime, &stime); + uid_entry->active_utime += utime; + uid_entry->active_stime += stime; + } while_each_thread(temp, task); + read_unlock(&tasklist_lock); + + hash_for_each(hash_table, bkt, uid_entry, hash) { + cputime_t total_utime = uid_entry->utime + + uid_entry->active_utime; + cputime_t total_stime = uid_entry->stime + + uid_entry->active_stime; + seq_printf(m, "%d: %llu %llu\n", uid_entry->uid, + (unsigned long long)jiffies_to_msecs( + cputime_to_jiffies(total_utime)) * USEC_PER_MSEC, + (unsigned long long)jiffies_to_msecs( + cputime_to_jiffies(total_stime)) * USEC_PER_MSEC); + } + + mutex_unlock(&uid_lock); + return 0; +} + +static int uid_stat_open(struct inode *inode, struct file *file) +{ + return single_open(file, uid_stat_show, PDE_DATA(inode)); +} + +static const struct file_operations uid_stat_fops = { + .open = uid_stat_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int uid_remove_open(struct inode *inode, struct file *file) +{ + return single_open(file, NULL, NULL); +} + +static ssize_t uid_remove_write(struct file *file, + const char __user *buffer, size_t count, loff_t *ppos) +{ + struct uid_entry *uid_entry; + struct hlist_node *tmp; + char uids[128]; + char *start_uid, *end_uid = NULL; + long int uid_start = 0, uid_end = 0; + + if (count >= sizeof(uids)) + count = sizeof(uids) - 1; + + if (copy_from_user(uids, buffer, count)) + return -EFAULT; + + uids[count] = '\0'; + end_uid = uids; + start_uid = strsep(&end_uid, "-"); + + if (!start_uid || !end_uid) + return -EINVAL; + + if (kstrtol(start_uid, 10, &uid_start) != 0 || + kstrtol(end_uid, 10, &uid_end) != 0) { + return -EINVAL; + } + mutex_lock(&uid_lock); + + for (; uid_start <= uid_end; uid_start++) { + hash_for_each_possible_safe(hash_table, uid_entry, tmp, + hash, (uid_t)uid_start) { + if (uid_start == uid_entry->uid) { + hash_del(&uid_entry->hash); + kfree(uid_entry); + } + } + } + + mutex_unlock(&uid_lock); + return count; +} + +static const struct file_operations uid_remove_fops = { + .open = uid_remove_open, + .release = single_release, + .write = uid_remove_write, +}; + +static int process_notifier(struct notifier_block *self, + unsigned long cmd, void *v) +{ + struct task_struct *task = v; + struct uid_entry *uid_entry; + cputime_t utime, stime; + uid_t uid; + + if (!task) + return NOTIFY_OK; + + mutex_lock(&uid_lock); + uid = from_kuid_munged(current_user_ns(), task_uid(task)); + uid_entry = find_or_register_uid(uid); + if (!uid_entry) { + pr_err("%s: failed to find uid %d\n", __func__, uid); + goto exit; + } + + task_cputime_adjusted(task, &utime, &stime); + uid_entry->utime += utime; + uid_entry->stime += stime; + +exit: + mutex_unlock(&uid_lock); + return NOTIFY_OK; +} + +static struct notifier_block process_notifier_block = { + .notifier_call = process_notifier, +}; + +static int __init proc_uid_cputime_init(void) +{ + hash_init(hash_table); + + parent = proc_mkdir("uid_cputime", NULL); + if (!parent) { + pr_err("%s: failed to create proc entry\n", __func__); + return -ENOMEM; + } + + proc_create_data("remove_uid_range", S_IWUGO, parent, &uid_remove_fops, + NULL); + + proc_create_data("show_uid_stat", S_IRUGO, parent, &uid_stat_fops, + NULL); + + profile_event_register(PROFILE_TASK_EXIT, &process_notifier_block); + + return 0; +} + +early_initcall(proc_uid_cputime_init); diff --git a/drivers/mmc/card/Kconfig b/drivers/mmc/card/Kconfig index 5562308699bc29..6142ec1b9dfbbc 100644 --- a/drivers/mmc/card/Kconfig +++ b/drivers/mmc/card/Kconfig @@ -68,3 +68,15 @@ config MMC_TEST This driver is only of interest to those developing or testing a host driver. Most people should say N here. + +config MMC_SIMULATE_MAX_SPEED + bool "Turn on maximum speed control per block device" + depends on MMC_BLOCK + help + Say Y here to enable MMC device speed limiting. Used to test and + simulate the behavior of the system when confronted with a slow MMC. + + Enables max_read_speed, max_write_speed and cache_size attributes to + control the write or read maximum KB/second speed behaviors. + + If unsure, say N here. diff --git a/drivers/mmc/card/block.c b/drivers/mmc/card/block.c index 709a872ed484a9..817fcf8c0ac6b8 100644 --- a/drivers/mmc/card/block.c +++ b/drivers/mmc/card/block.c @@ -287,6 +287,250 @@ static ssize_t force_ro_store(struct device *dev, struct device_attribute *attr, return ret; } +#ifdef CONFIG_MMC_SIMULATE_MAX_SPEED + +static int max_read_speed, max_write_speed, cache_size = 4; + +module_param(max_read_speed, int, S_IRUSR | S_IRGRP); +MODULE_PARM_DESC(max_read_speed, "maximum KB/s read speed 0=off"); +module_param(max_write_speed, int, S_IRUSR | S_IRGRP); +MODULE_PARM_DESC(max_write_speed, "maximum KB/s write speed 0=off"); +module_param(cache_size, int, S_IRUSR | S_IRGRP); +MODULE_PARM_DESC(cache_size, "MB high speed memory or SLC cache"); + +/* + * helper macros and expectations: + * size - unsigned long number of bytes + * jiffies - unsigned long HZ timestamp difference + * speed - unsigned KB/s transfer rate + */ +#define size_and_speed_to_jiffies(size, speed) \ + ((size) * HZ / (speed) / 1024UL) +#define jiffies_and_speed_to_size(jiffies, speed) \ + (((speed) * (jiffies) * 1024UL) / HZ) +#define jiffies_and_size_to_speed(jiffies, size) \ + ((size) * HZ / (jiffies) / 1024UL) + +/* Limits to report warning */ +/* jiffies_and_size_to_speed(10*HZ, queue_max_hw_sectors(q) * 512UL) ~ 25 */ +#define MIN_SPEED(q) 250 /* 10 times faster than a floppy disk */ +#define MAX_SPEED(q) jiffies_and_size_to_speed(1, queue_max_sectors(q) * 512UL) + +#define speed_valid(speed) ((speed) > 0) + +static const char off[] = "off\n"; + +static int max_speed_show(int speed, char *buf) +{ + if (speed) + return scnprintf(buf, PAGE_SIZE, "%uKB/s\n", speed); + else + return scnprintf(buf, PAGE_SIZE, off); +} + +static int max_speed_store(const char *buf, struct request_queue *q) +{ + unsigned int limit, set = 0; + + if (!strncasecmp(off, buf, sizeof(off) - 2)) + return set; + if (kstrtouint(buf, 0, &set) || (set > INT_MAX)) + return -EINVAL; + if (set == 0) + return set; + limit = MAX_SPEED(q); + if (set > limit) + pr_warn("max speed %u ineffective above %u\n", set, limit); + limit = MIN_SPEED(q); + if (set < limit) + pr_warn("max speed %u painful below %u\n", set, limit); + return set; +} + +static ssize_t max_write_speed_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct mmc_blk_data *md = mmc_blk_get(dev_to_disk(dev)); + int ret = max_speed_show(atomic_read(&md->queue.max_write_speed), buf); + + mmc_blk_put(md); + return ret; +} + +static ssize_t max_write_speed_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct mmc_blk_data *md = mmc_blk_get(dev_to_disk(dev)); + int set = max_speed_store(buf, md->queue.queue); + + if (set < 0) { + mmc_blk_put(md); + return set; + } + + atomic_set(&md->queue.max_write_speed, set); + mmc_blk_put(md); + return count; +} + +static const DEVICE_ATTR(max_write_speed, S_IRUGO | S_IWUSR, + max_write_speed_show, max_write_speed_store); + +static ssize_t max_read_speed_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct mmc_blk_data *md = mmc_blk_get(dev_to_disk(dev)); + int ret = max_speed_show(atomic_read(&md->queue.max_read_speed), buf); + + mmc_blk_put(md); + return ret; +} + +static ssize_t max_read_speed_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct mmc_blk_data *md = mmc_blk_get(dev_to_disk(dev)); + int set = max_speed_store(buf, md->queue.queue); + + if (set < 0) { + mmc_blk_put(md); + return set; + } + + atomic_set(&md->queue.max_read_speed, set); + mmc_blk_put(md); + return count; +} + +static const DEVICE_ATTR(max_read_speed, S_IRUGO | S_IWUSR, + max_read_speed_show, max_read_speed_store); + +static ssize_t cache_size_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct mmc_blk_data *md = mmc_blk_get(dev_to_disk(dev)); + struct mmc_queue *mq = &md->queue; + int cache_size = atomic_read(&mq->cache_size); + int ret; + + if (!cache_size) + ret = scnprintf(buf, PAGE_SIZE, off); + else { + int speed = atomic_read(&mq->max_write_speed); + + if (!speed_valid(speed)) + ret = scnprintf(buf, PAGE_SIZE, "%uMB\n", cache_size); + else { /* We accept race between cache_jiffies and cache_used */ + unsigned long size = jiffies_and_speed_to_size( + jiffies - mq->cache_jiffies, speed); + long used = atomic_long_read(&mq->cache_used); + + if (size >= used) + size = 0; + else + size = (used - size) * 100 / cache_size + / 1024UL / 1024UL; + + ret = scnprintf(buf, PAGE_SIZE, "%uMB %lu%% used\n", + cache_size, size); + } + } + + mmc_blk_put(md); + return ret; +} + +static ssize_t cache_size_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct mmc_blk_data *md; + unsigned int set = 0; + + if (strncasecmp(off, buf, sizeof(off) - 2) + && (kstrtouint(buf, 0, &set) || (set > INT_MAX))) + return -EINVAL; + + md = mmc_blk_get(dev_to_disk(dev)); + atomic_set(&md->queue.cache_size, set); + mmc_blk_put(md); + return count; +} + +static const DEVICE_ATTR(cache_size, S_IRUGO | S_IWUSR, + cache_size_show, cache_size_store); + +/* correct for write-back */ +static long mmc_blk_cache_used(struct mmc_queue *mq, unsigned long waitfor) +{ + long used = 0; + int speed = atomic_read(&mq->max_write_speed); + + if (speed_valid(speed)) { + unsigned long size = jiffies_and_speed_to_size( + waitfor - mq->cache_jiffies, speed); + used = atomic_long_read(&mq->cache_used); + + if (size >= used) + used = 0; + else + used -= size; + } + + atomic_long_set(&mq->cache_used, used); + mq->cache_jiffies = waitfor; + + return used; +} + +static void mmc_blk_simulate_delay( + struct mmc_queue *mq, + struct request *req, + unsigned long waitfor) +{ + int max_speed; + + if (!req) + return; + + max_speed = (rq_data_dir(req) == READ) + ? atomic_read(&mq->max_read_speed) + : atomic_read(&mq->max_write_speed); + if (speed_valid(max_speed)) { + unsigned long bytes = blk_rq_bytes(req); + + if (rq_data_dir(req) != READ) { + int cache_size = atomic_read(&mq->cache_size); + + if (cache_size) { + unsigned long size = cache_size * 1024L * 1024L; + long used = mmc_blk_cache_used(mq, waitfor); + + used += bytes; + atomic_long_set(&mq->cache_used, used); + bytes = 0; + if (used > size) + bytes = used - size; + } + } + waitfor += size_and_speed_to_jiffies(bytes, max_speed); + if (time_is_after_jiffies(waitfor)) { + long msecs = jiffies_to_msecs(waitfor - jiffies); + + if (likely(msecs > 0)) + msleep(msecs); + } + } +} + +#else + +#define mmc_blk_simulate_delay(mq, req, waitfor) + +#endif + static int mmc_blk_open(struct block_device *bdev, fmode_t mode) { struct mmc_blk_data *md = mmc_blk_get(bdev->bd_disk); @@ -1284,6 +1528,23 @@ static int mmc_blk_issue_flush(struct mmc_queue *mq, struct request *req) if (ret) ret = -EIO; +#ifdef CONFIG_MMC_SIMULATE_MAX_SPEED + else if (atomic_read(&mq->cache_size)) { + long used = mmc_blk_cache_used(mq, jiffies); + + if (used) { + int speed = atomic_read(&mq->max_write_speed); + + if (speed_valid(speed)) { + unsigned long msecs = jiffies_to_msecs( + size_and_speed_to_jiffies( + used, speed)); + if (msecs) + msleep(msecs); + } + } + } +#endif blk_end_request_all(req, ret); return ret ? 0 : 1; @@ -1965,6 +2226,9 @@ static int mmc_blk_issue_rw_rq(struct mmc_queue *mq, struct request *rqc) struct mmc_async_req *areq; const u8 packed_nr = 2; u8 reqs = 0; +#ifdef CONFIG_MMC_SIMULATE_MAX_SPEED + unsigned long waitfor = jiffies; +#endif if (!rqc && !mq->mqrq_prev->req) return 0; @@ -2015,6 +2279,8 @@ static int mmc_blk_issue_rw_rq(struct mmc_queue *mq, struct request *rqc) */ mmc_blk_reset_success(md, type); + mmc_blk_simulate_delay(mq, rqc, waitfor); + if (mmc_packed_cmd(mq_rq->cmd_type)) { ret = mmc_blk_end_packed_req(mq_rq); break; @@ -2437,6 +2703,14 @@ static void mmc_blk_remove_req(struct mmc_blk_data *md) card->ext_csd.boot_ro_lockable) device_remove_file(disk_to_dev(md->disk), &md->power_ro_lock); +#ifdef CONFIG_MMC_SIMULATE_MAX_SPEED + device_remove_file(disk_to_dev(md->disk), + &dev_attr_max_write_speed); + device_remove_file(disk_to_dev(md->disk), + &dev_attr_max_read_speed); + device_remove_file(disk_to_dev(md->disk), + &dev_attr_cache_size); +#endif del_gendisk(md->disk); } @@ -2471,6 +2745,24 @@ static int mmc_add_disk(struct mmc_blk_data *md) ret = device_create_file(disk_to_dev(md->disk), &md->force_ro); if (ret) goto force_ro_fail; +#ifdef CONFIG_MMC_SIMULATE_MAX_SPEED + atomic_set(&md->queue.max_write_speed, max_write_speed); + ret = device_create_file(disk_to_dev(md->disk), + &dev_attr_max_write_speed); + if (ret) + goto max_write_speed_fail; + atomic_set(&md->queue.max_read_speed, max_read_speed); + ret = device_create_file(disk_to_dev(md->disk), + &dev_attr_max_read_speed); + if (ret) + goto max_read_speed_fail; + atomic_set(&md->queue.cache_size, cache_size); + atomic_long_set(&md->queue.cache_used, 0); + md->queue.cache_jiffies = jiffies; + ret = device_create_file(disk_to_dev(md->disk), &dev_attr_cache_size); + if (ret) + goto cache_size_fail; +#endif if ((md->area_type & MMC_BLK_DATA_AREA_BOOT) && card->ext_csd.boot_ro_lockable) { @@ -2495,6 +2787,14 @@ static int mmc_add_disk(struct mmc_blk_data *md) return ret; power_ro_lock_fail: +#ifdef CONFIG_MMC_SIMULATE_MAX_SPEED + device_remove_file(disk_to_dev(md->disk), &dev_attr_cache_size); +cache_size_fail: + device_remove_file(disk_to_dev(md->disk), &dev_attr_max_read_speed); +max_read_speed_fail: + device_remove_file(disk_to_dev(md->disk), &dev_attr_max_write_speed); +max_write_speed_fail: +#endif device_remove_file(disk_to_dev(md->disk), &md->force_ro); force_ro_fail: del_gendisk(md->disk); diff --git a/drivers/mmc/card/mmc_test.c b/drivers/mmc/card/mmc_test.c index 3678220964fe62..df382be626343d 100644 --- a/drivers/mmc/card/mmc_test.c +++ b/drivers/mmc/card/mmc_test.c @@ -818,7 +818,7 @@ static int mmc_test_nonblock_transfer(struct mmc_test_card *test, struct mmc_async_req *cur_areq = &test_areq[0].areq; struct mmc_async_req *other_areq = &test_areq[1].areq; int i; - int ret; + int ret = RESULT_OK; test_areq[0].test = test; test_areq[1].test = test; diff --git a/drivers/mmc/card/queue.c b/drivers/mmc/card/queue.c index 8037f73a109a14..1810f765f0d1e6 100644 --- a/drivers/mmc/card/queue.c +++ b/drivers/mmc/card/queue.c @@ -19,6 +19,7 @@ #include #include +#include #include "queue.h" #include "block.h" @@ -53,6 +54,11 @@ static int mmc_queue_thread(void *d) { struct mmc_queue *mq = d; struct request_queue *q = mq->queue; + struct sched_param scheduler_params = {0}; + + scheduler_params.sched_priority = 1; + + sched_setscheduler(current, SCHED_FIFO, &scheduler_params); current->flags |= PF_MEMALLOC; diff --git a/drivers/mmc/card/queue.h b/drivers/mmc/card/queue.h index 342f1e3f301e9e..fe58d31cbc7ec8 100644 --- a/drivers/mmc/card/queue.h +++ b/drivers/mmc/card/queue.h @@ -62,6 +62,14 @@ struct mmc_queue { struct mmc_queue_req mqrq[2]; struct mmc_queue_req *mqrq_cur; struct mmc_queue_req *mqrq_prev; +#ifdef CONFIG_MMC_SIMULATE_MAX_SPEED + atomic_t max_write_speed; + atomic_t max_read_speed; + atomic_t cache_size; + /* i/o tracking */ + atomic_long_t cache_used; + unsigned long cache_jiffies; +#endif }; extern int mmc_init_queue(struct mmc_queue *, struct mmc_card *, spinlock_t *, diff --git a/drivers/mmc/core/Kconfig b/drivers/mmc/core/Kconfig index 250f223aaa8038..daad32f850334d 100644 --- a/drivers/mmc/core/Kconfig +++ b/drivers/mmc/core/Kconfig @@ -22,3 +22,18 @@ config PWRSEQ_SIMPLE This driver can also be built as a module. If so, the module will be called pwrseq_simple. + +config MMC_EMBEDDED_SDIO + boolean "MMC embedded SDIO device support (EXPERIMENTAL)" + help + If you say Y here, support will be added for embedded SDIO + devices which do not contain the necessary enumeration + support in hardware to be properly detected. + +config MMC_PARANOID_SD_INIT + bool "Enable paranoid SD card initialization (EXPERIMENTAL)" + help + If you say Y here, the MMC layer will be extra paranoid + about re-trying SD init requests. This can be a useful + work-around for buggy controllers and hardware. Enable + if you are experiencing issues with SD detection. diff --git a/drivers/mmc/core/core.c b/drivers/mmc/core/core.c index 2553d903a82b9e..40ddc3e69a4d09 100644 --- a/drivers/mmc/core/core.c +++ b/drivers/mmc/core/core.c @@ -201,6 +201,19 @@ void mmc_request_done(struct mmc_host *host, struct mmc_request *mrq) pr_debug("%s: %d bytes transferred: %d\n", mmc_hostname(host), mrq->data->bytes_xfered, mrq->data->error); +#ifdef CONFIG_BLOCK + if (mrq->lat_hist_enabled) { + ktime_t completion; + u_int64_t delta_us; + + completion = ktime_get(); + delta_us = ktime_us_delta(completion, + mrq->io_start); + blk_update_latency_hist(&host->io_lat_s, + (mrq->data->flags & MMC_DATA_READ), + delta_us); + } +#endif } if (mrq->stop) { @@ -699,8 +712,16 @@ struct mmc_async_req *mmc_start_req(struct mmc_host *host, } } - if (!err && areq) + if (!err && areq) { +#ifdef CONFIG_BLOCK + if (host->latency_hist_enabled) { + areq->mrq->io_start = ktime_get(); + areq->mrq->lat_hist_enabled = 1; + } else + areq->mrq->lat_hist_enabled = 0; +#endif start_err = __mmc_start_data_req(host, areq->mrq); + } if (host->areq) mmc_post_req(host, host->areq->mrq, 0); @@ -2051,7 +2072,7 @@ void mmc_init_erase(struct mmc_card *card) } static unsigned int mmc_mmc_erase_timeout(struct mmc_card *card, - unsigned int arg, unsigned int qty) + unsigned int arg, unsigned int qty) { unsigned int erase_timeout; @@ -3026,6 +3047,22 @@ void mmc_init_context_info(struct mmc_host *host) init_waitqueue_head(&host->context_info.wait); } +#ifdef CONFIG_MMC_EMBEDDED_SDIO +void mmc_set_embedded_sdio_data(struct mmc_host *host, + struct sdio_cis *cis, + struct sdio_cccr *cccr, + struct sdio_embedded_func *funcs, + int num_funcs) +{ + host->embedded_sdio_data.cis = cis; + host->embedded_sdio_data.cccr = cccr; + host->embedded_sdio_data.funcs = funcs; + host->embedded_sdio_data.num_funcs = num_funcs; +} + +EXPORT_SYMBOL(mmc_set_embedded_sdio_data); +#endif + static int __init mmc_init(void) { int ret; @@ -3058,6 +3095,56 @@ static void __exit mmc_exit(void) mmc_unregister_bus(); } +#ifdef CONFIG_BLOCK +static ssize_t +latency_hist_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct mmc_host *host = cls_dev_to_mmc_host(dev); + + return blk_latency_hist_show(&host->io_lat_s, buf); +} + +/* + * Values permitted 0, 1, 2. + * 0 -> Disable IO latency histograms (default) + * 1 -> Enable IO latency histograms + * 2 -> Zero out IO latency histograms + */ +static ssize_t +latency_hist_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct mmc_host *host = cls_dev_to_mmc_host(dev); + long value; + + if (kstrtol(buf, 0, &value)) + return -EINVAL; + if (value == BLK_IO_LAT_HIST_ZERO) + blk_zero_latency_hist(&host->io_lat_s); + else if (value == BLK_IO_LAT_HIST_ENABLE || + value == BLK_IO_LAT_HIST_DISABLE) + host->latency_hist_enabled = value; + return count; +} + +static DEVICE_ATTR(latency_hist, S_IRUGO | S_IWUSR, + latency_hist_show, latency_hist_store); + +void +mmc_latency_hist_sysfs_init(struct mmc_host *host) +{ + if (device_create_file(&host->class_dev, &dev_attr_latency_hist)) + dev_err(&host->class_dev, + "Failed to create latency_hist sysfs entry\n"); +} + +void +mmc_latency_hist_sysfs_exit(struct mmc_host *host) +{ + device_remove_file(&host->class_dev, &dev_attr_latency_hist); +} +#endif + subsys_initcall(mmc_init); module_exit(mmc_exit); diff --git a/drivers/mmc/core/host.c b/drivers/mmc/core/host.c index 98f25ffb42583a..348b58b77b3b6a 100644 --- a/drivers/mmc/core/host.c +++ b/drivers/mmc/core/host.c @@ -31,8 +31,6 @@ #include "slot-gpio.h" #include "pwrseq.h" -#define cls_dev_to_mmc_host(d) container_of(d, struct mmc_host, class_dev) - static DEFINE_IDA(mmc_host_ida); static DEFINE_SPINLOCK(mmc_host_lock); @@ -430,8 +428,13 @@ int mmc_add_host(struct mmc_host *host) mmc_add_host_debugfs(host); #endif +#ifdef CONFIG_BLOCK + mmc_latency_hist_sysfs_init(host); +#endif + mmc_start_host(host); - mmc_register_pm_notifier(host); + if (!(host->pm_flags & MMC_PM_IGNORE_PM_NOTIFY)) + mmc_register_pm_notifier(host); return 0; } @@ -448,13 +451,18 @@ EXPORT_SYMBOL(mmc_add_host); */ void mmc_remove_host(struct mmc_host *host) { - mmc_unregister_pm_notifier(host); + if (!(host->pm_flags & MMC_PM_IGNORE_PM_NOTIFY)) + mmc_unregister_pm_notifier(host); mmc_stop_host(host); #ifdef CONFIG_DEBUG_FS mmc_remove_host_debugfs(host); #endif +#ifdef CONFIG_BLOCK + mmc_latency_hist_sysfs_exit(host); +#endif + device_del(&host->class_dev); led_trigger_unregister_simple(host->led); diff --git a/drivers/mmc/core/host.h b/drivers/mmc/core/host.h index 992bf53976337f..bf38533406fd88 100644 --- a/drivers/mmc/core/host.h +++ b/drivers/mmc/core/host.h @@ -12,6 +12,8 @@ #define _MMC_CORE_HOST_H #include +#define cls_dev_to_mmc_host(d) container_of(d, struct mmc_host, class_dev) + int mmc_register_host_class(void); void mmc_unregister_host_class(void); @@ -21,5 +23,8 @@ void mmc_retune_hold(struct mmc_host *host); void mmc_retune_release(struct mmc_host *host); int mmc_retune(struct mmc_host *host); +void mmc_latency_hist_sysfs_init(struct mmc_host *host); +void mmc_latency_hist_sysfs_exit(struct mmc_host *host); + #endif diff --git a/drivers/mmc/core/sd.c b/drivers/mmc/core/sd.c index 73c762a28dfed8..a0aa64eac123c5 100644 --- a/drivers/mmc/core/sd.c +++ b/drivers/mmc/core/sd.c @@ -223,6 +223,7 @@ static int mmc_decode_scr(struct mmc_card *card) static int mmc_read_ssr(struct mmc_card *card) { unsigned int au, es, et, eo; + u32 *raw_ssr; int i; if (!(card->csd.cmdclass & CCC_APP_SPEC)) { @@ -231,14 +232,21 @@ static int mmc_read_ssr(struct mmc_card *card) return 0; } - if (mmc_app_sd_status(card, card->raw_ssr)) { + raw_ssr = kmalloc(sizeof(card->raw_ssr), GFP_KERNEL); + if (!raw_ssr) + return -ENOMEM; + + if (mmc_app_sd_status(card, raw_ssr)) { pr_warn("%s: problem reading SD Status register\n", mmc_hostname(card->host)); + kfree(raw_ssr); return 0; } for (i = 0; i < 16; i++) - card->raw_ssr[i] = be32_to_cpu(card->raw_ssr[i]); + card->raw_ssr[i] = be32_to_cpu(raw_ssr[i]); + + kfree(raw_ssr); /* * UNSTUFF_BITS only works with four u32s so we have to offset the @@ -839,6 +847,9 @@ int mmc_sd_setup_card(struct mmc_host *host, struct mmc_card *card, bool reinit) { int err; +#ifdef CONFIG_MMC_PARANOID_SD_INIT + int retries; +#endif if (!reinit) { /* @@ -865,7 +876,26 @@ int mmc_sd_setup_card(struct mmc_host *host, struct mmc_card *card, /* * Fetch switch information from card. */ +#ifdef CONFIG_MMC_PARANOID_SD_INIT + for (retries = 1; retries <= 3; retries++) { + err = mmc_read_switch(card); + if (!err) { + if (retries > 1) { + printk(KERN_WARNING + "%s: recovered\n", + mmc_hostname(host)); + } + break; + } else { + printk(KERN_WARNING + "%s: read switch failed (attempt %d)\n", + mmc_hostname(host), retries); + } + } +#else err = mmc_read_switch(card); +#endif + if (err) return err; } @@ -1063,7 +1093,10 @@ static int mmc_sd_alive(struct mmc_host *host) */ static void mmc_sd_detect(struct mmc_host *host) { - int err; + int err = 0; +#ifdef CONFIG_MMC_PARANOID_SD_INIT + int retries = 5; +#endif BUG_ON(!host); BUG_ON(!host->card); @@ -1073,7 +1106,23 @@ static void mmc_sd_detect(struct mmc_host *host) /* * Just check if our card has been removed. */ +#ifdef CONFIG_MMC_PARANOID_SD_INIT + while(retries) { + err = mmc_send_status(host->card, NULL); + if (err) { + retries--; + udelay(5); + continue; + } + break; + } + if (!retries) { + printk(KERN_ERR "%s(%s): Unable to re-detect card (%d)\n", + __func__, mmc_hostname(host), err); + } +#else err = _mmc_detect_card_removed(host); +#endif mmc_put_card(host->card); @@ -1135,6 +1184,9 @@ static int mmc_sd_suspend(struct mmc_host *host) static int _mmc_sd_resume(struct mmc_host *host) { int err = 0; +#ifdef CONFIG_MMC_PARANOID_SD_INIT + int retries; +#endif BUG_ON(!host); BUG_ON(!host->card); @@ -1145,7 +1197,23 @@ static int _mmc_sd_resume(struct mmc_host *host) goto out; mmc_power_up(host, host->card->ocr); +#ifdef CONFIG_MMC_PARANOID_SD_INIT + retries = 5; + while (retries) { + err = mmc_sd_init_card(host, host->card->ocr, host->card); + + if (err) { + printk(KERN_ERR "%s: Re-init card rc = %d (retries = %d)\n", + mmc_hostname(host), err, retries); + mdelay(5); + retries--; + continue; + } + break; + } +#else err = mmc_sd_init_card(host, host->card->ocr, host->card); +#endif mmc_card_clr_suspended(host->card); out: @@ -1220,6 +1288,9 @@ int mmc_attach_sd(struct mmc_host *host) { int err; u32 ocr, rocr; +#ifdef CONFIG_MMC_PARANOID_SD_INIT + int retries; +#endif BUG_ON(!host); WARN_ON(!host->claimed); @@ -1256,9 +1327,27 @@ int mmc_attach_sd(struct mmc_host *host) /* * Detect and init the card. */ +#ifdef CONFIG_MMC_PARANOID_SD_INIT + retries = 5; + while (retries) { + err = mmc_sd_init_card(host, rocr, NULL); + if (err) { + retries--; + continue; + } + break; + } + + if (!retries) { + printk(KERN_ERR "%s: mmc_sd_init_card() failure (err = %d)\n", + mmc_hostname(host), err); + goto err; + } +#else err = mmc_sd_init_card(host, rocr, NULL); if (err) goto err; +#endif mmc_release_host(host); err = mmc_add_card(host->card); diff --git a/drivers/mmc/core/sdio.c b/drivers/mmc/core/sdio.c index bd44ba8116d158..b5ec3c8cf58027 100644 --- a/drivers/mmc/core/sdio.c +++ b/drivers/mmc/core/sdio.c @@ -10,6 +10,7 @@ */ #include +#include #include #include @@ -21,6 +22,7 @@ #include "core.h" #include "bus.h" +#include "host.h" #include "sd.h" #include "sdio_bus.h" #include "mmc_ops.h" @@ -28,6 +30,10 @@ #include "sdio_ops.h" #include "sdio_cis.h" +#ifdef CONFIG_MMC_EMBEDDED_SDIO +#include +#endif + static int sdio_read_fbr(struct sdio_func *func) { int ret; @@ -697,19 +703,35 @@ static int mmc_sdio_init_card(struct mmc_host *host, u32 ocr, goto finish; } - /* - * Read the common registers. - */ - err = sdio_read_cccr(card, ocr); - if (err) - goto remove; +#ifdef CONFIG_MMC_EMBEDDED_SDIO + if (host->embedded_sdio_data.cccr) + memcpy(&card->cccr, host->embedded_sdio_data.cccr, sizeof(struct sdio_cccr)); + else { +#endif + /* + * Read the common registers. + */ + err = sdio_read_cccr(card, ocr); + if (err) + goto remove; +#ifdef CONFIG_MMC_EMBEDDED_SDIO + } +#endif - /* - * Read the common CIS tuples. - */ - err = sdio_read_common_cis(card); - if (err) - goto remove; +#ifdef CONFIG_MMC_EMBEDDED_SDIO + if (host->embedded_sdio_data.cis) + memcpy(&card->cis, host->embedded_sdio_data.cis, sizeof(struct sdio_cis)); + else { +#endif + /* + * Read the common CIS tuples. + */ + err = sdio_read_common_cis(card); + if (err) + goto remove; +#ifdef CONFIG_MMC_EMBEDDED_SDIO + } +#endif if (oldcard) { int same = (card->cis.vendor == oldcard->cis.vendor && @@ -1118,14 +1140,36 @@ int mmc_attach_sdio(struct mmc_host *host) funcs = (ocr & 0x70000000) >> 28; card->sdio_funcs = 0; +#ifdef CONFIG_MMC_EMBEDDED_SDIO + if (host->embedded_sdio_data.funcs) + card->sdio_funcs = funcs = host->embedded_sdio_data.num_funcs; +#endif + /* * Initialize (but don't add) all present functions. */ for (i = 0; i < funcs; i++, card->sdio_funcs++) { - err = sdio_init_func(host->card, i + 1); - if (err) - goto remove; - +#ifdef CONFIG_MMC_EMBEDDED_SDIO + if (host->embedded_sdio_data.funcs) { + struct sdio_func *tmp; + + tmp = sdio_alloc_func(host->card); + if (IS_ERR(tmp)) + goto remove; + tmp->num = (i + 1); + card->sdio_func[i] = tmp; + tmp->class = host->embedded_sdio_data.funcs[i].f_class; + tmp->max_blksize = host->embedded_sdio_data.funcs[i].f_maxblksize; + tmp->vendor = card->cis.vendor; + tmp->device = card->cis.device; + } else { +#endif + err = sdio_init_func(host->card, i + 1); + if (err) + goto remove; +#ifdef CONFIG_MMC_EMBEDDED_SDIO + } +#endif /* * Enable Runtime PM for this func (if supported) */ @@ -1173,3 +1217,42 @@ int mmc_attach_sdio(struct mmc_host *host) return err; } +int sdio_reset_comm(struct mmc_card *card) +{ + struct mmc_host *host = card->host; + u32 ocr; + u32 rocr; + int err; + + printk("%s():\n", __func__); + mmc_claim_host(host); + + mmc_retune_disable(host); + + mmc_go_idle(host); + + mmc_set_clock(host, host->f_min); + + err = mmc_send_io_op_cond(host, 0, &ocr); + if (err) + goto err; + + rocr = mmc_select_voltage(host, ocr); + if (!rocr) { + err = -EINVAL; + goto err; + } + + err = mmc_sdio_init_card(host, rocr, card, 0); + if (err) + goto err; + + mmc_release_host(host); + return 0; +err: + printk("%s: Error resetting SDIO communications (%d)\n", + mmc_hostname(host), err); + mmc_release_host(host); + return err; +} +EXPORT_SYMBOL(sdio_reset_comm); diff --git a/drivers/mmc/core/sdio_bus.c b/drivers/mmc/core/sdio_bus.c index 86f5b3223aaeec..1499d5333c79ba 100644 --- a/drivers/mmc/core/sdio_bus.c +++ b/drivers/mmc/core/sdio_bus.c @@ -28,6 +28,10 @@ #include "sdio_cis.h" #include "sdio_bus.h" +#ifdef CONFIG_MMC_EMBEDDED_SDIO +#include +#endif + #define to_sdio_driver(d) container_of(d, struct sdio_driver, drv) /* show configuration fields */ @@ -263,7 +267,14 @@ static void sdio_release_func(struct device *dev) { struct sdio_func *func = dev_to_sdio_func(dev); - sdio_free_func_cis(func); +#ifdef CONFIG_MMC_EMBEDDED_SDIO + /* + * If this device is embedded then we never allocated + * cis tables for this func + */ + if (!func->card->host->embedded_sdio_data.funcs) +#endif + sdio_free_func_cis(func); kfree(func->info); diff --git a/drivers/mmc/core/sdio_io.c b/drivers/mmc/core/sdio_io.c index 406e5f037e3203..3734cba53dbb77 100644 --- a/drivers/mmc/core/sdio_io.c +++ b/drivers/mmc/core/sdio_io.c @@ -389,6 +389,39 @@ u8 sdio_readb(struct sdio_func *func, unsigned int addr, int *err_ret) } EXPORT_SYMBOL_GPL(sdio_readb); +/** + * sdio_readb_ext - read a single byte from a SDIO function + * @func: SDIO function to access + * @addr: address to read + * @err_ret: optional status value from transfer + * @in: value to add to argument + * + * Reads a single byte from the address space of a given SDIO + * function. If there is a problem reading the address, 0xff + * is returned and @err_ret will contain the error code. + */ +unsigned char sdio_readb_ext(struct sdio_func *func, unsigned int addr, + int *err_ret, unsigned in) +{ + int ret; + unsigned char val; + + BUG_ON(!func); + + if (err_ret) + *err_ret = 0; + + ret = mmc_io_rw_direct(func->card, 0, func->num, addr, (u8)in, &val); + if (ret) { + if (err_ret) + *err_ret = ret; + return 0xFF; + } + + return val; +} +EXPORT_SYMBOL_GPL(sdio_readb_ext); + /** * sdio_writeb - write a single byte to a SDIO function * @func: SDIO function to access diff --git a/drivers/mmc/host/mxs-mmc.c b/drivers/mmc/host/mxs-mmc.c index 44ecebd1ea8c18..c8b8ac66ff7e3a 100644 --- a/drivers/mmc/host/mxs-mmc.c +++ b/drivers/mmc/host/mxs-mmc.c @@ -309,6 +309,9 @@ static void mxs_mmc_ac(struct mxs_mmc_host *host) cmd0 = BF_SSP(cmd->opcode, CMD0_CMD); cmd1 = cmd->arg; + if (cmd->opcode == MMC_STOP_TRANSMISSION) + cmd0 |= BM_SSP_CMD0_APPEND_8CYC; + if (host->sdio_irq_en) { ctrl0 |= BM_SSP_CTRL0_SDIO_IRQ_CHECK; cmd0 |= BM_SSP_CMD0_CONT_CLKING_EN | BM_SSP_CMD0_SLOW_CLKING_EN; @@ -417,8 +420,7 @@ static void mxs_mmc_adtc(struct mxs_mmc_host *host) ssp->base + HW_SSP_BLOCK_SIZE); } - if ((cmd->opcode == MMC_STOP_TRANSMISSION) || - (cmd->opcode == SD_IO_RW_EXTENDED)) + if (cmd->opcode == SD_IO_RW_EXTENDED) cmd0 |= BM_SSP_CMD0_APPEND_8CYC; cmd1 = cmd->arg; diff --git a/drivers/mmc/host/sdhci-acpi.c b/drivers/mmc/host/sdhci-acpi.c index 81d4dc034793dd..fddd0be196f4e3 100644 --- a/drivers/mmc/host/sdhci-acpi.c +++ b/drivers/mmc/host/sdhci-acpi.c @@ -394,7 +394,8 @@ static int sdhci_acpi_probe(struct platform_device *pdev) /* Power on the SDHCI controller and its children */ acpi_device_fix_up_power(device); list_for_each_entry(child, &device->children, node) - acpi_device_fix_up_power(child); + if (child->status.present && child->status.enabled) + acpi_device_fix_up_power(child); if (acpi_bus_get_status(device) || !device->status.present) return -ENODEV; diff --git a/drivers/mmc/host/sdhci.c b/drivers/mmc/host/sdhci.c index 42ef3ebb1d8cf9..ba637ff8aa7e44 100644 --- a/drivers/mmc/host/sdhci.c +++ b/drivers/mmc/host/sdhci.c @@ -2086,16 +2086,32 @@ static int sdhci_execute_tuning(struct mmc_host *mmc, u32 opcode) if (!host->tuning_done) { pr_info(DRIVER_NAME ": Timeout waiting for Buffer Read Ready interrupt during tuning procedure, falling back to fixed sampling clock\n"); - - sdhci_do_reset(host, SDHCI_RESET_CMD); - sdhci_do_reset(host, SDHCI_RESET_DATA); - ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2); ctrl &= ~SDHCI_CTRL_TUNED_CLK; ctrl &= ~SDHCI_CTRL_EXEC_TUNING; sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2); + sdhci_do_reset(host, SDHCI_RESET_CMD); + sdhci_do_reset(host, SDHCI_RESET_DATA); + err = -EIO; + + if (cmd.opcode != MMC_SEND_TUNING_BLOCK_HS200) + goto out; + + sdhci_writel(host, host->ier, SDHCI_INT_ENABLE); + sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE); + + spin_unlock_irqrestore(&host->lock, flags); + + memset(&cmd, 0, sizeof(cmd)); + cmd.opcode = MMC_STOP_TRANSMISSION; + cmd.flags = MMC_RSP_SPI_R1B | MMC_RSP_R1B | MMC_CMD_AC; + cmd.busy_timeout = 50; + mmc_wait_for_cmd(mmc, &cmd, 0); + + spin_lock_irqsave(&host->lock, flags); + goto out; } @@ -2703,7 +2719,8 @@ static irqreturn_t sdhci_irq(int irq, void *dev_id) if (intmask & SDHCI_INT_RETUNE) mmc_retune_needed(host->mmc); - if (intmask & SDHCI_INT_CARD_INT) { + if ((intmask & SDHCI_INT_CARD_INT) && + (host->ier & SDHCI_INT_CARD_INT)) { sdhci_enable_sdio_irq_nolock(host, false); host->thread_isr |= SDHCI_INT_CARD_INT; result = IRQ_WAKE_THREAD; diff --git a/drivers/mtd/nand/Kconfig b/drivers/mtd/nand/Kconfig index 7b7a887b4709f0..50ee1bad3690f5 100644 --- a/drivers/mtd/nand/Kconfig +++ b/drivers/mtd/nand/Kconfig @@ -1,3 +1,10 @@ +config MTD_NAND_IDS + tristate "Include chip ids for known NAND devices." + depends on MTD + help + Useful for NAND drivers that do not use the NAND subsystem but + still like to take advantage of the known chip information. + config MTD_NAND_ECC tristate @@ -109,9 +116,6 @@ config MTD_NAND_OMAP_BCH config MTD_NAND_OMAP_BCH_BUILD def_tristate MTD_NAND_OMAP2 && MTD_NAND_OMAP_BCH -config MTD_NAND_IDS - tristate - config MTD_NAND_RICOH tristate "Ricoh xD card reader" default n @@ -537,7 +541,7 @@ config MTD_NAND_FSMC Flexible Static Memory Controller (FSMC) config MTD_NAND_XWAY - tristate "Support for NAND on Lantiq XWAY SoC" + bool "Support for NAND on Lantiq XWAY SoC" depends on LANTIQ && SOC_TYPE_XWAY help Enables support for NAND Flash chips on Lantiq XWAY SoCs. NAND is attached diff --git a/drivers/mtd/nand/lpc32xx_mlc.c b/drivers/mtd/nand/lpc32xx_mlc.c index 852388171f2033..bc6e49af063a25 100644 --- a/drivers/mtd/nand/lpc32xx_mlc.c +++ b/drivers/mtd/nand/lpc32xx_mlc.c @@ -776,7 +776,7 @@ static int lpc32xx_nand_probe(struct platform_device *pdev) init_completion(&host->comp_controller); host->irq = platform_get_irq(pdev, 0); - if ((host->irq < 0) || (host->irq >= NR_IRQS)) { + if (host->irq < 0) { dev_err(&pdev->dev, "failed to get platform irq\n"); res = -EINVAL; goto err_exit3; diff --git a/drivers/mtd/nand/xway_nand.c b/drivers/mtd/nand/xway_nand.c index 1f2948c0c458d6..895101a5e68645 100644 --- a/drivers/mtd/nand/xway_nand.c +++ b/drivers/mtd/nand/xway_nand.c @@ -232,7 +232,6 @@ static const struct of_device_id xway_nand_match[] = { { .compatible = "lantiq,nand-xway" }, {}, }; -MODULE_DEVICE_TABLE(of, xway_nand_match); static struct platform_driver xway_nand_driver = { .probe = xway_nand_probe, @@ -243,6 +242,4 @@ static struct platform_driver xway_nand_driver = { }, }; -module_platform_driver(xway_nand_driver); - -MODULE_LICENSE("GPL"); +builtin_platform_driver(xway_nand_driver); diff --git a/drivers/mtd/spi-nor/cadence-quadspi.c b/drivers/mtd/spi-nor/cadence-quadspi.c index d403ba7b8f432e..d489fbd07c12b6 100644 --- a/drivers/mtd/spi-nor/cadence-quadspi.c +++ b/drivers/mtd/spi-nor/cadence-quadspi.c @@ -1077,12 +1077,14 @@ static int cqspi_setup_flash(struct cqspi_st *cqspi, struct device_node *np) /* Get flash device data */ for_each_available_child_of_node(dev->of_node, np) { - if (of_property_read_u32(np, "reg", &cs)) { + ret = of_property_read_u32(np, "reg", &cs); + if (ret) { dev_err(dev, "Couldn't determine chip select.\n"); goto err; } - if (cs > CQSPI_MAX_CHIPSELECT) { + if (cs >= CQSPI_MAX_CHIPSELECT) { + ret = -EINVAL; dev_err(dev, "Chip select %d out of range.\n", cs); goto err; } diff --git a/drivers/net/can/c_can/c_can_pci.c b/drivers/net/can/c_can/c_can_pci.c index 7be393c96b1a04..cf7c18947189a1 100644 --- a/drivers/net/can/c_can/c_can_pci.c +++ b/drivers/net/can/c_can/c_can_pci.c @@ -161,6 +161,7 @@ static int c_can_pci_probe(struct pci_dev *pdev, dev->irq = pdev->irq; priv->base = addr; + priv->device = &pdev->dev; if (!c_can_pci_data->freq) { dev_err(&pdev->dev, "no clock frequency defined\n"); diff --git a/drivers/net/can/ti_hecc.c b/drivers/net/can/ti_hecc.c index 680d1ff07a55dd..6749b182946941 100644 --- a/drivers/net/can/ti_hecc.c +++ b/drivers/net/can/ti_hecc.c @@ -948,7 +948,12 @@ static int ti_hecc_probe(struct platform_device *pdev) netif_napi_add(ndev, &priv->napi, ti_hecc_rx_poll, HECC_DEF_NAPI_WEIGHT); - clk_enable(priv->clk); + err = clk_prepare_enable(priv->clk); + if (err) { + dev_err(&pdev->dev, "clk_prepare_enable() failed\n"); + goto probe_exit_clk; + } + err = register_candev(ndev); if (err) { dev_err(&pdev->dev, "register_candev() failed\n"); @@ -981,7 +986,7 @@ static int ti_hecc_remove(struct platform_device *pdev) struct ti_hecc_priv *priv = netdev_priv(ndev); unregister_candev(ndev); - clk_disable(priv->clk); + clk_disable_unprepare(priv->clk); clk_put(priv->clk); res = platform_get_resource(pdev, IORESOURCE_MEM, 0); iounmap(priv->base); @@ -1006,7 +1011,7 @@ static int ti_hecc_suspend(struct platform_device *pdev, pm_message_t state) hecc_set_bit(priv, HECC_CANMC, HECC_CANMC_PDR); priv->can.state = CAN_STATE_SLEEPING; - clk_disable(priv->clk); + clk_disable_unprepare(priv->clk); return 0; } @@ -1015,8 +1020,11 @@ static int ti_hecc_resume(struct platform_device *pdev) { struct net_device *dev = platform_get_drvdata(pdev); struct ti_hecc_priv *priv = netdev_priv(dev); + int err; - clk_enable(priv->clk); + err = clk_prepare_enable(priv->clk); + if (err) + return err; hecc_clear_bit(priv, HECC_CANMC, HECC_CANMC_PDR); priv->can.state = CAN_STATE_ERROR_ACTIVE; diff --git a/drivers/net/dsa/bcm_sf2.c b/drivers/net/dsa/bcm_sf2.c index 9ec33b51a0edad..2ce7ae97ac9148 100644 --- a/drivers/net/dsa/bcm_sf2.c +++ b/drivers/net/dsa/bcm_sf2.c @@ -393,7 +393,7 @@ static int bcm_sf2_sw_mdio_read(struct mii_bus *bus, int addr, int regnum) if (addr == BRCM_PSEUDO_PHY_ADDR && priv->indir_phy_mask & BIT(addr)) return bcm_sf2_sw_indir_rw(priv, 1, addr, regnum, 0); else - return mdiobus_read(priv->master_mii_bus, addr, regnum); + return mdiobus_read_nested(priv->master_mii_bus, addr, regnum); } static int bcm_sf2_sw_mdio_write(struct mii_bus *bus, int addr, int regnum, @@ -407,7 +407,7 @@ static int bcm_sf2_sw_mdio_write(struct mii_bus *bus, int addr, int regnum, if (addr == BRCM_PSEUDO_PHY_ADDR && priv->indir_phy_mask & BIT(addr)) bcm_sf2_sw_indir_rw(priv, 0, addr, regnum, val); else - mdiobus_write(priv->master_mii_bus, addr, regnum, val); + mdiobus_write_nested(priv->master_mii_bus, addr, regnum, val); return 0; } @@ -982,6 +982,7 @@ static int bcm_sf2_sw_probe(struct platform_device *pdev) const char *reg_names[BCM_SF2_REGS_NUM] = BCM_SF2_REGS_NAME; struct device_node *dn = pdev->dev.of_node; struct b53_platform_data *pdata; + struct dsa_switch_ops *ops; struct bcm_sf2_priv *priv; struct b53_device *dev; struct dsa_switch *ds; @@ -995,6 +996,10 @@ static int bcm_sf2_sw_probe(struct platform_device *pdev) if (!priv) return -ENOMEM; + ops = devm_kzalloc(&pdev->dev, sizeof(*ops), GFP_KERNEL); + if (!ops) + return -ENOMEM; + dev = b53_switch_alloc(&pdev->dev, &bcm_sf2_io_ops, priv); if (!dev) return -ENOMEM; @@ -1014,6 +1019,8 @@ static int bcm_sf2_sw_probe(struct platform_device *pdev) ds = dev->ds; /* Override the parts that are non-standard wrt. normal b53 devices */ + memcpy(ops, ds->ops, sizeof(*ops)); + ds->ops = ops; ds->ops->get_tag_protocol = bcm_sf2_sw_get_tag_protocol; ds->ops->setup = bcm_sf2_sw_setup; ds->ops->get_phy_flags = bcm_sf2_sw_get_phy_flags; diff --git a/drivers/net/ethernet/broadcom/bcmsysport.c b/drivers/net/ethernet/broadcom/bcmsysport.c index 25d1eb4933d0b8..be7ec5a76a5410 100644 --- a/drivers/net/ethernet/broadcom/bcmsysport.c +++ b/drivers/net/ethernet/broadcom/bcmsysport.c @@ -710,11 +710,8 @@ static unsigned int __bcm_sysport_tx_reclaim(struct bcm_sysport_priv *priv, unsigned int c_index, last_c_index, last_tx_cn, num_tx_cbs; unsigned int pkts_compl = 0, bytes_compl = 0; struct bcm_sysport_cb *cb; - struct netdev_queue *txq; u32 hw_ind; - txq = netdev_get_tx_queue(ndev, ring->index); - /* Compute how many descriptors have been processed since last call */ hw_ind = tdma_readl(priv, TDMA_DESC_RING_PROD_CONS_INDEX(ring->index)); c_index = (hw_ind >> RING_CONS_INDEX_SHIFT) & RING_CONS_INDEX_MASK; @@ -745,9 +742,6 @@ static unsigned int __bcm_sysport_tx_reclaim(struct bcm_sysport_priv *priv, ring->c_index = c_index; - if (netif_tx_queue_stopped(txq) && pkts_compl) - netif_tx_wake_queue(txq); - netif_dbg(priv, tx_done, ndev, "ring=%d c_index=%d pkts_compl=%d, bytes_compl=%d\n", ring->index, ring->c_index, pkts_compl, bytes_compl); @@ -759,16 +753,33 @@ static unsigned int __bcm_sysport_tx_reclaim(struct bcm_sysport_priv *priv, static unsigned int bcm_sysport_tx_reclaim(struct bcm_sysport_priv *priv, struct bcm_sysport_tx_ring *ring) { + struct netdev_queue *txq; unsigned int released; unsigned long flags; + txq = netdev_get_tx_queue(priv->netdev, ring->index); + spin_lock_irqsave(&ring->lock, flags); released = __bcm_sysport_tx_reclaim(priv, ring); + if (released) + netif_tx_wake_queue(txq); + spin_unlock_irqrestore(&ring->lock, flags); return released; } +/* Locked version of the per-ring TX reclaim, but does not wake the queue */ +static void bcm_sysport_tx_clean(struct bcm_sysport_priv *priv, + struct bcm_sysport_tx_ring *ring) +{ + unsigned long flags; + + spin_lock_irqsave(&ring->lock, flags); + __bcm_sysport_tx_reclaim(priv, ring); + spin_unlock_irqrestore(&ring->lock, flags); +} + static int bcm_sysport_tx_poll(struct napi_struct *napi, int budget) { struct bcm_sysport_tx_ring *ring = @@ -1253,7 +1264,7 @@ static void bcm_sysport_fini_tx_ring(struct bcm_sysport_priv *priv, napi_disable(&ring->napi); netif_napi_del(&ring->napi); - bcm_sysport_tx_reclaim(priv, ring); + bcm_sysport_tx_clean(priv, ring); kfree(ring->cbs); ring->cbs = NULL; diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_reg.h b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_reg.h index 878950a42e6c23..2cf8b1d82d6a44 100644 --- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_reg.h +++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_reg.h @@ -1007,9 +1007,7 @@ static inline void dsaf_write_reg(void __iomem *base, u32 reg, u32 value) { - u8 __iomem *reg_addr = ACCESS_ONCE(base); - - writel(value, reg_addr + reg); + writel(value, base + reg); } #define dsaf_write_dev(a, reg, value) \ @@ -1017,9 +1015,7 @@ static inline void dsaf_write_reg(void __iomem *base, u32 reg, u32 value) static inline u32 dsaf_read_reg(u8 __iomem *base, u32 reg) { - u8 __iomem *reg_addr = ACCESS_ONCE(base); - - return readl(reg_addr + reg); + return readl(base + reg); } static inline void dsaf_write_syscon(struct regmap *base, u32 reg, u32 value) diff --git a/drivers/net/ethernet/marvell/mvpp2.c b/drivers/net/ethernet/marvell/mvpp2.c index 1026c452e39de4..930c8165f2a812 100644 --- a/drivers/net/ethernet/marvell/mvpp2.c +++ b/drivers/net/ethernet/marvell/mvpp2.c @@ -770,6 +770,17 @@ struct mvpp2_rx_desc { u32 reserved8; }; +struct mvpp2_txq_pcpu_buf { + /* Transmitted SKB */ + struct sk_buff *skb; + + /* Physical address of transmitted buffer */ + dma_addr_t phys; + + /* Size transmitted */ + size_t size; +}; + /* Per-CPU Tx queue control */ struct mvpp2_txq_pcpu { int cpu; @@ -785,11 +796,8 @@ struct mvpp2_txq_pcpu { /* Number of Tx DMA descriptors reserved for each CPU */ int reserved_num; - /* Array of transmitted skb */ - struct sk_buff **tx_skb; - - /* Array of transmitted buffers' physical addresses */ - dma_addr_t *tx_buffs; + /* Infos about transmitted buffers */ + struct mvpp2_txq_pcpu_buf *buffs; /* Index of last TX DMA descriptor that was inserted */ int txq_put_index; @@ -979,10 +987,11 @@ static void mvpp2_txq_inc_put(struct mvpp2_txq_pcpu *txq_pcpu, struct sk_buff *skb, struct mvpp2_tx_desc *tx_desc) { - txq_pcpu->tx_skb[txq_pcpu->txq_put_index] = skb; - if (skb) - txq_pcpu->tx_buffs[txq_pcpu->txq_put_index] = - tx_desc->buf_phys_addr; + struct mvpp2_txq_pcpu_buf *tx_buf = + txq_pcpu->buffs + txq_pcpu->txq_put_index; + tx_buf->skb = skb; + tx_buf->size = tx_desc->data_size; + tx_buf->phys = tx_desc->buf_phys_addr; txq_pcpu->txq_put_index++; if (txq_pcpu->txq_put_index == txq_pcpu->size) txq_pcpu->txq_put_index = 0; @@ -4401,17 +4410,16 @@ static void mvpp2_txq_bufs_free(struct mvpp2_port *port, int i; for (i = 0; i < num; i++) { - dma_addr_t buf_phys_addr = - txq_pcpu->tx_buffs[txq_pcpu->txq_get_index]; - struct sk_buff *skb = txq_pcpu->tx_skb[txq_pcpu->txq_get_index]; + struct mvpp2_txq_pcpu_buf *tx_buf = + txq_pcpu->buffs + txq_pcpu->txq_get_index; mvpp2_txq_inc_get(txq_pcpu); - dma_unmap_single(port->dev->dev.parent, buf_phys_addr, - skb_headlen(skb), DMA_TO_DEVICE); - if (!skb) + dma_unmap_single(port->dev->dev.parent, tx_buf->phys, + tx_buf->size, DMA_TO_DEVICE); + if (!tx_buf->skb) continue; - dev_kfree_skb_any(skb); + dev_kfree_skb_any(tx_buf->skb); } } @@ -4651,15 +4659,10 @@ static int mvpp2_txq_init(struct mvpp2_port *port, for_each_present_cpu(cpu) { txq_pcpu = per_cpu_ptr(txq->pcpu, cpu); txq_pcpu->size = txq->size; - txq_pcpu->tx_skb = kmalloc(txq_pcpu->size * - sizeof(*txq_pcpu->tx_skb), - GFP_KERNEL); - if (!txq_pcpu->tx_skb) - goto error; - - txq_pcpu->tx_buffs = kmalloc(txq_pcpu->size * - sizeof(dma_addr_t), GFP_KERNEL); - if (!txq_pcpu->tx_buffs) + txq_pcpu->buffs = kmalloc(txq_pcpu->size * + sizeof(struct mvpp2_txq_pcpu_buf), + GFP_KERNEL); + if (!txq_pcpu->buffs) goto error; txq_pcpu->count = 0; @@ -4673,8 +4676,7 @@ static int mvpp2_txq_init(struct mvpp2_port *port, error: for_each_present_cpu(cpu) { txq_pcpu = per_cpu_ptr(txq->pcpu, cpu); - kfree(txq_pcpu->tx_skb); - kfree(txq_pcpu->tx_buffs); + kfree(txq_pcpu->buffs); } dma_free_coherent(port->dev->dev.parent, @@ -4693,8 +4695,7 @@ static void mvpp2_txq_deinit(struct mvpp2_port *port, for_each_present_cpu(cpu) { txq_pcpu = per_cpu_ptr(txq->pcpu, cpu); - kfree(txq_pcpu->tx_skb); - kfree(txq_pcpu->tx_buffs); + kfree(txq_pcpu->buffs); } if (txq->descs) diff --git a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c index fb8bb027b69c33..d223e7cb68ba31 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c @@ -1740,8 +1740,11 @@ int mlx4_en_start_port(struct net_device *dev) /* Process all completions if exist to prevent * the queues freezing if they are full */ - for (i = 0; i < priv->rx_ring_num; i++) + for (i = 0; i < priv->rx_ring_num; i++) { + local_bh_disable(); napi_schedule(&priv->rx_cq[i]->napi); + local_bh_enable(); + } netif_tx_start_all_queues(dev); netif_device_attach(dev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c index 27ff401cec201d..51c6a57ca873cf 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c @@ -991,6 +991,7 @@ static int mlx5e_set_rxfh(struct net_device *dev, const u32 *indir, { struct mlx5e_priv *priv = netdev_priv(dev); int inlen = MLX5_ST_SZ_BYTES(modify_tir_in); + bool hash_changed = false; void *in; if ((hfunc != ETH_RSS_HASH_NO_CHANGE) && @@ -1012,14 +1013,21 @@ static int mlx5e_set_rxfh(struct net_device *dev, const u32 *indir, mlx5e_redirect_rqt(priv, rqtn, MLX5E_INDIR_RQT_SIZE, 0); } - if (key) + if (hfunc != ETH_RSS_HASH_NO_CHANGE && + hfunc != priv->params.rss_hfunc) { + priv->params.rss_hfunc = hfunc; + hash_changed = true; + } + + if (key) { memcpy(priv->params.toeplitz_hash_key, key, sizeof(priv->params.toeplitz_hash_key)); + hash_changed = hash_changed || + priv->params.rss_hfunc == ETH_RSS_HASH_TOP; + } - if (hfunc != ETH_RSS_HASH_NO_CHANGE) - priv->params.rss_hfunc = hfunc; - - mlx5e_modify_tirs_hash(priv, in, inlen); + if (hash_changed) + mlx5e_modify_tirs_hash(priv, in, inlen); mutex_unlock(&priv->state_lock); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_fs_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_fs_ethtool.c index d17c242279003b..90e81ae9f3bc29 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_fs_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_fs_ethtool.c @@ -247,6 +247,7 @@ static int set_flow_attrs(u32 *match_c, u32 *match_v, } if (fs->flow_type & FLOW_MAC_EXT && !is_zero_ether_addr(fs->m_ext.h_dest)) { + mask_spec(fs->m_ext.h_dest, fs->h_ext.h_dest, ETH_ALEN); ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c, dmac_47_16), fs->m_ext.h_dest); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 246d98ebb58890..5dc3e2453ff598 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -3773,14 +3773,7 @@ static void mlx5e_nic_enable(struct mlx5e_priv *priv) mlx5_lag_add(mdev, netdev); - if (mlx5e_vxlan_allowed(mdev)) { - rtnl_lock(); - udp_tunnel_get_rx_info(netdev); - rtnl_unlock(); - } - mlx5e_enable_async_events(priv); - queue_work(priv->wq, &priv->set_rx_mode_work); if (MLX5_CAP_GEN(mdev, vport_group_manager)) { mlx5_query_nic_vport_mac_address(mdev, 0, rep.hw_id); @@ -3790,6 +3783,18 @@ static void mlx5e_nic_enable(struct mlx5e_priv *priv) rep.priv_data = priv; mlx5_eswitch_register_vport_rep(esw, 0, &rep); } + + if (netdev->reg_state != NETREG_REGISTERED) + return; + + /* Device already registered: sync netdev system state */ + if (mlx5e_vxlan_allowed(mdev)) { + rtnl_lock(); + udp_tunnel_get_rx_info(netdev); + rtnl_unlock(); + } + + queue_work(priv->wq, &priv->set_rx_mode_work); } static void mlx5e_nic_disable(struct mlx5e_priv *priv) @@ -3937,10 +3942,6 @@ void mlx5e_detach_netdev(struct mlx5_core_dev *mdev, struct net_device *netdev) const struct mlx5e_profile *profile = priv->profile; set_bit(MLX5E_STATE_DESTROYING, &priv->state); - if (profile->disable) - profile->disable(priv); - - flush_workqueue(priv->wq); rtnl_lock(); if (netif_running(netdev)) @@ -3948,6 +3949,10 @@ void mlx5e_detach_netdev(struct mlx5_core_dev *mdev, struct net_device *netdev) netif_device_detach(netdev); rtnl_unlock(); + if (profile->disable) + profile->disable(priv); + flush_workqueue(priv->wq); + mlx5e_destroy_q_counter(priv); profile->cleanup_rx(priv); mlx5e_close_drop_rq(priv); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index 33495d88aeb214..e7b2158bb48a05 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -193,6 +193,9 @@ static inline bool mlx5e_rx_cache_put(struct mlx5e_rq *rq, return false; } + if (unlikely(page_is_pfmemalloc(dma_info->page))) + return false; + cache->page_cache[cache->tail] = *dma_info; cache->tail = tail_next; return true; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx_am.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx_am.c index 1fffe48a93cc35..cbfac06b7ffd1d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx_am.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx_am.c @@ -109,7 +109,6 @@ static bool mlx5e_am_on_top(struct mlx5e_rx_am *am) switch (am->tune_state) { case MLX5E_AM_PARKING_ON_TOP: case MLX5E_AM_PARKING_TIRED: - WARN_ONCE(true, "mlx5e_am_on_top: PARKING\n"); return true; case MLX5E_AM_GOING_RIGHT: return (am->steps_left > 1) && (am->steps_right == 1); @@ -123,7 +122,6 @@ static void mlx5e_am_turn(struct mlx5e_rx_am *am) switch (am->tune_state) { case MLX5E_AM_PARKING_ON_TOP: case MLX5E_AM_PARKING_TIRED: - WARN_ONCE(true, "mlx5e_am_turn: PARKING\n"); break; case MLX5E_AM_GOING_RIGHT: am->tune_state = MLX5E_AM_GOING_LEFT; @@ -144,7 +142,6 @@ static int mlx5e_am_step(struct mlx5e_rx_am *am) switch (am->tune_state) { case MLX5E_AM_PARKING_ON_TOP: case MLX5E_AM_PARKING_TIRED: - WARN_ONCE(true, "mlx5e_am_step: PARKING\n"); break; case MLX5E_AM_GOING_RIGHT: if (am->profile_ix == (MLX5E_PARAMS_AM_NUM_PROFILES - 1)) @@ -282,10 +279,8 @@ static void mlx5e_am_calc_stats(struct mlx5e_rx_am_sample *start, u32 delta_us = ktime_us_delta(end->time, start->time); unsigned int npkts = end->pkt_ctr - start->pkt_ctr; - if (!delta_us) { - WARN_ONCE(true, "mlx5e_am_calc_stats: delta_us=0\n"); + if (!delta_us) return; - } curr_stats->ppms = (npkts * USEC_PER_MSEC) / delta_us; curr_stats->epms = (MLX5E_AM_NEVENTS * USEC_PER_MSEC) / delta_us; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c index be1f7333ab7f68..c7011ef4e351ff 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c @@ -1703,7 +1703,7 @@ int mlx5_eswitch_set_vport_mac(struct mlx5_eswitch *esw, if (!ESW_ALLOWED(esw)) return -EPERM; - if (!LEGAL_VPORT(esw, vport)) + if (!LEGAL_VPORT(esw, vport) || is_multicast_ether_addr(mac)) return -EINVAL; mutex_lock(&esw->state_lock); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index ada24e103b02ac..0c9ef8729ca7b2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -468,6 +468,13 @@ static int handle_hca_cap(struct mlx5_core_dev *dev) MLX5_SET(cmd_hca_cap, set_hca_cap, pkey_table_size, to_fw_pkey_sz(dev, 128)); + /* Check log_max_qp from HCA caps to set in current profile */ + if (MLX5_CAP_GEN_MAX(dev, log_max_qp) < profile[prof_sel].log_max_qp) { + mlx5_core_warn(dev, "log_max_qp value in current profile is %d, changing it to HCA capability limit (%d)\n", + profile[prof_sel].log_max_qp, + MLX5_CAP_GEN_MAX(dev, log_max_qp)); + profile[prof_sel].log_max_qp = MLX5_CAP_GEN_MAX(dev, log_max_qp); + } if (prof->mask & MLX5_PROF_MASK_QP_SIZE) MLX5_SET(cmd_hca_cap, set_hca_cap, log_max_qp, prof->log_max_qp); @@ -540,7 +547,6 @@ static int mlx5_irq_set_affinity_hint(struct mlx5_core_dev *mdev, int i) struct mlx5_priv *priv = &mdev->priv; struct msix_entry *msix = priv->msix_arr; int irq = msix[i + MLX5_EQ_VEC_COMP_BASE].vector; - int numa_node = priv->numa_node; int err; if (!zalloc_cpumask_var(&priv->irq_info[i].mask, GFP_KERNEL)) { @@ -548,7 +554,7 @@ static int mlx5_irq_set_affinity_hint(struct mlx5_core_dev *mdev, int i) return -ENOMEM; } - cpumask_set_cpu(cpumask_local_spread(i, numa_node), + cpumask_set_cpu(cpumask_local_spread(i, priv->numa_node), priv->irq_info[i].mask); err = irq_set_affinity_hint(irq, priv->irq_info[i].mask); @@ -1152,6 +1158,9 @@ static int mlx5_unload_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv, { int err = 0; + if (cleanup) + mlx5_drain_health_wq(dev); + mutex_lock(&dev->intf_state_mutex); if (test_bit(MLX5_INTERFACE_STATE_DOWN, &dev->intf_state)) { dev_warn(&dev->pdev->dev, "%s: interface is down, NOP\n", @@ -1312,7 +1321,7 @@ static pci_ers_result_t mlx5_pci_err_detected(struct pci_dev *pdev, mlx5_enter_error_state(dev); mlx5_unload_one(dev, priv, false); - /* In case of kernel call save the pci state and drain health wq */ + /* In case of kernel call save the pci state and drain the health wq */ if (state) { pci_save_state(pdev); mlx5_drain_health_wq(dev); diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci.h b/drivers/net/ethernet/mellanox/mlxsw/pci.h index d942a3e6fa4151..846fd4df7dabbc 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/pci.h +++ b/drivers/net/ethernet/mellanox/mlxsw/pci.h @@ -211,21 +211,21 @@ MLXSW_ITEM32(pci, eqe, owner, 0x0C, 0, 1); /* pci_eqe_cmd_token * Command completion event - token */ -MLXSW_ITEM32(pci, eqe, cmd_token, 0x08, 16, 16); +MLXSW_ITEM32(pci, eqe, cmd_token, 0x00, 16, 16); /* pci_eqe_cmd_status * Command completion event - status */ -MLXSW_ITEM32(pci, eqe, cmd_status, 0x08, 0, 8); +MLXSW_ITEM32(pci, eqe, cmd_status, 0x00, 0, 8); /* pci_eqe_cmd_out_param_h * Command completion event - output parameter - higher part */ -MLXSW_ITEM32(pci, eqe, cmd_out_param_h, 0x0C, 0, 32); +MLXSW_ITEM32(pci, eqe, cmd_out_param_h, 0x04, 0, 32); /* pci_eqe_cmd_out_param_l * Command completion event - output parameter - lower part */ -MLXSW_ITEM32(pci, eqe, cmd_out_param_l, 0x10, 0, 32); +MLXSW_ITEM32(pci, eqe, cmd_out_param_l, 0x08, 0, 32); #endif diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c index dda5761e91bcbe..f902c4d3de9981 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c @@ -684,6 +684,7 @@ static netdev_tx_t mlxsw_sp_port_xmit(struct sk_buff *skb, dev_kfree_skb_any(skb_orig); return NETDEV_TX_OK; } + dev_consume_skb_any(skb_orig); } if (eth_skb_pad(skb)) { diff --git a/drivers/net/ethernet/mellanox/mlxsw/switchx2.c b/drivers/net/ethernet/mellanox/mlxsw/switchx2.c index 92bda8703f8752..d548f0a5517454 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/switchx2.c +++ b/drivers/net/ethernet/mellanox/mlxsw/switchx2.c @@ -314,6 +314,7 @@ static netdev_tx_t mlxsw_sx_port_xmit(struct sk_buff *skb, dev_kfree_skb_any(skb_orig); return NETDEV_TX_OK; } + dev_consume_skb_any(skb_orig); } mlxsw_sx_txhdr_construct(skb, &tx_info); /* TX header is consumed by HW on the way so we shouldn't count its diff --git a/drivers/net/ethernet/renesas/ravb_main.c b/drivers/net/ethernet/renesas/ravb_main.c index d6a217874a8bc2..862f18ed602283 100644 --- a/drivers/net/ethernet/renesas/ravb_main.c +++ b/drivers/net/ethernet/renesas/ravb_main.c @@ -1508,6 +1508,19 @@ static netdev_tx_t ravb_start_xmit(struct sk_buff *skb, struct net_device *ndev) buffer = PTR_ALIGN(priv->tx_align[q], DPTR_ALIGN) + entry / NUM_TX_DESC * DPTR_ALIGN; len = PTR_ALIGN(skb->data, DPTR_ALIGN) - skb->data; + /* Zero length DMA descriptors are problematic as they seem to + * terminate DMA transfers. Avoid them by simply using a length of + * DPTR_ALIGN (4) when skb data is aligned to DPTR_ALIGN. + * + * As skb is guaranteed to have at least ETH_ZLEN (60) bytes of + * data by the call to skb_put_padto() above this is safe with + * respect to both the length of the first DMA descriptor (len) + * overflowing the available data and the length of the second DMA + * descriptor (skb->len - len) being negative. + */ + if (len == 0) + len = DPTR_ALIGN; + memcpy(buffer, skb->data, len); dma_addr = dma_map_single(ndev->dev.parent, buffer, len, DMA_TO_DEVICE); if (dma_mapping_error(ndev->dev.parent, dma_addr)) diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index caf069a465f234..b2893fbe25e528 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -3349,12 +3349,6 @@ int stmmac_dvr_probe(struct device *device, spin_lock_init(&priv->lock); spin_lock_init(&priv->tx_lock); - ret = register_netdev(ndev); - if (ret) { - pr_err("%s: ERROR %i registering the device\n", __func__, ret); - goto error_netdev_register; - } - /* If a specific clk_csr value is passed from the platform * this means that the CSR Clock Range selection cannot be * changed at run-time and it is fixed. Viceversa the driver'll try to @@ -3376,15 +3370,24 @@ int stmmac_dvr_probe(struct device *device, if (ret < 0) { pr_debug("%s: MDIO bus (id: %d) registration failed", __func__, priv->plat->bus_id); - goto error_mdio_register; + goto error_napi_register; } } - return 0; + ret = register_netdev(ndev); + if (ret) { + pr_err("%s: ERROR %i registering the device\n", __func__, ret); + goto error_netdev_register; + } + + return ret; -error_mdio_register: - unregister_netdev(ndev); error_netdev_register: + if (priv->hw->pcs != STMMAC_PCS_RGMII && + priv->hw->pcs != STMMAC_PCS_TBI && + priv->hw->pcs != STMMAC_PCS_RTBI) + stmmac_mdio_unregister(ndev); +error_napi_register: netif_napi_del(&priv->napi); error_hw_init: clk_disable_unprepare(priv->pclk); diff --git a/drivers/net/hyperv/netvsc.c b/drivers/net/hyperv/netvsc.c index 720b5fa9e62544..c2ac39a940f719 100644 --- a/drivers/net/hyperv/netvsc.c +++ b/drivers/net/hyperv/netvsc.c @@ -1288,6 +1288,9 @@ void netvsc_channel_cb(void *context) ndev = hv_get_drvdata(device); buffer = get_per_channel_state(channel); + /* commit_rd_index() -> hv_signal_on_read() needs this. */ + init_cached_read_index(channel); + do { desc = get_next_pkt_raw(channel); if (desc != NULL) { @@ -1340,6 +1343,9 @@ void netvsc_channel_cb(void *context) bufferlen = bytes_recvd; } + + init_cached_read_index(channel); + } while (1); if (bufferlen > NETVSC_PACKET_SIZE) diff --git a/drivers/net/hyperv/netvsc_drv.c b/drivers/net/hyperv/netvsc_drv.c index c9140c3aeb67d5..ff038e507fd6e9 100644 --- a/drivers/net/hyperv/netvsc_drv.c +++ b/drivers/net/hyperv/netvsc_drv.c @@ -659,6 +659,7 @@ int netvsc_recv_callback(struct hv_device *device_obj, * policy filters on the host). Deliver these via the VF * interface in the guest. */ + rcu_read_lock(); vf_netdev = rcu_dereference(net_device_ctx->vf_netdev); if (vf_netdev && (vf_netdev->flags & IFF_UP)) net = vf_netdev; @@ -667,6 +668,7 @@ int netvsc_recv_callback(struct hv_device *device_obj, skb = netvsc_alloc_recv_skb(net, packet, csum_info, *data, vlan_tci); if (unlikely(!skb)) { ++net->stats.rx_dropped; + rcu_read_unlock(); return NVSP_STAT_FAIL; } @@ -696,6 +698,7 @@ int netvsc_recv_callback(struct hv_device *device_obj, * TODO - use NAPI? */ netif_rx(skb); + rcu_read_unlock(); return 0; } diff --git a/drivers/net/ieee802154/atusb.c b/drivers/net/ieee802154/atusb.c index 1056ed142411d3..f186e0460cde31 100644 --- a/drivers/net/ieee802154/atusb.c +++ b/drivers/net/ieee802154/atusb.c @@ -112,13 +112,26 @@ static int atusb_read_reg(struct atusb *atusb, uint8_t reg) { struct usb_device *usb_dev = atusb->usb_dev; int ret; + uint8_t *buffer; uint8_t value; + buffer = kmalloc(1, GFP_KERNEL); + if (!buffer) + return -ENOMEM; + dev_dbg(&usb_dev->dev, "atusb: reg = 0x%x\n", reg); ret = atusb_control_msg(atusb, usb_rcvctrlpipe(usb_dev, 0), ATUSB_REG_READ, ATUSB_REQ_FROM_DEV, - 0, reg, &value, 1, 1000); - return ret >= 0 ? value : ret; + 0, reg, buffer, 1, 1000); + + if (ret >= 0) { + value = buffer[0]; + kfree(buffer); + return value; + } else { + kfree(buffer); + return ret; + } } static int atusb_write_subreg(struct atusb *atusb, uint8_t reg, uint8_t mask, @@ -587,9 +600,13 @@ static struct ieee802154_ops atusb_ops = { static int atusb_get_and_show_revision(struct atusb *atusb) { struct usb_device *usb_dev = atusb->usb_dev; - unsigned char buffer[3]; + unsigned char *buffer; int ret; + buffer = kmalloc(3, GFP_KERNEL); + if (!buffer) + return -ENOMEM; + /* Get a couple of the ATMega Firmware values */ ret = atusb_control_msg(atusb, usb_rcvctrlpipe(usb_dev, 0), ATUSB_ID, ATUSB_REQ_FROM_DEV, 0, 0, @@ -605,15 +622,20 @@ static int atusb_get_and_show_revision(struct atusb *atusb) dev_info(&usb_dev->dev, "Please update to version 0.2 or newer"); } + kfree(buffer); return ret; } static int atusb_get_and_show_build(struct atusb *atusb) { struct usb_device *usb_dev = atusb->usb_dev; - char build[ATUSB_BUILD_SIZE + 1]; + char *build; int ret; + build = kmalloc(ATUSB_BUILD_SIZE + 1, GFP_KERNEL); + if (!build) + return -ENOMEM; + ret = atusb_control_msg(atusb, usb_rcvctrlpipe(usb_dev, 0), ATUSB_BUILD, ATUSB_REQ_FROM_DEV, 0, 0, build, ATUSB_BUILD_SIZE, 1000); @@ -622,6 +644,7 @@ static int atusb_get_and_show_build(struct atusb *atusb) dev_info(&usb_dev->dev, "Firmware: build %s\n", build); } + kfree(build); return ret; } diff --git a/drivers/net/macvtap.c b/drivers/net/macvtap.c index 7869b0651576fa..6f38daf2d978bd 100644 --- a/drivers/net/macvtap.c +++ b/drivers/net/macvtap.c @@ -827,7 +827,7 @@ static ssize_t macvtap_put_user(struct macvtap_queue *q, return -EINVAL; ret = virtio_net_hdr_from_skb(skb, &vnet_hdr, - macvtap_is_little_endian(q)); + macvtap_is_little_endian(q), true); if (ret) BUG(); diff --git a/drivers/net/phy/bcm63xx.c b/drivers/net/phy/bcm63xx.c index e741bf614c4eec..b0492ef2cdaa0d 100644 --- a/drivers/net/phy/bcm63xx.c +++ b/drivers/net/phy/bcm63xx.c @@ -21,6 +21,23 @@ MODULE_DESCRIPTION("Broadcom 63xx internal PHY driver"); MODULE_AUTHOR("Maxime Bizon "); MODULE_LICENSE("GPL"); +static int bcm63xx_config_intr(struct phy_device *phydev) +{ + int reg, err; + + reg = phy_read(phydev, MII_BCM63XX_IR); + if (reg < 0) + return reg; + + if (phydev->interrupts == PHY_INTERRUPT_ENABLED) + reg &= ~MII_BCM63XX_IR_GMASK; + else + reg |= MII_BCM63XX_IR_GMASK; + + err = phy_write(phydev, MII_BCM63XX_IR, reg); + return err; +} + static int bcm63xx_config_init(struct phy_device *phydev) { int reg, err; @@ -55,7 +72,7 @@ static struct phy_driver bcm63xx_driver[] = { .config_aneg = genphy_config_aneg, .read_status = genphy_read_status, .ack_interrupt = bcm_phy_ack_intr, - .config_intr = bcm_phy_config_intr, + .config_intr = bcm63xx_config_intr, }, { /* same phy as above, with just a different OUI */ .phy_id = 0x002bdc00, @@ -67,7 +84,7 @@ static struct phy_driver bcm63xx_driver[] = { .config_aneg = genphy_config_aneg, .read_status = genphy_read_status, .ack_interrupt = bcm_phy_ack_intr, - .config_intr = bcm_phy_config_intr, + .config_intr = bcm63xx_config_intr, } }; module_phy_driver(bcm63xx_driver); diff --git a/drivers/net/ppp/Kconfig b/drivers/net/ppp/Kconfig index 1373c6d7278d84..282aec4860ebe5 100644 --- a/drivers/net/ppp/Kconfig +++ b/drivers/net/ppp/Kconfig @@ -149,6 +149,23 @@ config PPPOL2TP tunnels. L2TP is replacing PPTP for VPN uses. if TTY +config PPPOLAC + tristate "PPP on L2TP Access Concentrator" + depends on PPP && INET + help + L2TP (RFC 2661) is a tunneling protocol widely used in virtual private + networks. This driver handles L2TP data packets between a UDP socket + and a PPP channel, but only permits one session per socket. Thus it is + fairly simple and suited for clients. + +config PPPOPNS + tristate "PPP on PPTP Network Server" + depends on PPP && INET + help + PPTP (RFC 2637) is a tunneling protocol widely used in virtual private + networks. This driver handles PPTP data packets between a RAW socket + and a PPP channel. It is fairly simple and easy to use. + config PPP_ASYNC tristate "PPP support for async serial ports" depends on PPP diff --git a/drivers/net/ppp/Makefile b/drivers/net/ppp/Makefile index a6b6297b00669a..d283d03c4683a9 100644 --- a/drivers/net/ppp/Makefile +++ b/drivers/net/ppp/Makefile @@ -11,3 +11,5 @@ obj-$(CONFIG_PPP_SYNC_TTY) += ppp_synctty.o obj-$(CONFIG_PPPOE) += pppox.o pppoe.o obj-$(CONFIG_PPPOL2TP) += pppox.o obj-$(CONFIG_PPTP) += pppox.o pptp.o +obj-$(CONFIG_PPPOLAC) += pppox.o pppolac.o +obj-$(CONFIG_PPPOPNS) += pppox.o pppopns.o diff --git a/drivers/net/ppp/pppolac.c b/drivers/net/ppp/pppolac.c new file mode 100644 index 00000000000000..0184c96579e984 --- /dev/null +++ b/drivers/net/ppp/pppolac.c @@ -0,0 +1,448 @@ +/* drivers/net/pppolac.c + * + * Driver for PPP on L2TP Access Concentrator / PPPoLAC Socket (RFC 2661) + * + * Copyright (C) 2009 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +/* This driver handles L2TP data packets between a UDP socket and a PPP channel. + * The socket must keep connected, and only one session per socket is permitted. + * Sequencing of outgoing packets is controlled by LNS. Incoming packets with + * sequences are reordered within a sliding window of one second. Currently + * reordering only happens when a packet is received. It is done for simplicity + * since no additional locks or threads are required. This driver only works on + * IPv4 due to the lack of UDP encapsulation support in IPv6. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define L2TP_CONTROL_BIT 0x80 +#define L2TP_LENGTH_BIT 0x40 +#define L2TP_SEQUENCE_BIT 0x08 +#define L2TP_OFFSET_BIT 0x02 +#define L2TP_VERSION 0x02 +#define L2TP_VERSION_MASK 0x0F + +#define PPP_ADDR 0xFF +#define PPP_CTRL 0x03 + +union unaligned { + __u32 u32; +} __attribute__((packed)); + +static inline union unaligned *unaligned(void *ptr) +{ + return (union unaligned *)ptr; +} + +struct meta { + __u32 sequence; + __u32 timestamp; +}; + +static inline struct meta *skb_meta(struct sk_buff *skb) +{ + return (struct meta *)skb->cb; +} + +/******************************************************************************/ + +static int pppolac_recv_core(struct sock *sk_udp, struct sk_buff *skb) +{ + struct sock *sk = (struct sock *)sk_udp->sk_user_data; + struct pppolac_opt *opt = &pppox_sk(sk)->proto.lac; + struct meta *meta = skb_meta(skb); + __u32 now = jiffies; + __u8 bits; + __u8 *ptr; + + /* Drop the packet if L2TP header is missing. */ + if (skb->len < sizeof(struct udphdr) + 6) + goto drop; + + /* Put it back if it is a control packet. */ + if (skb->data[sizeof(struct udphdr)] & L2TP_CONTROL_BIT) + return opt->backlog_rcv(sk_udp, skb); + + /* Skip UDP header. */ + skb_pull(skb, sizeof(struct udphdr)); + + /* Check the version. */ + if ((skb->data[1] & L2TP_VERSION_MASK) != L2TP_VERSION) + goto drop; + bits = skb->data[0]; + ptr = &skb->data[2]; + + /* Check the length if it is present. */ + if (bits & L2TP_LENGTH_BIT) { + if ((ptr[0] << 8 | ptr[1]) != skb->len) + goto drop; + ptr += 2; + } + + /* Skip all fields including optional ones. */ + if (!skb_pull(skb, 6 + (bits & L2TP_SEQUENCE_BIT ? 4 : 0) + + (bits & L2TP_LENGTH_BIT ? 2 : 0) + + (bits & L2TP_OFFSET_BIT ? 2 : 0))) + goto drop; + + /* Skip the offset padding if it is present. */ + if (bits & L2TP_OFFSET_BIT && + !skb_pull(skb, skb->data[-2] << 8 | skb->data[-1])) + goto drop; + + /* Check the tunnel and the session. */ + if (unaligned(ptr)->u32 != opt->local) + goto drop; + + /* Check the sequence if it is present. */ + if (bits & L2TP_SEQUENCE_BIT) { + meta->sequence = ptr[4] << 8 | ptr[5]; + if ((__s16)(meta->sequence - opt->recv_sequence) < 0) + goto drop; + } + + /* Skip PPP address and control if they are present. */ + if (skb->len >= 2 && skb->data[0] == PPP_ADDR && + skb->data[1] == PPP_CTRL) + skb_pull(skb, 2); + + /* Fix PPP protocol if it is compressed. */ + if (skb->len >= 1 && skb->data[0] & 1) + skb_push(skb, 1)[0] = 0; + + /* Drop the packet if PPP protocol is missing. */ + if (skb->len < 2) + goto drop; + + /* Perform reordering if sequencing is enabled. */ + atomic_set(&opt->sequencing, bits & L2TP_SEQUENCE_BIT); + if (bits & L2TP_SEQUENCE_BIT) { + struct sk_buff *skb1; + + /* Insert the packet into receive queue in order. */ + skb_set_owner_r(skb, sk); + skb_queue_walk(&sk->sk_receive_queue, skb1) { + struct meta *meta1 = skb_meta(skb1); + __s16 order = meta->sequence - meta1->sequence; + if (order == 0) + goto drop; + if (order < 0) { + meta->timestamp = meta1->timestamp; + skb_insert(skb1, skb, &sk->sk_receive_queue); + skb = NULL; + break; + } + } + if (skb) { + meta->timestamp = now; + skb_queue_tail(&sk->sk_receive_queue, skb); + } + + /* Remove packets from receive queue as long as + * 1. the receive buffer is full, + * 2. they are queued longer than one second, or + * 3. there are no missing packets before them. */ + skb_queue_walk_safe(&sk->sk_receive_queue, skb, skb1) { + meta = skb_meta(skb); + if (atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf && + now - meta->timestamp < HZ && + meta->sequence != opt->recv_sequence) + break; + skb_unlink(skb, &sk->sk_receive_queue); + opt->recv_sequence = (__u16)(meta->sequence + 1); + skb_orphan(skb); + ppp_input(&pppox_sk(sk)->chan, skb); + } + return NET_RX_SUCCESS; + } + + /* Flush receive queue if sequencing is disabled. */ + skb_queue_purge(&sk->sk_receive_queue); + skb_orphan(skb); + ppp_input(&pppox_sk(sk)->chan, skb); + return NET_RX_SUCCESS; +drop: + kfree_skb(skb); + return NET_RX_DROP; +} + +static int pppolac_recv(struct sock *sk_udp, struct sk_buff *skb) +{ + sock_hold(sk_udp); + sk_receive_skb(sk_udp, skb, 0); + return 0; +} + +static struct sk_buff_head delivery_queue; + +static void pppolac_xmit_core(struct work_struct *delivery_work) +{ + mm_segment_t old_fs = get_fs(); + struct sk_buff *skb; + + set_fs(KERNEL_DS); + while ((skb = skb_dequeue(&delivery_queue))) { + struct sock *sk_udp = skb->sk; + struct kvec iov = {.iov_base = skb->data, .iov_len = skb->len}; + struct msghdr msg = { 0 }; + + iov_iter_kvec(&msg.msg_iter, WRITE | ITER_KVEC, &iov, 1, + skb->len); + sk_udp->sk_prot->sendmsg(sk_udp, &msg, skb->len); + kfree_skb(skb); + } + set_fs(old_fs); +} + +static DECLARE_WORK(delivery_work, pppolac_xmit_core); + +static int pppolac_xmit(struct ppp_channel *chan, struct sk_buff *skb) +{ + struct sock *sk_udp = (struct sock *)chan->private; + struct pppolac_opt *opt = &pppox_sk(sk_udp->sk_user_data)->proto.lac; + + /* Install PPP address and control. */ + skb_push(skb, 2); + skb->data[0] = PPP_ADDR; + skb->data[1] = PPP_CTRL; + + /* Install L2TP header. */ + if (atomic_read(&opt->sequencing)) { + skb_push(skb, 10); + skb->data[0] = L2TP_SEQUENCE_BIT; + skb->data[6] = opt->xmit_sequence >> 8; + skb->data[7] = opt->xmit_sequence; + skb->data[8] = 0; + skb->data[9] = 0; + opt->xmit_sequence++; + } else { + skb_push(skb, 6); + skb->data[0] = 0; + } + skb->data[1] = L2TP_VERSION; + unaligned(&skb->data[2])->u32 = opt->remote; + + /* Now send the packet via the delivery queue. */ + skb_set_owner_w(skb, sk_udp); + skb_queue_tail(&delivery_queue, skb); + schedule_work(&delivery_work); + return 1; +} + +/******************************************************************************/ + +static struct ppp_channel_ops pppolac_channel_ops = { + .start_xmit = pppolac_xmit, +}; + +static int pppolac_connect(struct socket *sock, struct sockaddr *useraddr, + int addrlen, int flags) +{ + struct sock *sk = sock->sk; + struct pppox_sock *po = pppox_sk(sk); + struct sockaddr_pppolac *addr = (struct sockaddr_pppolac *)useraddr; + struct socket *sock_udp = NULL; + struct sock *sk_udp; + int error; + + if (addrlen != sizeof(struct sockaddr_pppolac) || + !addr->local.tunnel || !addr->local.session || + !addr->remote.tunnel || !addr->remote.session) { + return -EINVAL; + } + + lock_sock(sk); + error = -EALREADY; + if (sk->sk_state != PPPOX_NONE) + goto out; + + sock_udp = sockfd_lookup(addr->udp_socket, &error); + if (!sock_udp) + goto out; + sk_udp = sock_udp->sk; + lock_sock(sk_udp); + + /* Remove this check when IPv6 supports UDP encapsulation. */ + error = -EAFNOSUPPORT; + if (sk_udp->sk_family != AF_INET) + goto out; + error = -EPROTONOSUPPORT; + if (sk_udp->sk_protocol != IPPROTO_UDP) + goto out; + error = -EDESTADDRREQ; + if (sk_udp->sk_state != TCP_ESTABLISHED) + goto out; + error = -EBUSY; + if (udp_sk(sk_udp)->encap_type || sk_udp->sk_user_data) + goto out; + if (!sk_udp->sk_bound_dev_if) { + struct dst_entry *dst = sk_dst_get(sk_udp); + error = -ENODEV; + if (!dst) + goto out; + sk_udp->sk_bound_dev_if = dst->dev->ifindex; + dst_release(dst); + } + + po->chan.hdrlen = 12; + po->chan.private = sk_udp; + po->chan.ops = &pppolac_channel_ops; + po->chan.mtu = PPP_MRU - 80; + po->proto.lac.local = unaligned(&addr->local)->u32; + po->proto.lac.remote = unaligned(&addr->remote)->u32; + atomic_set(&po->proto.lac.sequencing, 1); + po->proto.lac.backlog_rcv = sk_udp->sk_backlog_rcv; + + error = ppp_register_channel(&po->chan); + if (error) + goto out; + + sk->sk_state = PPPOX_CONNECTED; + udp_sk(sk_udp)->encap_type = UDP_ENCAP_L2TPINUDP; + udp_sk(sk_udp)->encap_rcv = pppolac_recv; + sk_udp->sk_backlog_rcv = pppolac_recv_core; + sk_udp->sk_user_data = sk; +out: + if (sock_udp) { + release_sock(sk_udp); + if (error) + sockfd_put(sock_udp); + } + release_sock(sk); + return error; +} + +static int pppolac_release(struct socket *sock) +{ + struct sock *sk = sock->sk; + + if (!sk) + return 0; + + lock_sock(sk); + if (sock_flag(sk, SOCK_DEAD)) { + release_sock(sk); + return -EBADF; + } + + if (sk->sk_state != PPPOX_NONE) { + struct sock *sk_udp = (struct sock *)pppox_sk(sk)->chan.private; + lock_sock(sk_udp); + skb_queue_purge(&sk->sk_receive_queue); + pppox_unbind_sock(sk); + udp_sk(sk_udp)->encap_type = 0; + udp_sk(sk_udp)->encap_rcv = NULL; + sk_udp->sk_backlog_rcv = pppox_sk(sk)->proto.lac.backlog_rcv; + sk_udp->sk_user_data = NULL; + release_sock(sk_udp); + sockfd_put(sk_udp->sk_socket); + } + + sock_orphan(sk); + sock->sk = NULL; + release_sock(sk); + sock_put(sk); + return 0; +} + +/******************************************************************************/ + +static struct proto pppolac_proto = { + .name = "PPPOLAC", + .owner = THIS_MODULE, + .obj_size = sizeof(struct pppox_sock), +}; + +static struct proto_ops pppolac_proto_ops = { + .family = PF_PPPOX, + .owner = THIS_MODULE, + .release = pppolac_release, + .bind = sock_no_bind, + .connect = pppolac_connect, + .socketpair = sock_no_socketpair, + .accept = sock_no_accept, + .getname = sock_no_getname, + .poll = sock_no_poll, + .ioctl = pppox_ioctl, + .listen = sock_no_listen, + .shutdown = sock_no_shutdown, + .setsockopt = sock_no_setsockopt, + .getsockopt = sock_no_getsockopt, + .sendmsg = sock_no_sendmsg, + .recvmsg = sock_no_recvmsg, + .mmap = sock_no_mmap, +}; + +static int pppolac_create(struct net *net, struct socket *sock, int kern) +{ + struct sock *sk; + + sk = sk_alloc(net, PF_PPPOX, GFP_KERNEL, &pppolac_proto, kern); + if (!sk) + return -ENOMEM; + + sock_init_data(sock, sk); + sock->state = SS_UNCONNECTED; + sock->ops = &pppolac_proto_ops; + sk->sk_protocol = PX_PROTO_OLAC; + sk->sk_state = PPPOX_NONE; + return 0; +} + +/******************************************************************************/ + +static struct pppox_proto pppolac_pppox_proto = { + .create = pppolac_create, + .owner = THIS_MODULE, +}; + +static int __init pppolac_init(void) +{ + int error; + + error = proto_register(&pppolac_proto, 0); + if (error) + return error; + + error = register_pppox_proto(PX_PROTO_OLAC, &pppolac_pppox_proto); + if (error) + proto_unregister(&pppolac_proto); + else + skb_queue_head_init(&delivery_queue); + return error; +} + +static void __exit pppolac_exit(void) +{ + unregister_pppox_proto(PX_PROTO_OLAC); + proto_unregister(&pppolac_proto); +} + +module_init(pppolac_init); +module_exit(pppolac_exit); + +MODULE_DESCRIPTION("PPP on L2TP Access Concentrator (PPPoLAC)"); +MODULE_AUTHOR("Chia-chi Yeh "); +MODULE_LICENSE("GPL"); diff --git a/drivers/net/ppp/pppopns.c b/drivers/net/ppp/pppopns.c new file mode 100644 index 00000000000000..d9e06039794e51 --- /dev/null +++ b/drivers/net/ppp/pppopns.c @@ -0,0 +1,427 @@ +/* drivers/net/pppopns.c + * + * Driver for PPP on PPTP Network Server / PPPoPNS Socket (RFC 2637) + * + * Copyright (C) 2009 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +/* This driver handles PPTP data packets between a RAW socket and a PPP channel. + * The socket is created in the kernel space and connected to the same address + * of the control socket. Outgoing packets are always sent with sequences but + * without acknowledgements. Incoming packets with sequences are reordered + * within a sliding window of one second. Currently reordering only happens when + * a packet is received. It is done for simplicity since no additional locks or + * threads are required. This driver should work on both IPv4 and IPv6. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define GRE_HEADER_SIZE 8 + +#define PPTP_GRE_BITS htons(0x2001) +#define PPTP_GRE_BITS_MASK htons(0xEF7F) +#define PPTP_GRE_SEQ_BIT htons(0x1000) +#define PPTP_GRE_ACK_BIT htons(0x0080) +#define PPTP_GRE_TYPE htons(0x880B) + +#define PPP_ADDR 0xFF +#define PPP_CTRL 0x03 + +struct header { + __u16 bits; + __u16 type; + __u16 length; + __u16 call; + __u32 sequence; +} __attribute__((packed)); + +struct meta { + __u32 sequence; + __u32 timestamp; +}; + +static inline struct meta *skb_meta(struct sk_buff *skb) +{ + return (struct meta *)skb->cb; +} + +/******************************************************************************/ + +static int pppopns_recv_core(struct sock *sk_raw, struct sk_buff *skb) +{ + struct sock *sk = (struct sock *)sk_raw->sk_user_data; + struct pppopns_opt *opt = &pppox_sk(sk)->proto.pns; + struct meta *meta = skb_meta(skb); + __u32 now = jiffies; + struct header *hdr; + + /* Skip transport header */ + skb_pull(skb, skb_transport_header(skb) - skb->data); + + /* Drop the packet if GRE header is missing. */ + if (skb->len < GRE_HEADER_SIZE) + goto drop; + hdr = (struct header *)skb->data; + + /* Check the header. */ + if (hdr->type != PPTP_GRE_TYPE || hdr->call != opt->local || + (hdr->bits & PPTP_GRE_BITS_MASK) != PPTP_GRE_BITS) + goto drop; + + /* Skip all fields including optional ones. */ + if (!skb_pull(skb, GRE_HEADER_SIZE + + (hdr->bits & PPTP_GRE_SEQ_BIT ? 4 : 0) + + (hdr->bits & PPTP_GRE_ACK_BIT ? 4 : 0))) + goto drop; + + /* Check the length. */ + if (skb->len != ntohs(hdr->length)) + goto drop; + + /* Check the sequence if it is present. */ + if (hdr->bits & PPTP_GRE_SEQ_BIT) { + meta->sequence = ntohl(hdr->sequence); + if ((__s32)(meta->sequence - opt->recv_sequence) < 0) + goto drop; + } + + /* Skip PPP address and control if they are present. */ + if (skb->len >= 2 && skb->data[0] == PPP_ADDR && + skb->data[1] == PPP_CTRL) + skb_pull(skb, 2); + + /* Fix PPP protocol if it is compressed. */ + if (skb->len >= 1 && skb->data[0] & 1) + skb_push(skb, 1)[0] = 0; + + /* Drop the packet if PPP protocol is missing. */ + if (skb->len < 2) + goto drop; + + /* Perform reordering if sequencing is enabled. */ + if (hdr->bits & PPTP_GRE_SEQ_BIT) { + struct sk_buff *skb1; + + /* Insert the packet into receive queue in order. */ + skb_set_owner_r(skb, sk); + skb_queue_walk(&sk->sk_receive_queue, skb1) { + struct meta *meta1 = skb_meta(skb1); + __s32 order = meta->sequence - meta1->sequence; + if (order == 0) + goto drop; + if (order < 0) { + meta->timestamp = meta1->timestamp; + skb_insert(skb1, skb, &sk->sk_receive_queue); + skb = NULL; + break; + } + } + if (skb) { + meta->timestamp = now; + skb_queue_tail(&sk->sk_receive_queue, skb); + } + + /* Remove packets from receive queue as long as + * 1. the receive buffer is full, + * 2. they are queued longer than one second, or + * 3. there are no missing packets before them. */ + skb_queue_walk_safe(&sk->sk_receive_queue, skb, skb1) { + meta = skb_meta(skb); + if (atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf && + now - meta->timestamp < HZ && + meta->sequence != opt->recv_sequence) + break; + skb_unlink(skb, &sk->sk_receive_queue); + opt->recv_sequence = meta->sequence + 1; + skb_orphan(skb); + ppp_input(&pppox_sk(sk)->chan, skb); + } + return NET_RX_SUCCESS; + } + + /* Flush receive queue if sequencing is disabled. */ + skb_queue_purge(&sk->sk_receive_queue); + skb_orphan(skb); + ppp_input(&pppox_sk(sk)->chan, skb); + return NET_RX_SUCCESS; +drop: + kfree_skb(skb); + return NET_RX_DROP; +} + +static void pppopns_recv(struct sock *sk_raw) +{ + struct sk_buff *skb; + while ((skb = skb_dequeue(&sk_raw->sk_receive_queue))) { + sock_hold(sk_raw); + sk_receive_skb(sk_raw, skb, 0); + } +} + +static struct sk_buff_head delivery_queue; + +static void pppopns_xmit_core(struct work_struct *delivery_work) +{ + mm_segment_t old_fs = get_fs(); + struct sk_buff *skb; + + set_fs(KERNEL_DS); + while ((skb = skb_dequeue(&delivery_queue))) { + struct sock *sk_raw = skb->sk; + struct kvec iov = {.iov_base = skb->data, .iov_len = skb->len}; + struct msghdr msg = { 0 }; + + iov_iter_kvec(&msg.msg_iter, WRITE | ITER_KVEC, &iov, 1, + skb->len); + sk_raw->sk_prot->sendmsg(sk_raw, &msg, skb->len); + kfree_skb(skb); + } + set_fs(old_fs); +} + +static DECLARE_WORK(delivery_work, pppopns_xmit_core); + +static int pppopns_xmit(struct ppp_channel *chan, struct sk_buff *skb) +{ + struct sock *sk_raw = (struct sock *)chan->private; + struct pppopns_opt *opt = &pppox_sk(sk_raw->sk_user_data)->proto.pns; + struct header *hdr; + __u16 length; + + /* Install PPP address and control. */ + skb_push(skb, 2); + skb->data[0] = PPP_ADDR; + skb->data[1] = PPP_CTRL; + length = skb->len; + + /* Install PPTP GRE header. */ + hdr = (struct header *)skb_push(skb, 12); + hdr->bits = PPTP_GRE_BITS | PPTP_GRE_SEQ_BIT; + hdr->type = PPTP_GRE_TYPE; + hdr->length = htons(length); + hdr->call = opt->remote; + hdr->sequence = htonl(opt->xmit_sequence); + opt->xmit_sequence++; + + /* Now send the packet via the delivery queue. */ + skb_set_owner_w(skb, sk_raw); + skb_queue_tail(&delivery_queue, skb); + schedule_work(&delivery_work); + return 1; +} + +/******************************************************************************/ + +static struct ppp_channel_ops pppopns_channel_ops = { + .start_xmit = pppopns_xmit, +}; + +static int pppopns_connect(struct socket *sock, struct sockaddr *useraddr, + int addrlen, int flags) +{ + struct sock *sk = sock->sk; + struct pppox_sock *po = pppox_sk(sk); + struct sockaddr_pppopns *addr = (struct sockaddr_pppopns *)useraddr; + struct sockaddr_storage ss; + struct socket *sock_tcp = NULL; + struct socket *sock_raw = NULL; + struct sock *sk_tcp; + struct sock *sk_raw; + int error; + + if (addrlen != sizeof(struct sockaddr_pppopns)) + return -EINVAL; + + lock_sock(sk); + error = -EALREADY; + if (sk->sk_state != PPPOX_NONE) + goto out; + + sock_tcp = sockfd_lookup(addr->tcp_socket, &error); + if (!sock_tcp) + goto out; + sk_tcp = sock_tcp->sk; + error = -EPROTONOSUPPORT; + if (sk_tcp->sk_protocol != IPPROTO_TCP) + goto out; + addrlen = sizeof(struct sockaddr_storage); + error = kernel_getpeername(sock_tcp, (struct sockaddr *)&ss, &addrlen); + if (error) + goto out; + if (!sk_tcp->sk_bound_dev_if) { + struct dst_entry *dst = sk_dst_get(sk_tcp); + error = -ENODEV; + if (!dst) + goto out; + sk_tcp->sk_bound_dev_if = dst->dev->ifindex; + dst_release(dst); + } + + error = sock_create(ss.ss_family, SOCK_RAW, IPPROTO_GRE, &sock_raw); + if (error) + goto out; + sk_raw = sock_raw->sk; + sk_raw->sk_bound_dev_if = sk_tcp->sk_bound_dev_if; + error = kernel_connect(sock_raw, (struct sockaddr *)&ss, addrlen, 0); + if (error) + goto out; + + po->chan.hdrlen = 14; + po->chan.private = sk_raw; + po->chan.ops = &pppopns_channel_ops; + po->chan.mtu = PPP_MRU - 80; + po->proto.pns.local = addr->local; + po->proto.pns.remote = addr->remote; + po->proto.pns.data_ready = sk_raw->sk_data_ready; + po->proto.pns.backlog_rcv = sk_raw->sk_backlog_rcv; + + error = ppp_register_channel(&po->chan); + if (error) + goto out; + + sk->sk_state = PPPOX_CONNECTED; + lock_sock(sk_raw); + sk_raw->sk_data_ready = pppopns_recv; + sk_raw->sk_backlog_rcv = pppopns_recv_core; + sk_raw->sk_user_data = sk; + release_sock(sk_raw); +out: + if (sock_tcp) + sockfd_put(sock_tcp); + if (error && sock_raw) + sock_release(sock_raw); + release_sock(sk); + return error; +} + +static int pppopns_release(struct socket *sock) +{ + struct sock *sk = sock->sk; + + if (!sk) + return 0; + + lock_sock(sk); + if (sock_flag(sk, SOCK_DEAD)) { + release_sock(sk); + return -EBADF; + } + + if (sk->sk_state != PPPOX_NONE) { + struct sock *sk_raw = (struct sock *)pppox_sk(sk)->chan.private; + lock_sock(sk_raw); + skb_queue_purge(&sk->sk_receive_queue); + pppox_unbind_sock(sk); + sk_raw->sk_data_ready = pppox_sk(sk)->proto.pns.data_ready; + sk_raw->sk_backlog_rcv = pppox_sk(sk)->proto.pns.backlog_rcv; + sk_raw->sk_user_data = NULL; + release_sock(sk_raw); + sock_release(sk_raw->sk_socket); + } + + sock_orphan(sk); + sock->sk = NULL; + release_sock(sk); + sock_put(sk); + return 0; +} + +/******************************************************************************/ + +static struct proto pppopns_proto = { + .name = "PPPOPNS", + .owner = THIS_MODULE, + .obj_size = sizeof(struct pppox_sock), +}; + +static struct proto_ops pppopns_proto_ops = { + .family = PF_PPPOX, + .owner = THIS_MODULE, + .release = pppopns_release, + .bind = sock_no_bind, + .connect = pppopns_connect, + .socketpair = sock_no_socketpair, + .accept = sock_no_accept, + .getname = sock_no_getname, + .poll = sock_no_poll, + .ioctl = pppox_ioctl, + .listen = sock_no_listen, + .shutdown = sock_no_shutdown, + .setsockopt = sock_no_setsockopt, + .getsockopt = sock_no_getsockopt, + .sendmsg = sock_no_sendmsg, + .recvmsg = sock_no_recvmsg, + .mmap = sock_no_mmap, +}; + +static int pppopns_create(struct net *net, struct socket *sock, int kern) +{ + struct sock *sk; + + sk = sk_alloc(net, PF_PPPOX, GFP_KERNEL, &pppopns_proto, kern); + if (!sk) + return -ENOMEM; + + sock_init_data(sock, sk); + sock->state = SS_UNCONNECTED; + sock->ops = &pppopns_proto_ops; + sk->sk_protocol = PX_PROTO_OPNS; + sk->sk_state = PPPOX_NONE; + return 0; +} + +/******************************************************************************/ + +static struct pppox_proto pppopns_pppox_proto = { + .create = pppopns_create, + .owner = THIS_MODULE, +}; + +static int __init pppopns_init(void) +{ + int error; + + error = proto_register(&pppopns_proto, 0); + if (error) + return error; + + error = register_pppox_proto(PX_PROTO_OPNS, &pppopns_pppox_proto); + if (error) + proto_unregister(&pppopns_proto); + else + skb_queue_head_init(&delivery_queue); + return error; +} + +static void __exit pppopns_exit(void) +{ + unregister_pppox_proto(PX_PROTO_OPNS); + proto_unregister(&pppopns_proto); +} + +module_init(pppopns_init); +module_exit(pppopns_exit); + +MODULE_DESCRIPTION("PPP on PPTP Network Server (PPPoPNS)"); +MODULE_AUTHOR("Chia-chi Yeh "); +MODULE_LICENSE("GPL"); diff --git a/drivers/net/tun.c b/drivers/net/tun.c index db6acecabeaa3f..e5d904110a060c 100644 --- a/drivers/net/tun.c +++ b/drivers/net/tun.c @@ -1374,7 +1374,7 @@ static ssize_t tun_put_user(struct tun_struct *tun, return -EINVAL; ret = virtio_net_hdr_from_skb(skb, &gso, - tun_is_little_endian(tun)); + tun_is_little_endian(tun), true); if (ret) { struct skb_shared_info *sinfo = skb_shinfo(skb); pr_err("unexpected GSO type: " @@ -1991,6 +1991,12 @@ static long __tun_chr_ioctl(struct file *file, unsigned int cmd, int le; int ret; +#ifdef CONFIG_ANDROID_PARANOID_NETWORK + if (cmd != TUNGETIFF && !capable(CAP_NET_ADMIN)) { + return -EPERM; + } +#endif + if (cmd == TUNSETIFF || cmd == TUNSETQUEUE || _IOC_TYPE(cmd) == 0x89) { if (copy_from_user(&ifr, argp, ifreq_len)) return -EFAULT; diff --git a/drivers/net/usb/cdc_ether.c b/drivers/net/usb/cdc_ether.c index dd623f67448759..b82be816256c41 100644 --- a/drivers/net/usb/cdc_ether.c +++ b/drivers/net/usb/cdc_ether.c @@ -531,6 +531,7 @@ static const struct driver_info wwan_info = { #define SAMSUNG_VENDOR_ID 0x04e8 #define LENOVO_VENDOR_ID 0x17ef #define NVIDIA_VENDOR_ID 0x0955 +#define HP_VENDOR_ID 0x03f0 static const struct usb_device_id products[] = { /* BLACKLIST !! @@ -677,6 +678,13 @@ static const struct usb_device_id products[] = { .driver_info = 0, }, +/* HP lt2523 (Novatel E371) - handled by qmi_wwan */ +{ + USB_DEVICE_AND_INTERFACE_INFO(HP_VENDOR_ID, 0x421d, USB_CLASS_COMM, + USB_CDC_SUBCLASS_ETHERNET, USB_CDC_PROTO_NONE), + .driver_info = 0, +}, + /* AnyDATA ADU960S - handled by qmi_wwan */ { USB_DEVICE_AND_INTERFACE_INFO(0x16d5, 0x650a, USB_CLASS_COMM, diff --git a/drivers/net/usb/qmi_wwan.c b/drivers/net/usb/qmi_wwan.c index 6fe1cdb0174f6c..24d5272cdce510 100644 --- a/drivers/net/usb/qmi_wwan.c +++ b/drivers/net/usb/qmi_wwan.c @@ -654,6 +654,13 @@ static const struct usb_device_id products[] = { USB_CDC_PROTO_NONE), .driver_info = (unsigned long)&qmi_wwan_info, }, + { /* HP lt2523 (Novatel E371) */ + USB_DEVICE_AND_INTERFACE_INFO(0x03f0, 0x421d, + USB_CLASS_COMM, + USB_CDC_SUBCLASS_ETHERNET, + USB_CDC_PROTO_NONE), + .driver_info = (unsigned long)&qmi_wwan_info, + }, { /* HP lt4112 LTE/HSPA+ Gobi 4G Module (Huawei me906e) */ USB_DEVICE_AND_INTERFACE_INFO(0x03f0, 0x581d, USB_CLASS_VENDOR_SPEC, 1, 7), .driver_info = (unsigned long)&qmi_wwan_info, diff --git a/drivers/net/usb/r8152.c b/drivers/net/usb/r8152.c index efb84f0924922a..90b426c5ffcec4 100644 --- a/drivers/net/usb/r8152.c +++ b/drivers/net/usb/r8152.c @@ -32,7 +32,7 @@ #define NETNEXT_VERSION "08" /* Information for net */ -#define NET_VERSION "6" +#define NET_VERSION "7" #define DRIVER_VERSION "v1." NETNEXT_VERSION "." NET_VERSION #define DRIVER_AUTHOR "Realtek linux nic maintainers " @@ -1730,7 +1730,7 @@ static u8 r8152_rx_csum(struct r8152 *tp, struct rx_desc *rx_desc) u8 checksum = CHECKSUM_NONE; u32 opts2, opts3; - if (tp->version == RTL_VER_01 || tp->version == RTL_VER_02) + if (!(tp->netdev->features & NETIF_F_RXCSUM)) goto return_result; opts2 = le32_to_cpu(rx_desc->opts2); @@ -3572,43 +3572,93 @@ static bool delay_autosuspend(struct r8152 *tp) */ if (!sw_linking && tp->rtl_ops.in_nway(tp)) return true; + else if (!skb_queue_empty(&tp->tx_queue)) + return true; else return false; } -static int rtl8152_suspend(struct usb_interface *intf, pm_message_t message) +static int rtl8152_rumtime_suspend(struct r8152 *tp) { - struct r8152 *tp = usb_get_intfdata(intf); struct net_device *netdev = tp->netdev; int ret = 0; - mutex_lock(&tp->control); + if (netif_running(netdev) && test_bit(WORK_ENABLE, &tp->flags)) { + u32 rcr = 0; - if (PMSG_IS_AUTO(message)) { - if (netif_running(netdev) && delay_autosuspend(tp)) { + if (delay_autosuspend(tp)) { ret = -EBUSY; goto out1; } - set_bit(SELECTIVE_SUSPEND, &tp->flags); - } else { - netif_device_detach(netdev); + if (netif_carrier_ok(netdev)) { + u32 ocp_data; + + rcr = ocp_read_dword(tp, MCU_TYPE_PLA, PLA_RCR); + ocp_data = rcr & ~RCR_ACPT_ALL; + ocp_write_dword(tp, MCU_TYPE_PLA, PLA_RCR, ocp_data); + rxdy_gated_en(tp, true); + ocp_data = ocp_read_byte(tp, MCU_TYPE_PLA, + PLA_OOB_CTRL); + if (!(ocp_data & RXFIFO_EMPTY)) { + rxdy_gated_en(tp, false); + ocp_write_dword(tp, MCU_TYPE_PLA, PLA_RCR, rcr); + ret = -EBUSY; + goto out1; + } + } + + clear_bit(WORK_ENABLE, &tp->flags); + usb_kill_urb(tp->intr_urb); + + tp->rtl_ops.autosuspend_en(tp, true); + + if (netif_carrier_ok(netdev)) { + napi_disable(&tp->napi); + rtl_stop_rx(tp); + rxdy_gated_en(tp, false); + ocp_write_dword(tp, MCU_TYPE_PLA, PLA_RCR, rcr); + napi_enable(&tp->napi); + } } + set_bit(SELECTIVE_SUSPEND, &tp->flags); + +out1: + return ret; +} + +static int rtl8152_system_suspend(struct r8152 *tp) +{ + struct net_device *netdev = tp->netdev; + int ret = 0; + + netif_device_detach(netdev); + if (netif_running(netdev) && test_bit(WORK_ENABLE, &tp->flags)) { clear_bit(WORK_ENABLE, &tp->flags); usb_kill_urb(tp->intr_urb); napi_disable(&tp->napi); - if (test_bit(SELECTIVE_SUSPEND, &tp->flags)) { - rtl_stop_rx(tp); - tp->rtl_ops.autosuspend_en(tp, true); - } else { - cancel_delayed_work_sync(&tp->schedule); - tp->rtl_ops.down(tp); - } + cancel_delayed_work_sync(&tp->schedule); + tp->rtl_ops.down(tp); napi_enable(&tp->napi); } -out1: + + return ret; +} + +static int rtl8152_suspend(struct usb_interface *intf, pm_message_t message) +{ + struct r8152 *tp = usb_get_intfdata(intf); + int ret; + + mutex_lock(&tp->control); + + if (PMSG_IS_AUTO(message)) + ret = rtl8152_rumtime_suspend(tp); + else + ret = rtl8152_system_suspend(tp); + mutex_unlock(&tp->control); return ret; @@ -4310,6 +4360,11 @@ static int rtl8152_probe(struct usb_interface *intf, NETIF_F_HIGHDMA | NETIF_F_FRAGLIST | NETIF_F_IPV6_CSUM | NETIF_F_TSO6; + if (tp->version == RTL_VER_01) { + netdev->features &= ~NETIF_F_RXCSUM; + netdev->hw_features &= ~NETIF_F_RXCSUM; + } + netdev->ethtool_ops = &ops; netif_set_gso_max_size(netdev, RTL_LIMITED_TSO_SIZE); diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index cbf1c613c67aef..51fc0c33a62f53 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -840,7 +840,7 @@ static int xmit_skb(struct send_queue *sq, struct sk_buff *skb) hdr = skb_vnet_hdr(skb); if (virtio_net_hdr_from_skb(skb, &hdr->hdr, - virtio_is_little_endian(vi->vdev))) + virtio_is_little_endian(vi->vdev), false)) BUG(); if (vi->mergeable_rx_bufs) diff --git a/drivers/net/vrf.c b/drivers/net/vrf.c index 820de6a9ddde1d..95cf1d84478158 100644 --- a/drivers/net/vrf.c +++ b/drivers/net/vrf.c @@ -263,7 +263,9 @@ static netdev_tx_t vrf_process_v4_outbound(struct sk_buff *skb, .flowi4_iif = LOOPBACK_IFINDEX, .flowi4_tos = RT_TOS(ip4h->tos), .flowi4_flags = FLOWI_FLAG_ANYSRC | FLOWI_FLAG_SKIP_NH_OIF, + .flowi4_proto = ip4h->protocol, .daddr = ip4h->daddr, + .saddr = ip4h->saddr, }; struct net *net = dev_net(vrf_dev); struct rtable *rt; @@ -371,6 +373,8 @@ static int vrf_finish_output6(struct net *net, struct sock *sk, struct in6_addr *nexthop; int ret; + nf_reset(skb); + skb->protocol = htons(ETH_P_IPV6); skb->dev = dev; @@ -552,6 +556,8 @@ static int vrf_finish_output(struct net *net, struct sock *sk, struct sk_buff *s u32 nexthop; int ret = -EINVAL; + nf_reset(skb); + /* Be paranoid, rather than too clever. */ if (unlikely(skb_headroom(skb) < hh_len && dev->header_ops)) { struct sk_buff *skb2; @@ -850,8 +856,6 @@ static struct sk_buff *vrf_rcv_nfhook(u8 pf, unsigned int hook, { struct net *net = dev_net(dev); - nf_reset(skb); - if (NF_HOOK(pf, hook, net, NULL, skb, dev, NULL, vrf_rcv_finish) < 0) skb = NULL; /* kfree_skb(skb) handled by nf code */ @@ -966,6 +970,7 @@ static struct sk_buff *vrf_ip6_rcv(struct net_device *vrf_dev, */ need_strict = rt6_need_strict(&ipv6_hdr(skb)->daddr); if (!ipv6_ndisc_frame(skb) && !need_strict) { + vrf_rx_stats(vrf_dev, skb->len); skb->dev = vrf_dev; skb->skb_iif = vrf_dev->ifindex; @@ -1007,6 +1012,8 @@ static struct sk_buff *vrf_ip_rcv(struct net_device *vrf_dev, goto out; } + vrf_rx_stats(vrf_dev, skb->len); + skb_push(skb, skb->mac_len); dev_queue_xmit_nit(skb, vrf_dev); skb_pull(skb, skb->mac_len); @@ -1232,6 +1239,8 @@ static int vrf_newlink(struct net *src_net, struct net_device *dev, return -EINVAL; vrf->tb_id = nla_get_u32(data[IFLA_VRF_TABLE]); + if (vrf->tb_id == RT_TABLE_UNSPEC) + return -EINVAL; dev->priv_flags |= IFF_L3MDEV_MASTER; diff --git a/drivers/net/vxlan.c b/drivers/net/vxlan.c index 2ba01ca02c9c5e..0fafaa9d903b1f 100644 --- a/drivers/net/vxlan.c +++ b/drivers/net/vxlan.c @@ -2887,7 +2887,7 @@ static int vxlan_dev_configure(struct net *src_net, struct net_device *dev, memcpy(&vxlan->cfg, conf, sizeof(*conf)); if (!vxlan->cfg.dst_port) { if (conf->flags & VXLAN_F_GPE) - vxlan->cfg.dst_port = 4790; /* IANA assigned VXLAN-GPE port */ + vxlan->cfg.dst_port = htons(4790); /* IANA VXLAN-GPE port */ else vxlan->cfg.dst_port = default_port; } diff --git a/drivers/net/wireless/ath/ath10k/core.c b/drivers/net/wireless/ath/ath10k/core.c index 21ae8d663e67c6..0c4532227f2530 100644 --- a/drivers/net/wireless/ath/ath10k/core.c +++ b/drivers/net/wireless/ath/ath10k/core.c @@ -1534,7 +1534,7 @@ static void ath10k_core_restart(struct work_struct *work) switch (ar->state) { case ATH10K_STATE_ON: ar->state = ATH10K_STATE_RESTARTING; - ath10k_hif_stop(ar); + ath10k_halt(ar); ath10k_scan_finish(ar); ieee80211_restart_hw(ar->hw); break; diff --git a/drivers/net/wireless/ath/ath10k/core.h b/drivers/net/wireless/ath/ath10k/core.h index 521f1c55c19ee1..be5b527472f95d 100644 --- a/drivers/net/wireless/ath/ath10k/core.h +++ b/drivers/net/wireless/ath/ath10k/core.h @@ -557,10 +557,8 @@ enum ath10k_fw_features { */ ATH10K_FW_FEATURE_BTCOEX_PARAM = 14, - /* Older firmware with HTT delivers incorrect tx status for null func - * frames to driver, but this fixed in 10.2 and 10.4 firmware versions. - * Also this workaround results in reporting of incorrect null func - * status for 10.4. This flag is used to skip the workaround. + /* Unused flag and proven to be not working, enable this if you want + * to experiment sending NULL func data frames in HTT TX */ ATH10K_FW_FEATURE_SKIP_NULL_FUNC_WAR = 15, diff --git a/drivers/net/wireless/ath/ath10k/mac.c b/drivers/net/wireless/ath/ath10k/mac.c index 76297d69f1ed43..f2e85eb22afe6b 100644 --- a/drivers/net/wireless/ath/ath10k/mac.c +++ b/drivers/net/wireless/ath/ath10k/mac.c @@ -3255,8 +3255,6 @@ ath10k_mac_tx_h_get_txmode(struct ath10k *ar, if (ar->htt.target_version_major < 3 && (ieee80211_is_nullfunc(fc) || ieee80211_is_qos_nullfunc(fc)) && !test_bit(ATH10K_FW_FEATURE_HAS_WMI_MGMT_TX, - ar->running_fw->fw_file.fw_features) && - !test_bit(ATH10K_FW_FEATURE_SKIP_NULL_FUNC_WAR, ar->running_fw->fw_file.fw_features)) return ATH10K_HW_TXRX_MGMT; @@ -4449,7 +4447,6 @@ static int ath10k_start(struct ieee80211_hw *hw) ar->state = ATH10K_STATE_ON; break; case ATH10K_STATE_RESTARTING: - ath10k_halt(ar); ar->state = ATH10K_STATE_RESTARTED; break; case ATH10K_STATE_ON: diff --git a/drivers/net/wireless/ath/ath10k/spectral.c b/drivers/net/wireless/ath/ath10k/spectral.c index 7d9b0da1b010a6..2ffc1fe4923b96 100644 --- a/drivers/net/wireless/ath/ath10k/spectral.c +++ b/drivers/net/wireless/ath/ath10k/spectral.c @@ -338,7 +338,7 @@ static ssize_t write_file_spec_scan_ctl(struct file *file, } else { res = -EINVAL; } - } else if (strncmp("background", buf, 9) == 0) { + } else if (strncmp("background", buf, 10) == 0) { res = ath10k_spectral_scan_config(ar, SPECTRAL_BACKGROUND); } else if (strncmp("manual", buf, 6) == 0) { res = ath10k_spectral_scan_config(ar, SPECTRAL_MANUAL); diff --git a/drivers/net/wireless/ath/ath9k/hw.c b/drivers/net/wireless/ath/ath9k/hw.c index 14b13f07cd1fa8..a35f78be8dec68 100644 --- a/drivers/net/wireless/ath/ath9k/hw.c +++ b/drivers/net/wireless/ath/ath9k/hw.c @@ -2792,7 +2792,7 @@ u32 ath9k_hw_gpio_get(struct ath_hw *ah, u32 gpio) WARN_ON(1); } - return val; + return !!val; } EXPORT_SYMBOL(ath9k_hw_gpio_get); diff --git a/drivers/net/wireless/ath/ath9k/pci.c b/drivers/net/wireless/ath/ath9k/pci.c index 0dd454acf22ad3..aff473dfa10df6 100644 --- a/drivers/net/wireless/ath/ath9k/pci.c +++ b/drivers/net/wireless/ath/ath9k/pci.c @@ -26,7 +26,6 @@ static const struct pci_device_id ath_pci_id_table[] = { { PCI_VDEVICE(ATHEROS, 0x0023) }, /* PCI */ { PCI_VDEVICE(ATHEROS, 0x0024) }, /* PCI-E */ { PCI_VDEVICE(ATHEROS, 0x0027) }, /* PCI */ - { PCI_VDEVICE(ATHEROS, 0x0029) }, /* PCI */ #ifdef CONFIG_ATH9K_PCOEM /* Mini PCI AR9220 MB92 cards: Compex WLM200NX, Wistron DNMA-92 */ @@ -37,7 +36,7 @@ static const struct pci_device_id ath_pci_id_table[] = { .driver_data = ATH9K_PCI_LED_ACT_HI }, #endif - { PCI_VDEVICE(ATHEROS, 0x002A) }, /* PCI-E */ + { PCI_VDEVICE(ATHEROS, 0x0029) }, /* PCI */ #ifdef CONFIG_ATH9K_PCOEM { PCI_DEVICE_SUB(PCI_VENDOR_ID_ATHEROS, @@ -85,7 +84,11 @@ static const struct pci_device_id ath_pci_id_table[] = { 0x10CF, /* Fujitsu */ 0x1536), .driver_data = ATH9K_PCI_D3_L1_WAR }, +#endif + { PCI_VDEVICE(ATHEROS, 0x002A) }, /* PCI-E */ + +#ifdef CONFIG_ATH9K_PCOEM /* AR9285 card for Asus */ { PCI_DEVICE_SUB(PCI_VENDOR_ID_ATHEROS, 0x002B, diff --git a/drivers/net/wireless/ath/ath9k/xmit.c b/drivers/net/wireless/ath/ath9k/xmit.c index 52bfbb9886117f..e47286bf378eb3 100644 --- a/drivers/net/wireless/ath/ath9k/xmit.c +++ b/drivers/net/wireless/ath/ath9k/xmit.c @@ -2787,7 +2787,7 @@ void ath_tx_edma_tasklet(struct ath_softc *sc) fifo_list = &txq->txq_fifo[txq->txq_tailidx]; if (list_empty(fifo_list)) { ath_txq_unlock(sc, txq); - return; + break; } bf = list_first_entry(fifo_list, struct ath_buf, list); diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-8000.c b/drivers/net/wireless/intel/iwlwifi/iwl-8000.c index d02ca1491d16ce..8d3e53fac1dabc 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-8000.c +++ b/drivers/net/wireless/intel/iwlwifi/iwl-8000.c @@ -91,7 +91,7 @@ #define IWL8000_FW_PRE "iwlwifi-8000C-" #define IWL8000_MODULE_FIRMWARE(api) \ - IWL8000_FW_PRE "-" __stringify(api) ".ucode" + IWL8000_FW_PRE __stringify(api) ".ucode" #define IWL8265_FW_PRE "iwlwifi-8265-" #define IWL8265_MODULE_FIRMWARE(api) \ diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/sta.c b/drivers/net/wireless/intel/iwlwifi/mvm/sta.c index fc771885e3831b..52de3c6d760ccf 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/sta.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/sta.c @@ -1144,9 +1144,10 @@ static void iwl_mvm_realloc_queues_after_restart(struct iwl_mvm *mvm, .frame_limit = IWL_FRAME_LIMIT, }; - /* Make sure reserved queue is still marked as such (or allocated) */ - mvm->queue_info[mvm_sta->reserved_queue].status = - IWL_MVM_QUEUE_RESERVED; + /* Make sure reserved queue is still marked as such (if allocated) */ + if (mvm_sta->reserved_queue != IEEE80211_INVAL_HW_QUEUE) + mvm->queue_info[mvm_sta->reserved_queue].status = + IWL_MVM_QUEUE_RESERVED; for (i = 0; i <= IWL_MAX_TID_COUNT; i++) { struct iwl_mvm_tid_data *tid_data = &mvm_sta->tid_data[i]; diff --git a/drivers/net/wireless/intersil/orinoco/mic.c b/drivers/net/wireless/intersil/orinoco/mic.c index bc7397d709d3ac..08bc7822f8209d 100644 --- a/drivers/net/wireless/intersil/orinoco/mic.c +++ b/drivers/net/wireless/intersil/orinoco/mic.c @@ -16,7 +16,7 @@ /********************************************************************/ int orinoco_mic_init(struct orinoco_private *priv) { - priv->tx_tfm_mic = crypto_alloc_ahash("michael_mic", 0, + priv->tx_tfm_mic = crypto_alloc_shash("michael_mic", 0, CRYPTO_ALG_ASYNC); if (IS_ERR(priv->tx_tfm_mic)) { printk(KERN_DEBUG "orinoco_mic_init: could not allocate " @@ -25,7 +25,7 @@ int orinoco_mic_init(struct orinoco_private *priv) return -ENOMEM; } - priv->rx_tfm_mic = crypto_alloc_ahash("michael_mic", 0, + priv->rx_tfm_mic = crypto_alloc_shash("michael_mic", 0, CRYPTO_ALG_ASYNC); if (IS_ERR(priv->rx_tfm_mic)) { printk(KERN_DEBUG "orinoco_mic_init: could not allocate " @@ -40,17 +40,16 @@ int orinoco_mic_init(struct orinoco_private *priv) void orinoco_mic_free(struct orinoco_private *priv) { if (priv->tx_tfm_mic) - crypto_free_ahash(priv->tx_tfm_mic); + crypto_free_shash(priv->tx_tfm_mic); if (priv->rx_tfm_mic) - crypto_free_ahash(priv->rx_tfm_mic); + crypto_free_shash(priv->rx_tfm_mic); } -int orinoco_mic(struct crypto_ahash *tfm_michael, u8 *key, +int orinoco_mic(struct crypto_shash *tfm_michael, u8 *key, u8 *da, u8 *sa, u8 priority, u8 *data, size_t data_len, u8 *mic) { - AHASH_REQUEST_ON_STACK(req, tfm_michael); - struct scatterlist sg[2]; + SHASH_DESC_ON_STACK(desc, tfm_michael); u8 hdr[ETH_HLEN + 2]; /* size of header + padding */ int err; @@ -67,18 +66,27 @@ int orinoco_mic(struct crypto_ahash *tfm_michael, u8 *key, hdr[ETH_ALEN * 2 + 2] = 0; hdr[ETH_ALEN * 2 + 3] = 0; - /* Use scatter gather to MIC header and data in one go */ - sg_init_table(sg, 2); - sg_set_buf(&sg[0], hdr, sizeof(hdr)); - sg_set_buf(&sg[1], data, data_len); + desc->tfm = tfm_michael; + desc->flags = 0; - if (crypto_ahash_setkey(tfm_michael, key, MIC_KEYLEN)) - return -1; + err = crypto_shash_setkey(tfm_michael, key, MIC_KEYLEN); + if (err) + return err; + + err = crypto_shash_init(desc); + if (err) + return err; + + err = crypto_shash_update(desc, hdr, sizeof(hdr)); + if (err) + return err; + + err = crypto_shash_update(desc, data, data_len); + if (err) + return err; + + err = crypto_shash_final(desc, mic); + shash_desc_zero(desc); - ahash_request_set_tfm(req, tfm_michael); - ahash_request_set_callback(req, 0, NULL, NULL); - ahash_request_set_crypt(req, sg, mic, data_len + sizeof(hdr)); - err = crypto_ahash_digest(req); - ahash_request_zero(req); return err; } diff --git a/drivers/net/wireless/intersil/orinoco/mic.h b/drivers/net/wireless/intersil/orinoco/mic.h index ce731d05cc98cd..e8724e8892194b 100644 --- a/drivers/net/wireless/intersil/orinoco/mic.h +++ b/drivers/net/wireless/intersil/orinoco/mic.h @@ -6,6 +6,7 @@ #define _ORINOCO_MIC_H_ #include +#include #define MICHAEL_MIC_LEN 8 @@ -15,7 +16,7 @@ struct crypto_ahash; int orinoco_mic_init(struct orinoco_private *priv); void orinoco_mic_free(struct orinoco_private *priv); -int orinoco_mic(struct crypto_ahash *tfm_michael, u8 *key, +int orinoco_mic(struct crypto_shash *tfm_michael, u8 *key, u8 *da, u8 *sa, u8 priority, u8 *data, size_t data_len, u8 *mic); diff --git a/drivers/net/wireless/intersil/orinoco/orinoco.h b/drivers/net/wireless/intersil/orinoco/orinoco.h index 2f0c84b1c440cd..5fa1c3e3713f83 100644 --- a/drivers/net/wireless/intersil/orinoco/orinoco.h +++ b/drivers/net/wireless/intersil/orinoco/orinoco.h @@ -152,8 +152,8 @@ struct orinoco_private { u8 *wpa_ie; int wpa_ie_len; - struct crypto_ahash *rx_tfm_mic; - struct crypto_ahash *tx_tfm_mic; + struct crypto_shash *rx_tfm_mic; + struct crypto_shash *tx_tfm_mic; unsigned int wpa_enabled:1; unsigned int tkip_cm_active:1; diff --git a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c index a5e6ec2152bff8..82d949ede2940f 100644 --- a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c +++ b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c @@ -4372,6 +4372,13 @@ void rtl8xxxu_gen1_report_connect(struct rtl8xxxu_priv *priv, void rtl8xxxu_gen2_report_connect(struct rtl8xxxu_priv *priv, u8 macid, bool connect) { +#ifdef RTL8XXXU_GEN2_REPORT_CONNECT + /* + * Barry Day reports this causes issues with 8192eu and 8723bu + * devices reconnecting. The reason for this is unclear, but + * until it is better understood, leave the code in place but + * disabled, so it is not lost. + */ struct h2c_cmd h2c; memset(&h2c, 0, sizeof(struct h2c_cmd)); @@ -4383,6 +4390,7 @@ void rtl8xxxu_gen2_report_connect(struct rtl8xxxu_priv *priv, h2c.media_status_rpt.parm &= ~BIT(0); rtl8xxxu_gen2_h2c_cmd(priv, &h2c, sizeof(h2c.media_status_rpt)); +#endif } void rtl8xxxu_gen1_init_aggregation(struct rtl8xxxu_priv *priv) diff --git a/drivers/net/wireless/realtek/rtlwifi/base.c b/drivers/net/wireless/realtek/rtlwifi/base.c index 264466f59c5702..4ac928bf1f8e6c 100644 --- a/drivers/net/wireless/realtek/rtlwifi/base.c +++ b/drivers/net/wireless/realtek/rtlwifi/base.c @@ -1303,12 +1303,13 @@ EXPORT_SYMBOL_GPL(rtl_action_proc); static void setup_arp_tx(struct rtl_priv *rtlpriv, struct rtl_ps_ctl *ppsc) { + struct ieee80211_hw *hw = rtlpriv->hw; + rtlpriv->ra.is_special_data = true; if (rtlpriv->cfg->ops->get_btc_status()) rtlpriv->btcoexist.btc_ops->btc_special_packet_notify( rtlpriv, 1); - rtlpriv->enter_ps = false; - schedule_work(&rtlpriv->works.lps_change_work); + rtl_lps_leave(hw); ppsc->last_delaylps_stamp_jiffies = jiffies; } @@ -1381,8 +1382,7 @@ u8 rtl_is_special_data(struct ieee80211_hw *hw, struct sk_buff *skb, u8 is_tx, if (is_tx) { rtlpriv->ra.is_special_data = true; - rtlpriv->enter_ps = false; - schedule_work(&rtlpriv->works.lps_change_work); + rtl_lps_leave(hw); ppsc->last_delaylps_stamp_jiffies = jiffies; } diff --git a/drivers/net/wireless/realtek/rtlwifi/core.c b/drivers/net/wireless/realtek/rtlwifi/core.c index 8e7f23c11680a5..4da4e458142c46 100644 --- a/drivers/net/wireless/realtek/rtlwifi/core.c +++ b/drivers/net/wireless/realtek/rtlwifi/core.c @@ -1150,10 +1150,8 @@ static void rtl_op_bss_info_changed(struct ieee80211_hw *hw, } else { mstatus = RT_MEDIA_DISCONNECT; - if (mac->link_state == MAC80211_LINKED) { - rtlpriv->enter_ps = false; - schedule_work(&rtlpriv->works.lps_change_work); - } + if (mac->link_state == MAC80211_LINKED) + rtl_lps_leave(hw); if (ppsc->p2p_ps_info.p2p_ps_mode > P2P_PS_NONE) rtl_p2p_ps_cmd(hw, P2P_PS_DISABLE); mac->link_state = MAC80211_NOLINK; @@ -1431,8 +1429,7 @@ static void rtl_op_sw_scan_start(struct ieee80211_hw *hw, } if (mac->link_state == MAC80211_LINKED) { - rtlpriv->enter_ps = false; - schedule_work(&rtlpriv->works.lps_change_work); + rtl_lps_leave(hw); mac->link_state = MAC80211_LINKED_SCANNING; } else { rtl_ips_nic_on(hw); diff --git a/drivers/net/wireless/realtek/rtlwifi/pci.c b/drivers/net/wireless/realtek/rtlwifi/pci.c index 0dfa9eac3926dd..5be4fc96002d1f 100644 --- a/drivers/net/wireless/realtek/rtlwifi/pci.c +++ b/drivers/net/wireless/realtek/rtlwifi/pci.c @@ -663,11 +663,9 @@ static void _rtl_pci_tx_isr(struct ieee80211_hw *hw, int prio) } if (((rtlpriv->link_info.num_rx_inperiod + - rtlpriv->link_info.num_tx_inperiod) > 8) || - (rtlpriv->link_info.num_rx_inperiod > 2)) { - rtlpriv->enter_ps = false; - schedule_work(&rtlpriv->works.lps_change_work); - } + rtlpriv->link_info.num_tx_inperiod) > 8) || + (rtlpriv->link_info.num_rx_inperiod > 2)) + rtl_lps_leave(hw); } static int _rtl_pci_init_one_rxdesc(struct ieee80211_hw *hw, @@ -918,10 +916,8 @@ static void _rtl_pci_rx_interrupt(struct ieee80211_hw *hw) } if (((rtlpriv->link_info.num_rx_inperiod + rtlpriv->link_info.num_tx_inperiod) > 8) || - (rtlpriv->link_info.num_rx_inperiod > 2)) { - rtlpriv->enter_ps = false; - schedule_work(&rtlpriv->works.lps_change_work); - } + (rtlpriv->link_info.num_rx_inperiod > 2)) + rtl_lps_leave(hw); skb = new_skb; no_new: if (rtlpriv->use_new_trx_flow) { diff --git a/drivers/net/wireless/realtek/rtlwifi/ps.c b/drivers/net/wireless/realtek/rtlwifi/ps.c index 18d979affc1861..d0ffc4d508cff2 100644 --- a/drivers/net/wireless/realtek/rtlwifi/ps.c +++ b/drivers/net/wireless/realtek/rtlwifi/ps.c @@ -407,8 +407,8 @@ void rtl_lps_set_psmode(struct ieee80211_hw *hw, u8 rt_psmode) } } -/*Enter the leisure power save mode.*/ -void rtl_lps_enter(struct ieee80211_hw *hw) +/* Interrupt safe routine to enter the leisure power save mode.*/ +static void rtl_lps_enter_core(struct ieee80211_hw *hw) { struct rtl_mac *mac = rtl_mac(rtl_priv(hw)); struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw)); @@ -444,10 +444,9 @@ void rtl_lps_enter(struct ieee80211_hw *hw) spin_unlock_irqrestore(&rtlpriv->locks.lps_lock, flag); } -EXPORT_SYMBOL(rtl_lps_enter); -/*Leave the leisure power save mode.*/ -void rtl_lps_leave(struct ieee80211_hw *hw) +/* Interrupt safe routine to leave the leisure power save mode.*/ +static void rtl_lps_leave_core(struct ieee80211_hw *hw) { struct rtl_priv *rtlpriv = rtl_priv(hw); struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw)); @@ -477,7 +476,6 @@ void rtl_lps_leave(struct ieee80211_hw *hw) } spin_unlock_irqrestore(&rtlpriv->locks.lps_lock, flag); } -EXPORT_SYMBOL(rtl_lps_leave); /* For sw LPS*/ void rtl_swlps_beacon(struct ieee80211_hw *hw, void *data, unsigned int len) @@ -670,12 +668,34 @@ void rtl_lps_change_work_callback(struct work_struct *work) struct rtl_priv *rtlpriv = rtl_priv(hw); if (rtlpriv->enter_ps) - rtl_lps_enter(hw); + rtl_lps_enter_core(hw); else - rtl_lps_leave(hw); + rtl_lps_leave_core(hw); } EXPORT_SYMBOL_GPL(rtl_lps_change_work_callback); +void rtl_lps_enter(struct ieee80211_hw *hw) +{ + struct rtl_priv *rtlpriv = rtl_priv(hw); + + if (!in_interrupt()) + return rtl_lps_enter_core(hw); + rtlpriv->enter_ps = true; + schedule_work(&rtlpriv->works.lps_change_work); +} +EXPORT_SYMBOL_GPL(rtl_lps_enter); + +void rtl_lps_leave(struct ieee80211_hw *hw) +{ + struct rtl_priv *rtlpriv = rtl_priv(hw); + + if (!in_interrupt()) + return rtl_lps_leave_core(hw); + rtlpriv->enter_ps = false; + schedule_work(&rtlpriv->works.lps_change_work); +} +EXPORT_SYMBOL_GPL(rtl_lps_leave); + void rtl_swlps_wq_callback(void *data) { struct rtl_works *rtlworks = container_of_dwork_rtl(data, diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/sw.c index 8b6e37ce3f6690..20bfb373dcd602 100644 --- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/sw.c +++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/sw.c @@ -96,7 +96,7 @@ int rtl92c_init_sw_vars(struct ieee80211_hw *hw) struct rtl_priv *rtlpriv = rtl_priv(hw); struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw)); struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw)); - char *fw_name = "rtlwifi/rtl8192cfwU.bin"; + char *fw_name; rtl8192ce_bt_reg_init(hw); @@ -168,8 +168,13 @@ int rtl92c_init_sw_vars(struct ieee80211_hw *hw) } /* request fw */ - if (IS_81XXC_VENDOR_UMC_B_CUT(rtlhal->version)) + if (IS_VENDOR_UMC_A_CUT(rtlhal->version) && + !IS_92C_SERIAL(rtlhal->version)) + fw_name = "rtlwifi/rtl8192cfwU.bin"; + else if (IS_81XXC_VENDOR_UMC_B_CUT(rtlhal->version)) fw_name = "rtlwifi/rtl8192cfwU_B.bin"; + else + fw_name = "rtlwifi/rtl8192cfw.bin"; rtlpriv->max_fw_size = 0x4000; pr_info("Using firmware %s\n", fw_name); diff --git a/drivers/net/wireless/realtek/rtlwifi/usb.c b/drivers/net/wireless/realtek/rtlwifi/usb.c index 32aa5c1d070a07..3837bbdecf059d 100644 --- a/drivers/net/wireless/realtek/rtlwifi/usb.c +++ b/drivers/net/wireless/realtek/rtlwifi/usb.c @@ -1067,6 +1067,7 @@ int rtl_usb_probe(struct usb_interface *intf, return -ENOMEM; } rtlpriv = hw->priv; + rtlpriv->hw = hw; rtlpriv->usb_data = kzalloc(RTL_USB_MAX_RX_COUNT * sizeof(u32), GFP_KERNEL); if (!rtlpriv->usb_data) diff --git a/drivers/net/wireless/ti/wlcore/init.c b/drivers/net/wireless/ti/wlcore/init.c index d0b7734030ef70..b7974b4dbb346b 100644 --- a/drivers/net/wireless/ti/wlcore/init.c +++ b/drivers/net/wireless/ti/wlcore/init.c @@ -549,6 +549,11 @@ static int wl12xx_init_ap_role(struct wl1271 *wl, struct wl12xx_vif *wlvif) { int ret; + /* Disable filtering */ + ret = wl1271_acx_group_address_tbl(wl, wlvif, false, NULL, 0); + if (ret < 0) + return ret; + ret = wl1271_acx_ap_max_tx_retry(wl, wlvif); if (ret < 0) return ret; diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c index bf2744e1e3db94..0cdcb216908340 100644 --- a/drivers/net/xen-netfront.c +++ b/drivers/net/xen-netfront.c @@ -1397,6 +1397,8 @@ static void xennet_disconnect_backend(struct netfront_info *info) for (i = 0; i < num_queues && info->queues; ++i) { struct netfront_queue *queue = &info->queues[i]; + del_timer_sync(&queue->rx_refill_timer); + if (queue->tx_irq && (queue->tx_irq == queue->rx_irq)) unbind_from_irqhandler(queue->tx_irq, queue); if (queue->tx_irq && (queue->tx_irq != queue->rx_irq)) { @@ -1751,7 +1753,6 @@ static void xennet_destroy_queues(struct netfront_info *info) if (netif_running(info->netdev)) napi_disable(&queue->napi); - del_timer_sync(&queue->rx_refill_timer); netif_napi_del(&queue->napi); } diff --git a/drivers/nvdimm/namespace_devs.c b/drivers/nvdimm/namespace_devs.c index abe5c6bc756c25..aefca644219b7f 100644 --- a/drivers/nvdimm/namespace_devs.c +++ b/drivers/nvdimm/namespace_devs.c @@ -957,25 +957,28 @@ static ssize_t __size_store(struct device *dev, unsigned long long val) { resource_size_t allocated = 0, available = 0; struct nd_region *nd_region = to_nd_region(dev->parent); + struct nd_namespace_common *ndns = to_ndns(dev); struct nd_mapping *nd_mapping; struct nvdimm_drvdata *ndd; struct nd_label_id label_id; u32 flags = 0, remainder; + int rc, i, id = -1; u8 *uuid = NULL; - int rc, i; - if (dev->driver || to_ndns(dev)->claim) + if (dev->driver || ndns->claim) return -EBUSY; if (is_namespace_pmem(dev)) { struct nd_namespace_pmem *nspm = to_nd_namespace_pmem(dev); uuid = nspm->uuid; + id = nspm->id; } else if (is_namespace_blk(dev)) { struct nd_namespace_blk *nsblk = to_nd_namespace_blk(dev); uuid = nsblk->uuid; flags = NSLABEL_FLAG_LOCAL; + id = nsblk->id; } /* @@ -1034,20 +1037,17 @@ static ssize_t __size_store(struct device *dev, unsigned long long val) nd_namespace_pmem_set_resource(nd_region, nspm, val * nd_region->ndr_mappings); - } else if (is_namespace_blk(dev)) { - struct nd_namespace_blk *nsblk = to_nd_namespace_blk(dev); - - /* - * Try to delete the namespace if we deleted all of its - * allocation, this is not the seed device for the - * region, and it is not actively claimed by a btt - * instance. - */ - if (val == 0 && nd_region->ns_seed != dev - && !nsblk->common.claim) - nd_device_unregister(dev, ND_ASYNC); } + /* + * Try to delete the namespace if we deleted all of its + * allocation, this is not the seed or 0th device for the + * region, and it is not actively claimed by a btt, pfn, or dax + * instance. + */ + if (val == 0 && id != 0 && nd_region->ns_seed != dev && !ndns->claim) + nd_device_unregister(dev, ND_ASYNC); + return rc; } diff --git a/drivers/nvdimm/pfn_devs.c b/drivers/nvdimm/pfn_devs.c index cea8350fbc7ec2..6c033c9a2f0692 100644 --- a/drivers/nvdimm/pfn_devs.c +++ b/drivers/nvdimm/pfn_devs.c @@ -108,7 +108,7 @@ static ssize_t align_show(struct device *dev, { struct nd_pfn *nd_pfn = to_nd_pfn_safe(dev); - return sprintf(buf, "%lx\n", nd_pfn->align); + return sprintf(buf, "%ld\n", nd_pfn->align); } static ssize_t __align_store(struct nd_pfn *nd_pfn, const char *buf) @@ -627,15 +627,12 @@ static int nd_pfn_init(struct nd_pfn *nd_pfn) size = resource_size(&nsio->res); npfns = (size - start_pad - end_trunc - SZ_8K) / SZ_4K; if (nd_pfn->mode == PFN_MODE_PMEM) { - unsigned long memmap_size; - /* * vmemmap_populate_hugepages() allocates the memmap array in * HPAGE_SIZE chunks. */ - memmap_size = ALIGN(64 * npfns, HPAGE_SIZE); - offset = ALIGN(start + SZ_8K + memmap_size + dax_label_reserve, - nd_pfn->align) - start; + offset = ALIGN(start + SZ_8K + 64 * npfns + dax_label_reserve, + max(nd_pfn->align, HPAGE_SIZE)) - start; } else if (nd_pfn->mode == PFN_MODE_RAM) offset = ALIGN(start + SZ_8K + dax_label_reserve, nd_pfn->align) - start; diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 79e679d12f3b36..da10b484bd25fe 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -1122,12 +1122,7 @@ int nvme_disable_ctrl(struct nvme_ctrl *ctrl, u64 cap) if (ret) return ret; - /* Checking for ctrl->tagset is a trick to avoid sleeping on module - * load, since we only need the quirk on reset_controller. Notice - * that the HGST device needs this delay only in firmware activation - * procedure; unfortunately we have no (easy) way to verify this. - */ - if ((ctrl->quirks & NVME_QUIRK_DELAY_BEFORE_CHK_RDY) && ctrl->tagset) + if (ctrl->quirks & NVME_QUIRK_DELAY_BEFORE_CHK_RDY) msleep(NVME_QUIRK_DELAY_AMOUNT); return nvme_wait_ready(ctrl, cap, false); diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c index af5e2dc4a3d503..011f88e5663e72 100644 --- a/drivers/nvme/target/configfs.c +++ b/drivers/nvme/target/configfs.c @@ -271,7 +271,7 @@ static ssize_t nvmet_ns_device_path_store(struct config_item *item, mutex_lock(&subsys->lock); ret = -EBUSY; - if (nvmet_ns_enabled(ns)) + if (ns->enabled) goto out_unlock; kfree(ns->device_path); @@ -307,7 +307,7 @@ static ssize_t nvmet_ns_device_nguid_store(struct config_item *item, int ret = 0; mutex_lock(&subsys->lock); - if (nvmet_ns_enabled(ns)) { + if (ns->enabled) { ret = -EBUSY; goto out_unlock; } @@ -339,7 +339,7 @@ CONFIGFS_ATTR(nvmet_ns_, device_nguid); static ssize_t nvmet_ns_enable_show(struct config_item *item, char *page) { - return sprintf(page, "%d\n", nvmet_ns_enabled(to_nvmet_ns(item))); + return sprintf(page, "%d\n", to_nvmet_ns(item)->enabled); } static ssize_t nvmet_ns_enable_store(struct config_item *item, diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index a21437a33adbef..55ce769cecee6c 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -264,7 +264,7 @@ int nvmet_ns_enable(struct nvmet_ns *ns) int ret = 0; mutex_lock(&subsys->lock); - if (!list_empty(&ns->dev_link)) + if (ns->enabled) goto out_unlock; ns->bdev = blkdev_get_by_path(ns->device_path, FMODE_READ | FMODE_WRITE, @@ -309,6 +309,7 @@ int nvmet_ns_enable(struct nvmet_ns *ns) list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) nvmet_add_async_event(ctrl, NVME_AER_TYPE_NOTICE, 0, 0); + ns->enabled = true; ret = 0; out_unlock: mutex_unlock(&subsys->lock); @@ -325,11 +326,11 @@ void nvmet_ns_disable(struct nvmet_ns *ns) struct nvmet_ctrl *ctrl; mutex_lock(&subsys->lock); - if (list_empty(&ns->dev_link)) { - mutex_unlock(&subsys->lock); - return; - } - list_del_init(&ns->dev_link); + if (!ns->enabled) + goto out_unlock; + + ns->enabled = false; + list_del_rcu(&ns->dev_link); mutex_unlock(&subsys->lock); /* @@ -351,6 +352,7 @@ void nvmet_ns_disable(struct nvmet_ns *ns) if (ns->bdev) blkdev_put(ns->bdev, FMODE_WRITE|FMODE_READ); +out_unlock: mutex_unlock(&subsys->lock); } diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h index 76b6eedccaf92c..7655a351320f4e 100644 --- a/drivers/nvme/target/nvmet.h +++ b/drivers/nvme/target/nvmet.h @@ -47,6 +47,7 @@ struct nvmet_ns { loff_t size; u8 nguid[16]; + bool enabled; struct nvmet_subsys *subsys; const char *device_path; @@ -61,11 +62,6 @@ static inline struct nvmet_ns *to_nvmet_ns(struct config_item *item) return container_of(to_config_group(item), struct nvmet_ns, group); } -static inline bool nvmet_ns_enabled(struct nvmet_ns *ns) -{ - return !list_empty_careful(&ns->dev_link); -} - struct nvmet_cq { u16 qid; u16 size; diff --git a/drivers/of/Kconfig b/drivers/of/Kconfig index ba7b034b2b91c9..5132e414a80887 100644 --- a/drivers/of/Kconfig +++ b/drivers/of/Kconfig @@ -112,4 +112,14 @@ config OF_OVERLAY config OF_NUMA bool +config OF_OVERLAY_MGR + bool "Enable Overlay manager" + default n + depends on OF_OVERLAY + help + Enables Overlay manager - it accepts DT entry from command line + overlay_mgr.overlay_dt_entry= and applies all overlays in + it to current DT. It is also possible to apply predefined DT + entry on the fly by writing it to 'current_overlay' sysfs entry. + endif # OF diff --git a/drivers/of/Makefile b/drivers/of/Makefile index d7efd9d458aab4..5b07ad146ce90d 100644 --- a/drivers/of/Makefile +++ b/drivers/of/Makefile @@ -14,5 +14,6 @@ obj-$(CONFIG_OF_RESERVED_MEM) += of_reserved_mem.o obj-$(CONFIG_OF_RESOLVE) += resolver.o obj-$(CONFIG_OF_OVERLAY) += overlay.o obj-$(CONFIG_OF_NUMA) += of_numa.o +obj-$(CONFIG_OF_OVERLAY_MGR) += overlay_mgr.o obj-$(CONFIG_OF_UNITTEST) += unittest-data/ diff --git a/drivers/of/fdt.c b/drivers/of/fdt.c index c89d5d231a0e30..86688088c50c2a 100644 --- a/drivers/of/fdt.c +++ b/drivers/of/fdt.c @@ -1054,42 +1054,66 @@ int __init early_init_dt_scan_memory(unsigned long node, const char *uname, return 0; } +/* + * Convert configs to something easy to use in C code + */ +#if defined(CONFIG_CMDLINE_FORCE) +static const int overwrite_incoming_cmdline = 1; +static const int read_dt_cmdline; +static const int concat_cmdline; +#elif defined(CONFIG_CMDLINE_EXTEND) +static const int overwrite_incoming_cmdline; +static const int read_dt_cmdline = 1; +static const int concat_cmdline = 1; +#else /* CMDLINE_FROM_BOOTLOADER */ +static const int overwrite_incoming_cmdline; +static const int read_dt_cmdline = 1; +static const int concat_cmdline; +#endif + +#ifdef CONFIG_CMDLINE +static const char *config_cmdline = CONFIG_CMDLINE; +#else +static const char *config_cmdline = ""; +#endif + int __init early_init_dt_scan_chosen(unsigned long node, const char *uname, int depth, void *data) { - int l; - const char *p; + int l = 0; + const char *p = NULL; + char *cmdline = data; pr_debug("search \"chosen\", depth: %d, uname: %s\n", depth, uname); - if (depth != 1 || !data || + if (depth != 1 || !cmdline || (strcmp(uname, "chosen") != 0 && strcmp(uname, "chosen@0") != 0)) return 0; early_init_dt_check_for_initrd(node); - /* Retrieve command line */ - p = of_get_flat_dt_prop(node, "bootargs", &l); - if (p != NULL && l > 0) - strlcpy(data, p, min((int)l, COMMAND_LINE_SIZE)); - - /* - * CONFIG_CMDLINE is meant to be a default in case nothing else - * managed to set the command line, unless CONFIG_CMDLINE_FORCE - * is set in which case we override whatever was found earlier. - */ -#ifdef CONFIG_CMDLINE -#if defined(CONFIG_CMDLINE_EXTEND) - strlcat(data, " ", COMMAND_LINE_SIZE); - strlcat(data, CONFIG_CMDLINE, COMMAND_LINE_SIZE); -#elif defined(CONFIG_CMDLINE_FORCE) - strlcpy(data, CONFIG_CMDLINE, COMMAND_LINE_SIZE); -#else - /* No arguments from boot loader, use kernel's cmdl*/ - if (!((char *)data)[0]) - strlcpy(data, CONFIG_CMDLINE, COMMAND_LINE_SIZE); -#endif -#endif /* CONFIG_CMDLINE */ + /* Put CONFIG_CMDLINE in if forced or if data had nothing in it to start */ + if (overwrite_incoming_cmdline || !cmdline[0]) + strlcpy(cmdline, config_cmdline, COMMAND_LINE_SIZE); + + /* Retrieve command line unless forcing */ + if (read_dt_cmdline) + p = of_get_flat_dt_prop(node, "bootargs", &l); + + if (p != NULL && l > 0) { + if (concat_cmdline) { + int cmdline_len; + int copy_len; + strlcat(cmdline, " ", COMMAND_LINE_SIZE); + cmdline_len = strlen(cmdline); + copy_len = COMMAND_LINE_SIZE - cmdline_len - 1; + copy_len = min((int)l, copy_len); + strncpy(cmdline + cmdline_len, p, copy_len); + cmdline[cmdline_len + copy_len] = '\0'; + } else { + strlcpy(cmdline, p, min((int)l, COMMAND_LINE_SIZE)); + } + } pr_debug("Command line is: %s\n", (char*)data); diff --git a/drivers/of/of_numa.c b/drivers/of/of_numa.c index f63d4b0deff009..a53982a330ea40 100644 --- a/drivers/of/of_numa.c +++ b/drivers/of/of_numa.c @@ -176,7 +176,12 @@ int of_node_to_nid(struct device_node *device) np->name); of_node_put(np); - if (!r) + /* + * If numa=off passed on command line, or with a defective + * device tree, the nid may not be in the set of possible + * nodes. Check for this case and return NUMA_NO_NODE. + */ + if (!r && nid < MAX_NUMNODES && node_possible(nid)) return nid; return NUMA_NO_NODE; diff --git a/drivers/of/overlay_mgr.c b/drivers/of/overlay_mgr.c new file mode 100644 index 00000000000000..6b15f7c3c1cfbd --- /dev/null +++ b/drivers/of/overlay_mgr.c @@ -0,0 +1,152 @@ +/* + * Overlay manager that allows to apply list of overlays from DT entry + * + * Copyright (C) 2016 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include + +static char *of_overlay_dt_entry; +module_param_named(overlay_dt_entry, of_overlay_dt_entry, charp, 0644); + +static char *of_overlay_dt_apply; +DEFINE_MUTEX(of_overlay_mgr_mutex); + +static int of_overlay_mgr_apply_overlay(struct device_node *onp) +{ + int ret; + + ret = of_overlay_create(onp); + if (ret < 0) { + pr_err("overlay_mgr: fail to create overlay: %d\n", ret); + of_node_put(onp); + return ret; + } + pr_info("overlay_mgr: %s overlay applied\n", onp->name); + return 0; +} + +static int of_overlay_mgr_apply_dt(struct device *dev, char *dt_entry) +{ + struct device_node *enp = dev->of_node; + struct device_node *next; + struct device_node *prev = NULL; + + if (!enp) { + pr_err("overlay_mgr: no dt entry\n"); + return -ENODEV; + } + + enp = of_get_child_by_name(enp, dt_entry); + if (!enp) { + pr_err("overlay_mgr: dt entry %s not found\n", dt_entry); + return -ENODEV; + } + pr_info("overlay_mgr: apply %s dt entry\n", enp->name); + while ((next = of_get_next_available_child(enp, prev)) != NULL) { + if (strncmp(next->name, "overlay", 7) == 0) + of_overlay_mgr_apply_overlay(next); + prev = next; + } + return 0; +} + +static ssize_t current_overlay_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + size_t len; + + mutex_lock(&of_overlay_mgr_mutex); + if (!of_overlay_dt_apply) { + mutex_unlock(&of_overlay_mgr_mutex); + return 0; + } + len = strlen(of_overlay_dt_apply); + if (len >= PAGE_SIZE) + len = PAGE_SIZE - 1; + memcpy(buf, of_overlay_dt_apply, len + 1); + mutex_unlock(&of_overlay_mgr_mutex); + return len; +} + +static ssize_t current_overlay_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + mutex_lock(&of_overlay_mgr_mutex); + kfree(of_overlay_dt_apply); + of_overlay_dt_apply = kmalloc(size, GFP_KERNEL); + if (!of_overlay_dt_apply) { + pr_err("overlay_mgr: fail to allocate memory\n"); + mutex_unlock(&of_overlay_mgr_mutex); + return 0; + } + memcpy(of_overlay_dt_apply, buf, size); + of_overlay_dt_apply[size - 1] = '\0'; + + if (of_overlay_mgr_apply_dt(dev, of_overlay_dt_apply)) { + kfree(of_overlay_dt_apply); + of_overlay_dt_apply = NULL; + size = 0; + } + mutex_unlock(&of_overlay_mgr_mutex); + return size; +} + +static DEVICE_ATTR(current_overlay, 0644, current_overlay_show, + current_overlay_store); + +static int of_overlay_mgr_probe(struct platform_device *pdev) +{ + char *cur_entry; + char *next_entry; + + if (device_create_file(&pdev->dev, &dev_attr_current_overlay)) + pr_err("overlay_mgr: fail to register apply entry\n"); + + if (!of_overlay_dt_entry) + return 0; + next_entry = of_overlay_dt_entry; + do { + cur_entry = next_entry; + next_entry = strchr(cur_entry, ','); + if (next_entry) + *next_entry++ = '\0'; + of_overlay_mgr_apply_dt(&pdev->dev, cur_entry); + } while (next_entry); + return 0; +} + +static const struct of_device_id of_overlay_mgr_match[] = { + { .compatible = "linux,overlay_manager", }, + {} +}; + +static struct platform_driver of_overlay_mgr_driver = { + .probe = of_overlay_mgr_probe, + .driver = { + .name = "overlay_manager", + .of_match_table = of_match_ptr(of_overlay_mgr_match), + }, +}; + +static int __init of_overlay_mgr_init(void) +{ + return platform_driver_register(&of_overlay_mgr_driver); +} + +postcore_initcall(of_overlay_mgr_init); diff --git a/drivers/pci/host/pcie-designware.c b/drivers/pci/host/pcie-designware.c index bed19994c1e94d..af8f6e92e8851c 100644 --- a/drivers/pci/host/pcie-designware.c +++ b/drivers/pci/host/pcie-designware.c @@ -807,11 +807,6 @@ void dw_pcie_setup_rc(struct pcie_port *pp) { u32 val; - /* get iATU unroll support */ - pp->iatu_unroll_enabled = dw_pcie_iatu_unroll_enabled(pp); - dev_dbg(pp->dev, "iATU unroll: %s\n", - pp->iatu_unroll_enabled ? "enabled" : "disabled"); - /* set the number of lanes */ val = dw_pcie_readl_rc(pp, PCIE_PORT_LINK_CONTROL); val &= ~PORT_LINK_MODE_MASK; @@ -882,6 +877,11 @@ void dw_pcie_setup_rc(struct pcie_port *pp) * we should not program the ATU here. */ if (!pp->ops->rd_other_conf) { + /* get iATU unroll support */ + pp->iatu_unroll_enabled = dw_pcie_iatu_unroll_enabled(pp); + dev_dbg(pp->dev, "iATU unroll: %s\n", + pp->iatu_unroll_enabled ? "enabled" : "disabled"); + dw_pcie_prog_outbound_atu(pp, PCIE_ATU_REGION_INDEX0, PCIE_ATU_TYPE_MEM, pp->mem_base, pp->mem_bus_addr, pp->mem_size); diff --git a/drivers/pci/host/pcie-rockchip.c b/drivers/pci/host/pcie-rockchip.c index e04f69beb42d0f..3452983d3569bd 100644 --- a/drivers/pci/host/pcie-rockchip.c +++ b/drivers/pci/host/pcie-rockchip.c @@ -533,7 +533,7 @@ static int rockchip_pcie_init_port(struct rockchip_pcie *rockchip) /* Fix the transmitted FTS count desired to exit from L0s. */ status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL_PLC1); - status = (status & PCIE_CORE_CTRL_PLC1_FTS_MASK) | + status = (status & ~PCIE_CORE_CTRL_PLC1_FTS_MASK) | (PCIE_CORE_CTRL_PLC1_FTS_CNT << PCIE_CORE_CTRL_PLC1_FTS_SHIFT); rockchip_pcie_write(rockchip, status, PCIE_CORE_CTRL_PLC1); @@ -590,8 +590,8 @@ static int rockchip_pcie_init_port(struct rockchip_pcie *rockchip) /* Check the final link width from negotiated lane counter from MGMT */ status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL); - status = 0x1 << ((status & PCIE_CORE_PL_CONF_LANE_MASK) >> - PCIE_CORE_PL_CONF_LANE_MASK); + status = 0x1 << ((status & PCIE_CORE_PL_CONF_LANE_MASK) >> + PCIE_CORE_PL_CONF_LANE_SHIFT); dev_dbg(dev, "current link width is x%d\n", status); rockchip_pcie_write(rockchip, ROCKCHIP_VENDOR_ID, diff --git a/drivers/pci/hotplug/rpadlpar_core.c b/drivers/pci/hotplug/rpadlpar_core.c index dc67f39779ecd3..c614ff7c3bc3f9 100644 --- a/drivers/pci/hotplug/rpadlpar_core.c +++ b/drivers/pci/hotplug/rpadlpar_core.c @@ -257,8 +257,13 @@ static int dlpar_add_phb(char *drc_name, struct device_node *dn) static int dlpar_add_vio_slot(char *drc_name, struct device_node *dn) { - if (vio_find_node(dn)) + struct vio_dev *vio_dev; + + vio_dev = vio_find_node(dn); + if (vio_dev) { + put_device(&vio_dev->dev); return -EINVAL; + } if (!vio_register_device_node(dn)) { printk(KERN_ERR @@ -334,6 +339,9 @@ static int dlpar_remove_vio_slot(char *drc_name, struct device_node *dn) return -EINVAL; vio_unregister_device(vio_dev); + + put_device(&vio_dev->dev); + return 0; } diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c index ad70507cfb566a..3455f752d5e4fb 100644 --- a/drivers/pci/msi.c +++ b/drivers/pci/msi.c @@ -1294,7 +1294,8 @@ const struct cpumask *pci_irq_get_affinity(struct pci_dev *dev, int nr) } else if (dev->msi_enabled) { struct msi_desc *entry = first_pci_msi_entry(dev); - if (WARN_ON_ONCE(!entry || nr >= entry->nvec_used)) + if (WARN_ON_ONCE(!entry || !entry->affinity || + nr >= entry->nvec_used)) return NULL; return &entry->affinity[nr]; diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index ba34907538f616..eda6a7cf0e5434 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -2106,6 +2106,10 @@ bool pci_dev_run_wake(struct pci_dev *dev) if (!dev->pme_support) return false; + /* PME-capable in principle, but not from the intended sleep state */ + if (!pci_pme_capable(dev, pci_target_state(dev))) + return false; + while (bus->parent) { struct pci_dev *bridge = bus->self; diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c index 0ec649d961d7a6..b0916b126923ea 100644 --- a/drivers/pci/pcie/aspm.c +++ b/drivers/pci/pcie/aspm.c @@ -518,25 +518,32 @@ static struct pcie_link_state *alloc_pcie_link_state(struct pci_dev *pdev) link = kzalloc(sizeof(*link), GFP_KERNEL); if (!link) return NULL; + INIT_LIST_HEAD(&link->sibling); INIT_LIST_HEAD(&link->children); INIT_LIST_HEAD(&link->link); link->pdev = pdev; - if (pci_pcie_type(pdev) != PCI_EXP_TYPE_ROOT_PORT) { + + /* + * Root Ports and PCI/PCI-X to PCIe Bridges are roots of PCIe + * hierarchies. + */ + if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT || + pci_pcie_type(pdev) == PCI_EXP_TYPE_PCIE_BRIDGE) { + link->root = link; + } else { struct pcie_link_state *parent; + parent = pdev->bus->parent->self->link_state; if (!parent) { kfree(link); return NULL; } + link->parent = parent; + link->root = link->parent->root; list_add(&link->link, &parent->children); } - /* Setup a pointer to the root port link */ - if (!link->parent) - link->root = link; - else - link->root = link->parent->root; list_add(&link->sibling, &link_list); pdev->link_state = link; diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index 104c46d5312188..300770cdc084ad 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -1050,6 +1050,7 @@ void set_pcie_port_type(struct pci_dev *pdev) pos = pci_find_capability(pdev, PCI_CAP_ID_EXP); if (!pos) return; + pdev->pcie_cap = pos; pci_read_config_word(pdev, pos + PCI_EXP_FLAGS, ®16); pdev->pcie_flags_reg = reg16; @@ -1057,13 +1058,14 @@ void set_pcie_port_type(struct pci_dev *pdev) pdev->pcie_mpss = reg16 & PCI_EXP_DEVCAP_PAYLOAD; /* - * A Root Port is always the upstream end of a Link. No PCIe - * component has two Links. Two Links are connected by a Switch - * that has a Port on each Link and internal logic to connect the - * two Ports. + * A Root Port or a PCI-to-PCIe bridge is always the upstream end + * of a Link. No PCIe component has two Links. Two Links are + * connected by a Switch that has a Port on each Link and internal + * logic to connect the two Ports. */ type = pci_pcie_type(pdev); - if (type == PCI_EXP_TYPE_ROOT_PORT) + if (type == PCI_EXP_TYPE_ROOT_PORT || + type == PCI_EXP_TYPE_PCIE_BRIDGE) pdev->has_secondary_link = 1; else if (type == PCI_EXP_TYPE_UPSTREAM || type == PCI_EXP_TYPE_DOWNSTREAM) { diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index c232729f5b1b68..3a035e073889a5 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -3137,8 +3137,9 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x22b5, quirk_remove_d3_delay); DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x22b7, quirk_remove_d3_delay); DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x2298, quirk_remove_d3_delay); DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x229c, quirk_remove_d3_delay); + /* - * Some devices may pass our check in pci_intx_mask_supported if + * Some devices may pass our check in pci_intx_mask_supported() if * PCI_COMMAND_INTX_DISABLE works though they actually do not properly * support this feature. */ @@ -3146,53 +3147,139 @@ static void quirk_broken_intx_masking(struct pci_dev *dev) { dev->broken_intx_masking = 1; } -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_CHELSIO, 0x0030, - quirk_broken_intx_masking); -DECLARE_PCI_FIXUP_HEADER(0x1814, 0x0601, /* Ralink RT2800 802.11n PCI */ - quirk_broken_intx_masking); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_CHELSIO, 0x0030, + quirk_broken_intx_masking); +DECLARE_PCI_FIXUP_FINAL(0x1814, 0x0601, /* Ralink RT2800 802.11n PCI */ + quirk_broken_intx_masking); + /* * Realtek RTL8169 PCI Gigabit Ethernet Controller (rev 10) * Subsystem: Realtek RTL8169/8110 Family PCI Gigabit Ethernet NIC * * RTL8110SC - Fails under PCI device assignment using DisINTx masking. */ -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_REALTEK, 0x8169, - quirk_broken_intx_masking); -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_MELLANOX, PCI_ANY_ID, - quirk_broken_intx_masking); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_REALTEK, 0x8169, + quirk_broken_intx_masking); /* * Intel i40e (XL710/X710) 10/20/40GbE NICs all have broken INTx masking, * DisINTx can be set but the interrupt status bit is non-functional. */ -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1572, - quirk_broken_intx_masking); -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1574, - quirk_broken_intx_masking); -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1580, - quirk_broken_intx_masking); -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1581, - quirk_broken_intx_masking); -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1583, - quirk_broken_intx_masking); -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1584, - quirk_broken_intx_masking); -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1585, - quirk_broken_intx_masking); -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1586, - quirk_broken_intx_masking); -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1587, - quirk_broken_intx_masking); -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1588, - quirk_broken_intx_masking); -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1589, - quirk_broken_intx_masking); -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x37d0, - quirk_broken_intx_masking); -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x37d1, - quirk_broken_intx_masking); -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x37d2, - quirk_broken_intx_masking); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1572, + quirk_broken_intx_masking); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1574, + quirk_broken_intx_masking); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1580, + quirk_broken_intx_masking); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1581, + quirk_broken_intx_masking); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1583, + quirk_broken_intx_masking); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1584, + quirk_broken_intx_masking); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1585, + quirk_broken_intx_masking); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1586, + quirk_broken_intx_masking); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1587, + quirk_broken_intx_masking); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1588, + quirk_broken_intx_masking); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1589, + quirk_broken_intx_masking); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x37d0, + quirk_broken_intx_masking); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x37d1, + quirk_broken_intx_masking); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x37d2, + quirk_broken_intx_masking); + +static u16 mellanox_broken_intx_devs[] = { + PCI_DEVICE_ID_MELLANOX_HERMON_SDR, + PCI_DEVICE_ID_MELLANOX_HERMON_DDR, + PCI_DEVICE_ID_MELLANOX_HERMON_QDR, + PCI_DEVICE_ID_MELLANOX_HERMON_DDR_GEN2, + PCI_DEVICE_ID_MELLANOX_HERMON_QDR_GEN2, + PCI_DEVICE_ID_MELLANOX_HERMON_EN, + PCI_DEVICE_ID_MELLANOX_HERMON_EN_GEN2, + PCI_DEVICE_ID_MELLANOX_CONNECTX_EN, + PCI_DEVICE_ID_MELLANOX_CONNECTX_EN_T_GEN2, + PCI_DEVICE_ID_MELLANOX_CONNECTX_EN_GEN2, + PCI_DEVICE_ID_MELLANOX_CONNECTX_EN_5_GEN2, + PCI_DEVICE_ID_MELLANOX_CONNECTX2, + PCI_DEVICE_ID_MELLANOX_CONNECTX3, + PCI_DEVICE_ID_MELLANOX_CONNECTX3_PRO, +}; + +#define CONNECTX_4_CURR_MAX_MINOR 99 +#define CONNECTX_4_INTX_SUPPORT_MINOR 14 + +/* + * Check ConnectX-4/LX FW version to see if it supports legacy interrupts. + * If so, don't mark it as broken. + * FW minor > 99 means older FW version format and no INTx masking support. + * FW minor < 14 means new FW version format and no INTx masking support. + */ +static void mellanox_check_broken_intx_masking(struct pci_dev *pdev) +{ + __be32 __iomem *fw_ver; + u16 fw_major; + u16 fw_minor; + u16 fw_subminor; + u32 fw_maj_min; + u32 fw_sub_min; + int i; + + for (i = 0; i < ARRAY_SIZE(mellanox_broken_intx_devs); i++) { + if (pdev->device == mellanox_broken_intx_devs[i]) { + pdev->broken_intx_masking = 1; + return; + } + } + + /* Getting here means Connect-IB cards and up. Connect-IB has no INTx + * support so shouldn't be checked further + */ + if (pdev->device == PCI_DEVICE_ID_MELLANOX_CONNECTIB) + return; + + if (pdev->device != PCI_DEVICE_ID_MELLANOX_CONNECTX4 && + pdev->device != PCI_DEVICE_ID_MELLANOX_CONNECTX4_LX) + return; + + /* For ConnectX-4 and ConnectX-4LX, need to check FW support */ + if (pci_enable_device_mem(pdev)) { + dev_warn(&pdev->dev, "Can't enable device memory\n"); + return; + } + + fw_ver = ioremap(pci_resource_start(pdev, 0), 4); + if (!fw_ver) { + dev_warn(&pdev->dev, "Can't map ConnectX-4 initialization segment\n"); + goto out; + } + + /* Reading from resource space should be 32b aligned */ + fw_maj_min = ioread32be(fw_ver); + fw_sub_min = ioread32be(fw_ver + 1); + fw_major = fw_maj_min & 0xffff; + fw_minor = fw_maj_min >> 16; + fw_subminor = fw_sub_min & 0xffff; + if (fw_minor > CONNECTX_4_CURR_MAX_MINOR || + fw_minor < CONNECTX_4_INTX_SUPPORT_MINOR) { + dev_warn(&pdev->dev, "ConnectX-4: FW %u.%u.%u doesn't support INTx masking, disabling. Please upgrade FW to %d.14.1100 and up for INTx support\n", + fw_major, fw_minor, fw_subminor, pdev->device == + PCI_DEVICE_ID_MELLANOX_CONNECTX4 ? 12 : 14); + pdev->broken_intx_masking = 1; + } + + iounmap(fw_ver); + +out: + pci_disable_device(pdev); +} +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_MELLANOX, PCI_ANY_ID, + mellanox_check_broken_intx_masking); static void quirk_no_bus_reset(struct pci_dev *dev) { @@ -3255,6 +3342,25 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CACTUS_RIDGE_4C DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_PORT_RIDGE, quirk_thunderbolt_hotplug_msi); +static void quirk_chelsio_extend_vpd(struct pci_dev *dev) +{ + pci_set_vpd_size(dev, 8192); +} + +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_CHELSIO, 0x20, quirk_chelsio_extend_vpd); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_CHELSIO, 0x21, quirk_chelsio_extend_vpd); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_CHELSIO, 0x22, quirk_chelsio_extend_vpd); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_CHELSIO, 0x23, quirk_chelsio_extend_vpd); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_CHELSIO, 0x24, quirk_chelsio_extend_vpd); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_CHELSIO, 0x25, quirk_chelsio_extend_vpd); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_CHELSIO, 0x26, quirk_chelsio_extend_vpd); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_CHELSIO, 0x30, quirk_chelsio_extend_vpd); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_CHELSIO, 0x31, quirk_chelsio_extend_vpd); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_CHELSIO, 0x32, quirk_chelsio_extend_vpd); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_CHELSIO, 0x35, quirk_chelsio_extend_vpd); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_CHELSIO, 0x36, quirk_chelsio_extend_vpd); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_CHELSIO, 0x37, quirk_chelsio_extend_vpd); + #ifdef CONFIG_ACPI /* * Apple: Shutdown Cactus Ridge Thunderbolt controller. diff --git a/drivers/pinctrl/freescale/pinctrl-imx.c b/drivers/pinctrl/freescale/pinctrl-imx.c index 79c4e14a5a75e9..5ef7e875b50e89 100644 --- a/drivers/pinctrl/freescale/pinctrl-imx.c +++ b/drivers/pinctrl/freescale/pinctrl-imx.c @@ -778,10 +778,10 @@ int imx_pinctrl_probe(struct platform_device *pdev, imx_pinctrl_desc->name = dev_name(&pdev->dev); imx_pinctrl_desc->pins = info->pins; imx_pinctrl_desc->npins = info->npins; - imx_pinctrl_desc->pctlops = &imx_pctrl_ops, - imx_pinctrl_desc->pmxops = &imx_pmx_ops, - imx_pinctrl_desc->confops = &imx_pinconf_ops, - imx_pinctrl_desc->owner = THIS_MODULE, + imx_pinctrl_desc->pctlops = &imx_pctrl_ops; + imx_pinctrl_desc->pmxops = &imx_pmx_ops; + imx_pinctrl_desc->confops = &imx_pinconf_ops; + imx_pinctrl_desc->owner = THIS_MODULE; ret = imx_pinctrl_probe_dt(pdev, info); if (ret) { diff --git a/drivers/pinctrl/intel/pinctrl-baytrail.c b/drivers/pinctrl/intel/pinctrl-baytrail.c index 71bbeb9321bad5..583ae3f38fc09a 100644 --- a/drivers/pinctrl/intel/pinctrl-baytrail.c +++ b/drivers/pinctrl/intel/pinctrl-baytrail.c @@ -731,16 +731,23 @@ static void __iomem *byt_gpio_reg(struct byt_gpio *vg, unsigned int offset, int reg) { struct byt_community *comm = byt_get_community(vg, offset); - u32 reg_offset = 0; + u32 reg_offset; if (!comm) return NULL; offset -= comm->pin_base; - if (reg == BYT_INT_STAT_REG) + switch (reg) { + case BYT_INT_STAT_REG: reg_offset = (offset / 32) * 4; - else + break; + case BYT_DEBOUNCE_REG: + reg_offset = 0; + break; + default: reg_offset = comm->pad_map[offset] * 16; + break; + } return comm->reg_base + reg_offset + reg; } @@ -1092,6 +1099,7 @@ static int byt_pin_config_get(struct pinctrl_dev *pctl_dev, unsigned int offset, enum pin_config_param param = pinconf_to_config_param(*config); void __iomem *conf_reg = byt_gpio_reg(vg, offset, BYT_CONF0_REG); void __iomem *val_reg = byt_gpio_reg(vg, offset, BYT_VAL_REG); + void __iomem *db_reg = byt_gpio_reg(vg, offset, BYT_DEBOUNCE_REG); unsigned long flags; u32 conf, pull, val, debounce; u16 arg = 0; @@ -1128,7 +1136,7 @@ static int byt_pin_config_get(struct pinctrl_dev *pctl_dev, unsigned int offset, return -EINVAL; raw_spin_lock_irqsave(&vg->lock, flags); - debounce = readl(byt_gpio_reg(vg, offset, BYT_DEBOUNCE_REG)); + debounce = readl(db_reg); raw_spin_unlock_irqrestore(&vg->lock, flags); switch (debounce & BYT_DEBOUNCE_PULSE_MASK) { @@ -1176,6 +1184,7 @@ static int byt_pin_config_set(struct pinctrl_dev *pctl_dev, unsigned int param, arg; void __iomem *conf_reg = byt_gpio_reg(vg, offset, BYT_CONF0_REG); void __iomem *val_reg = byt_gpio_reg(vg, offset, BYT_VAL_REG); + void __iomem *db_reg = byt_gpio_reg(vg, offset, BYT_DEBOUNCE_REG); unsigned long flags; u32 conf, val, debounce; int i, ret = 0; @@ -1238,36 +1247,40 @@ static int byt_pin_config_set(struct pinctrl_dev *pctl_dev, break; case PIN_CONFIG_INPUT_DEBOUNCE: - debounce = readl(byt_gpio_reg(vg, offset, - BYT_DEBOUNCE_REG)); - conf &= ~BYT_DEBOUNCE_PULSE_MASK; + debounce = readl(db_reg); + debounce &= ~BYT_DEBOUNCE_PULSE_MASK; switch (arg) { + case 0: + conf &= BYT_DEBOUNCE_EN; + break; case 375: - conf |= BYT_DEBOUNCE_PULSE_375US; + debounce |= BYT_DEBOUNCE_PULSE_375US; break; case 750: - conf |= BYT_DEBOUNCE_PULSE_750US; + debounce |= BYT_DEBOUNCE_PULSE_750US; break; case 1500: - conf |= BYT_DEBOUNCE_PULSE_1500US; + debounce |= BYT_DEBOUNCE_PULSE_1500US; break; case 3000: - conf |= BYT_DEBOUNCE_PULSE_3MS; + debounce |= BYT_DEBOUNCE_PULSE_3MS; break; case 6000: - conf |= BYT_DEBOUNCE_PULSE_6MS; + debounce |= BYT_DEBOUNCE_PULSE_6MS; break; case 12000: - conf |= BYT_DEBOUNCE_PULSE_12MS; + debounce |= BYT_DEBOUNCE_PULSE_12MS; break; case 24000: - conf |= BYT_DEBOUNCE_PULSE_24MS; + debounce |= BYT_DEBOUNCE_PULSE_24MS; break; default: ret = -EINVAL; } + if (!ret) + writel(debounce, db_reg); break; default: ret = -ENOTSUPP; @@ -1606,7 +1619,9 @@ static void byt_gpio_irq_handler(struct irq_desc *desc) continue; } + raw_spin_lock(&vg->lock); pending = readl(reg); + raw_spin_unlock(&vg->lock); for_each_set_bit(pin, &pending, 32) { virq = irq_find_mapping(vg->chip.irqdomain, base + pin); generic_handle_irq(virq); diff --git a/drivers/pinctrl/intel/pinctrl-broxton.c b/drivers/pinctrl/intel/pinctrl-broxton.c index 59cb7a6fc5bef3..901b356b09d716 100644 --- a/drivers/pinctrl/intel/pinctrl-broxton.c +++ b/drivers/pinctrl/intel/pinctrl-broxton.c @@ -19,7 +19,7 @@ #define BXT_PAD_OWN 0x020 #define BXT_HOSTSW_OWN 0x080 -#define BXT_PADCFGLOCK 0x090 +#define BXT_PADCFGLOCK 0x060 #define BXT_GPI_IE 0x110 #define BXT_COMMUNITY(s, e) \ diff --git a/drivers/pinctrl/intel/pinctrl-merrifield.c b/drivers/pinctrl/intel/pinctrl-merrifield.c index 7826c7f0cb7cac..9931be6af0cadf 100644 --- a/drivers/pinctrl/intel/pinctrl-merrifield.c +++ b/drivers/pinctrl/intel/pinctrl-merrifield.c @@ -794,6 +794,9 @@ static int mrfld_config_set(struct pinctrl_dev *pctldev, unsigned int pin, unsigned int i; int ret; + if (!mrfld_buf_available(mp, pin)) + return -ENOTSUPP; + for (i = 0; i < nconfigs; i++) { switch (pinconf_to_config_param(configs[i])) { case PIN_CONFIG_BIAS_DISABLE: diff --git a/drivers/pinctrl/meson/pinctrl-meson.c b/drivers/pinctrl/meson/pinctrl-meson.c index 57122eda155afd..9443c9d408c602 100644 --- a/drivers/pinctrl/meson/pinctrl-meson.c +++ b/drivers/pinctrl/meson/pinctrl-meson.c @@ -212,7 +212,7 @@ static int meson_pmx_request_gpio(struct pinctrl_dev *pcdev, { struct meson_pinctrl *pc = pinctrl_dev_get_drvdata(pcdev); - meson_pmx_disable_other_groups(pc, range->pin_base + offset, -1); + meson_pmx_disable_other_groups(pc, offset, -1); return 0; } diff --git a/drivers/pinctrl/pinctrl-amd.c b/drivers/pinctrl/pinctrl-amd.c index aea310a918210e..c9a146948192db 100644 --- a/drivers/pinctrl/pinctrl-amd.c +++ b/drivers/pinctrl/pinctrl-amd.c @@ -382,26 +382,21 @@ static int amd_gpio_irq_set_type(struct irq_data *d, unsigned int type) { int ret = 0; u32 pin_reg; - unsigned long flags; - bool level_trig; - u32 active_level; + unsigned long flags, irq_flags; struct gpio_chip *gc = irq_data_get_irq_chip_data(d); struct amd_gpio *gpio_dev = gpiochip_get_data(gc); spin_lock_irqsave(&gpio_dev->lock, flags); pin_reg = readl(gpio_dev->base + (d->hwirq)*4); - /* - * When level_trig is set EDGE and active_level is set HIGH in BIOS - * default settings, ignore incoming settings from client and use - * BIOS settings to configure GPIO register. + /* Ignore the settings coming from the client and + * read the values from the ACPI tables + * while setting the trigger type */ - level_trig = !(pin_reg & (LEVEL_TRIGGER << LEVEL_TRIG_OFF)); - active_level = pin_reg & (ACTIVE_LEVEL_MASK << ACTIVE_LEVEL_OFF); - if(level_trig && - ((active_level >> ACTIVE_LEVEL_OFF) == ACTIVE_HIGH)) - type = IRQ_TYPE_EDGE_FALLING; + irq_flags = irq_get_trigger_type(d->irq); + if (irq_flags != IRQ_TYPE_NONE) + type = irq_flags; switch (type & IRQ_TYPE_SENSE_MASK) { case IRQ_TYPE_EDGE_RISING: diff --git a/drivers/pinctrl/sh-pfc/core.c b/drivers/pinctrl/sh-pfc/core.c index f3a8897d4e8f40..cf80ce1dd7cea5 100644 --- a/drivers/pinctrl/sh-pfc/core.c +++ b/drivers/pinctrl/sh-pfc/core.c @@ -389,6 +389,21 @@ int sh_pfc_config_mux(struct sh_pfc *pfc, unsigned mark, int pinmux_type) return 0; } +const struct sh_pfc_bias_info * +sh_pfc_pin_to_bias_info(const struct sh_pfc_bias_info *info, + unsigned int num, unsigned int pin) +{ + unsigned int i; + + for (i = 0; i < num; i++) + if (info[i].pin == pin) + return &info[i]; + + WARN_ONCE(1, "Pin %u is not in bias info list\n", pin); + + return NULL; +} + static int sh_pfc_init_ranges(struct sh_pfc *pfc) { struct sh_pfc_pin_range *range; diff --git a/drivers/pinctrl/sh-pfc/core.h b/drivers/pinctrl/sh-pfc/core.h index 0bbdea5849f417..6d598dd6372085 100644 --- a/drivers/pinctrl/sh-pfc/core.h +++ b/drivers/pinctrl/sh-pfc/core.h @@ -33,4 +33,8 @@ void sh_pfc_write_reg(struct sh_pfc *pfc, u32 reg, unsigned int width, int sh_pfc_get_pin_index(struct sh_pfc *pfc, unsigned int pin); int sh_pfc_config_mux(struct sh_pfc *pfc, unsigned mark, int pinmux_type); +const struct sh_pfc_bias_info * +sh_pfc_pin_to_bias_info(const struct sh_pfc_bias_info *info, + unsigned int num, unsigned int pin); + #endif /* __SH_PFC_CORE_H__ */ diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a7795.c b/drivers/pinctrl/sh-pfc/pfc-r8a7795.c index 2e8cc2adbed7e2..84cee66b1e0843 100644 --- a/drivers/pinctrl/sh-pfc/pfc-r8a7795.c +++ b/drivers/pinctrl/sh-pfc/pfc-r8a7795.c @@ -5188,184 +5188,183 @@ static int r8a7795_pin_to_pocctrl(struct sh_pfc *pfc, unsigned int pin, u32 *poc #define PU5 0x14 #define PU6 0x18 -static const struct { - u16 reg : 11; - u16 bit : 5; -} pullups[] = { - [RCAR_GP_PIN(2, 11)] = { PU0, 31 }, /* AVB_PHY_INT */ - [RCAR_GP_PIN(2, 10)] = { PU0, 30 }, /* AVB_MAGIC */ - [RCAR_GP_PIN(2, 9)] = { PU0, 29 }, /* AVB_MDC */ - - [RCAR_GP_PIN(1, 19)] = { PU1, 31 }, /* A19 */ - [RCAR_GP_PIN(1, 18)] = { PU1, 30 }, /* A18 */ - [RCAR_GP_PIN(1, 17)] = { PU1, 29 }, /* A17 */ - [RCAR_GP_PIN(1, 16)] = { PU1, 28 }, /* A16 */ - [RCAR_GP_PIN(1, 15)] = { PU1, 27 }, /* A15 */ - [RCAR_GP_PIN(1, 14)] = { PU1, 26 }, /* A14 */ - [RCAR_GP_PIN(1, 13)] = { PU1, 25 }, /* A13 */ - [RCAR_GP_PIN(1, 12)] = { PU1, 24 }, /* A12 */ - [RCAR_GP_PIN(1, 11)] = { PU1, 23 }, /* A11 */ - [RCAR_GP_PIN(1, 10)] = { PU1, 22 }, /* A10 */ - [RCAR_GP_PIN(1, 9)] = { PU1, 21 }, /* A9 */ - [RCAR_GP_PIN(1, 8)] = { PU1, 20 }, /* A8 */ - [RCAR_GP_PIN(1, 7)] = { PU1, 19 }, /* A7 */ - [RCAR_GP_PIN(1, 6)] = { PU1, 18 }, /* A6 */ - [RCAR_GP_PIN(1, 5)] = { PU1, 17 }, /* A5 */ - [RCAR_GP_PIN(1, 4)] = { PU1, 16 }, /* A4 */ - [RCAR_GP_PIN(1, 3)] = { PU1, 15 }, /* A3 */ - [RCAR_GP_PIN(1, 2)] = { PU1, 14 }, /* A2 */ - [RCAR_GP_PIN(1, 1)] = { PU1, 13 }, /* A1 */ - [RCAR_GP_PIN(1, 0)] = { PU1, 12 }, /* A0 */ - [RCAR_GP_PIN(2, 8)] = { PU1, 11 }, /* PWM2_A */ - [RCAR_GP_PIN(2, 7)] = { PU1, 10 }, /* PWM1_A */ - [RCAR_GP_PIN(2, 6)] = { PU1, 9 }, /* PWM0 */ - [RCAR_GP_PIN(2, 5)] = { PU1, 8 }, /* IRQ5 */ - [RCAR_GP_PIN(2, 4)] = { PU1, 7 }, /* IRQ4 */ - [RCAR_GP_PIN(2, 3)] = { PU1, 6 }, /* IRQ3 */ - [RCAR_GP_PIN(2, 2)] = { PU1, 5 }, /* IRQ2 */ - [RCAR_GP_PIN(2, 1)] = { PU1, 4 }, /* IRQ1 */ - [RCAR_GP_PIN(2, 0)] = { PU1, 3 }, /* IRQ0 */ - [RCAR_GP_PIN(2, 14)] = { PU1, 2 }, /* AVB_AVTP_CAPTURE_A */ - [RCAR_GP_PIN(2, 13)] = { PU1, 1 }, /* AVB_AVTP_MATCH_A */ - [RCAR_GP_PIN(2, 12)] = { PU1, 0 }, /* AVB_LINK */ - - [RCAR_GP_PIN(7, 3)] = { PU2, 29 }, /* HDMI1_CEC */ - [RCAR_GP_PIN(7, 2)] = { PU2, 28 }, /* HDMI0_CEC */ - [RCAR_GP_PIN(7, 1)] = { PU2, 27 }, /* AVS2 */ - [RCAR_GP_PIN(7, 0)] = { PU2, 26 }, /* AVS1 */ - [RCAR_GP_PIN(0, 15)] = { PU2, 25 }, /* D15 */ - [RCAR_GP_PIN(0, 14)] = { PU2, 24 }, /* D14 */ - [RCAR_GP_PIN(0, 13)] = { PU2, 23 }, /* D13 */ - [RCAR_GP_PIN(0, 12)] = { PU2, 22 }, /* D12 */ - [RCAR_GP_PIN(0, 11)] = { PU2, 21 }, /* D11 */ - [RCAR_GP_PIN(0, 10)] = { PU2, 20 }, /* D10 */ - [RCAR_GP_PIN(0, 9)] = { PU2, 19 }, /* D9 */ - [RCAR_GP_PIN(0, 8)] = { PU2, 18 }, /* D8 */ - [RCAR_GP_PIN(0, 7)] = { PU2, 17 }, /* D7 */ - [RCAR_GP_PIN(0, 6)] = { PU2, 16 }, /* D6 */ - [RCAR_GP_PIN(0, 5)] = { PU2, 15 }, /* D5 */ - [RCAR_GP_PIN(0, 4)] = { PU2, 14 }, /* D4 */ - [RCAR_GP_PIN(0, 3)] = { PU2, 13 }, /* D3 */ - [RCAR_GP_PIN(0, 2)] = { PU2, 12 }, /* D2 */ - [RCAR_GP_PIN(0, 1)] = { PU2, 11 }, /* D1 */ - [RCAR_GP_PIN(0, 0)] = { PU2, 10 }, /* D0 */ - [RCAR_GP_PIN(1, 27)] = { PU2, 8 }, /* EX_WAIT0_A */ - [RCAR_GP_PIN(1, 26)] = { PU2, 7 }, /* WE1_N */ - [RCAR_GP_PIN(1, 25)] = { PU2, 6 }, /* WE0_N */ - [RCAR_GP_PIN(1, 24)] = { PU2, 5 }, /* RD_WR_N */ - [RCAR_GP_PIN(1, 23)] = { PU2, 4 }, /* RD_N */ - [RCAR_GP_PIN(1, 22)] = { PU2, 3 }, /* BS_N */ - [RCAR_GP_PIN(1, 21)] = { PU2, 2 }, /* CS1_N_A26 */ - [RCAR_GP_PIN(1, 20)] = { PU2, 1 }, /* CS0_N */ - - [RCAR_GP_PIN(4, 9)] = { PU3, 31 }, /* SD3_DAT0 */ - [RCAR_GP_PIN(4, 8)] = { PU3, 30 }, /* SD3_CMD */ - [RCAR_GP_PIN(4, 7)] = { PU3, 29 }, /* SD3_CLK */ - [RCAR_GP_PIN(4, 6)] = { PU3, 28 }, /* SD2_DS */ - [RCAR_GP_PIN(4, 5)] = { PU3, 27 }, /* SD2_DAT3 */ - [RCAR_GP_PIN(4, 4)] = { PU3, 26 }, /* SD2_DAT2 */ - [RCAR_GP_PIN(4, 3)] = { PU3, 25 }, /* SD2_DAT1 */ - [RCAR_GP_PIN(4, 2)] = { PU3, 24 }, /* SD2_DAT0 */ - [RCAR_GP_PIN(4, 1)] = { PU3, 23 }, /* SD2_CMD */ - [RCAR_GP_PIN(4, 0)] = { PU3, 22 }, /* SD2_CLK */ - [RCAR_GP_PIN(3, 11)] = { PU3, 21 }, /* SD1_DAT3 */ - [RCAR_GP_PIN(3, 10)] = { PU3, 20 }, /* SD1_DAT2 */ - [RCAR_GP_PIN(3, 9)] = { PU3, 19 }, /* SD1_DAT1 */ - [RCAR_GP_PIN(3, 8)] = { PU3, 18 }, /* SD1_DAT0 */ - [RCAR_GP_PIN(3, 7)] = { PU3, 17 }, /* SD1_CMD */ - [RCAR_GP_PIN(3, 6)] = { PU3, 16 }, /* SD1_CLK */ - [RCAR_GP_PIN(3, 5)] = { PU3, 15 }, /* SD0_DAT3 */ - [RCAR_GP_PIN(3, 4)] = { PU3, 14 }, /* SD0_DAT2 */ - [RCAR_GP_PIN(3, 3)] = { PU3, 13 }, /* SD0_DAT1 */ - [RCAR_GP_PIN(3, 2)] = { PU3, 12 }, /* SD0_DAT0 */ - [RCAR_GP_PIN(3, 1)] = { PU3, 11 }, /* SD0_CMD */ - [RCAR_GP_PIN(3, 0)] = { PU3, 10 }, /* SD0_CLK */ - - [RCAR_GP_PIN(5, 19)] = { PU4, 31 }, /* MSIOF0_SS1 */ - [RCAR_GP_PIN(5, 18)] = { PU4, 30 }, /* MSIOF0_SYNC */ - [RCAR_GP_PIN(5, 17)] = { PU4, 29 }, /* MSIOF0_SCK */ - [RCAR_GP_PIN(5, 16)] = { PU4, 28 }, /* HRTS0_N */ - [RCAR_GP_PIN(5, 15)] = { PU4, 27 }, /* HCTS0_N */ - [RCAR_GP_PIN(5, 14)] = { PU4, 26 }, /* HTX0 */ - [RCAR_GP_PIN(5, 13)] = { PU4, 25 }, /* HRX0 */ - [RCAR_GP_PIN(5, 12)] = { PU4, 24 }, /* HSCK0 */ - [RCAR_GP_PIN(5, 11)] = { PU4, 23 }, /* RX2_A */ - [RCAR_GP_PIN(5, 10)] = { PU4, 22 }, /* TX2_A */ - [RCAR_GP_PIN(5, 9)] = { PU4, 21 }, /* SCK2 */ - [RCAR_GP_PIN(5, 8)] = { PU4, 20 }, /* RTS1_N_TANS */ - [RCAR_GP_PIN(5, 7)] = { PU4, 19 }, /* CTS1_N */ - [RCAR_GP_PIN(5, 6)] = { PU4, 18 }, /* TX1_A */ - [RCAR_GP_PIN(5, 5)] = { PU4, 17 }, /* RX1_A */ - [RCAR_GP_PIN(5, 4)] = { PU4, 16 }, /* RTS0_N_TANS */ - [RCAR_GP_PIN(5, 3)] = { PU4, 15 }, /* CTS0_N */ - [RCAR_GP_PIN(5, 2)] = { PU4, 14 }, /* TX0 */ - [RCAR_GP_PIN(5, 1)] = { PU4, 13 }, /* RX0 */ - [RCAR_GP_PIN(5, 0)] = { PU4, 12 }, /* SCK0 */ - [RCAR_GP_PIN(3, 15)] = { PU4, 11 }, /* SD1_WP */ - [RCAR_GP_PIN(3, 14)] = { PU4, 10 }, /* SD1_CD */ - [RCAR_GP_PIN(3, 13)] = { PU4, 9 }, /* SD0_WP */ - [RCAR_GP_PIN(3, 12)] = { PU4, 8 }, /* SD0_CD */ - [RCAR_GP_PIN(4, 17)] = { PU4, 7 }, /* SD3_DS */ - [RCAR_GP_PIN(4, 16)] = { PU4, 6 }, /* SD3_DAT7 */ - [RCAR_GP_PIN(4, 15)] = { PU4, 5 }, /* SD3_DAT6 */ - [RCAR_GP_PIN(4, 14)] = { PU4, 4 }, /* SD3_DAT5 */ - [RCAR_GP_PIN(4, 13)] = { PU4, 3 }, /* SD3_DAT4 */ - [RCAR_GP_PIN(4, 12)] = { PU4, 2 }, /* SD3_DAT3 */ - [RCAR_GP_PIN(4, 11)] = { PU4, 1 }, /* SD3_DAT2 */ - [RCAR_GP_PIN(4, 10)] = { PU4, 0 }, /* SD3_DAT1 */ - - [RCAR_GP_PIN(6, 24)] = { PU5, 31 }, /* USB0_PWEN */ - [RCAR_GP_PIN(6, 23)] = { PU5, 30 }, /* AUDIO_CLKB_B */ - [RCAR_GP_PIN(6, 22)] = { PU5, 29 }, /* AUDIO_CLKA_A */ - [RCAR_GP_PIN(6, 21)] = { PU5, 28 }, /* SSI_SDATA9_A */ - [RCAR_GP_PIN(6, 20)] = { PU5, 27 }, /* SSI_SDATA8 */ - [RCAR_GP_PIN(6, 19)] = { PU5, 26 }, /* SSI_SDATA7 */ - [RCAR_GP_PIN(6, 18)] = { PU5, 25 }, /* SSI_WS78 */ - [RCAR_GP_PIN(6, 17)] = { PU5, 24 }, /* SSI_SCK78 */ - [RCAR_GP_PIN(6, 16)] = { PU5, 23 }, /* SSI_SDATA6 */ - [RCAR_GP_PIN(6, 15)] = { PU5, 22 }, /* SSI_WS6 */ - [RCAR_GP_PIN(6, 14)] = { PU5, 21 }, /* SSI_SCK6 */ - [RCAR_GP_PIN(6, 13)] = { PU5, 20 }, /* SSI_SDATA5 */ - [RCAR_GP_PIN(6, 12)] = { PU5, 19 }, /* SSI_WS5 */ - [RCAR_GP_PIN(6, 11)] = { PU5, 18 }, /* SSI_SCK5 */ - [RCAR_GP_PIN(6, 10)] = { PU5, 17 }, /* SSI_SDATA4 */ - [RCAR_GP_PIN(6, 9)] = { PU5, 16 }, /* SSI_WS4 */ - [RCAR_GP_PIN(6, 8)] = { PU5, 15 }, /* SSI_SCK4 */ - [RCAR_GP_PIN(6, 7)] = { PU5, 14 }, /* SSI_SDATA3 */ - [RCAR_GP_PIN(6, 6)] = { PU5, 13 }, /* SSI_WS34 */ - [RCAR_GP_PIN(6, 5)] = { PU5, 12 }, /* SSI_SCK34 */ - [RCAR_GP_PIN(6, 4)] = { PU5, 11 }, /* SSI_SDATA2_A */ - [RCAR_GP_PIN(6, 3)] = { PU5, 10 }, /* SSI_SDATA1_A */ - [RCAR_GP_PIN(6, 2)] = { PU5, 9 }, /* SSI_SDATA0 */ - [RCAR_GP_PIN(6, 1)] = { PU5, 8 }, /* SSI_WS01239 */ - [RCAR_GP_PIN(6, 0)] = { PU5, 7 }, /* SSI_SCK01239 */ - [RCAR_GP_PIN(5, 25)] = { PU5, 5 }, /* MLB_DAT */ - [RCAR_GP_PIN(5, 24)] = { PU5, 4 }, /* MLB_SIG */ - [RCAR_GP_PIN(5, 23)] = { PU5, 3 }, /* MLB_CLK */ - [RCAR_GP_PIN(5, 22)] = { PU5, 2 }, /* MSIOF0_RXD */ - [RCAR_GP_PIN(5, 21)] = { PU5, 1 }, /* MSIOF0_SS2 */ - [RCAR_GP_PIN(5, 20)] = { PU5, 0 }, /* MSIOF0_TXD */ - - [RCAR_GP_PIN(6, 31)] = { PU6, 6 }, /* USB31_OVC */ - [RCAR_GP_PIN(6, 30)] = { PU6, 5 }, /* USB31_PWEN */ - [RCAR_GP_PIN(6, 29)] = { PU6, 4 }, /* USB30_OVC */ - [RCAR_GP_PIN(6, 28)] = { PU6, 3 }, /* USB30_PWEN */ - [RCAR_GP_PIN(6, 27)] = { PU6, 2 }, /* USB1_OVC */ - [RCAR_GP_PIN(6, 26)] = { PU6, 1 }, /* USB1_PWEN */ - [RCAR_GP_PIN(6, 25)] = { PU6, 0 }, /* USB0_OVC */ +static const struct sh_pfc_bias_info bias_info[] = { + { RCAR_GP_PIN(2, 11), PU0, 31 }, /* AVB_PHY_INT */ + { RCAR_GP_PIN(2, 10), PU0, 30 }, /* AVB_MAGIC */ + { RCAR_GP_PIN(2, 9), PU0, 29 }, /* AVB_MDC */ + + { RCAR_GP_PIN(1, 19), PU1, 31 }, /* A19 */ + { RCAR_GP_PIN(1, 18), PU1, 30 }, /* A18 */ + { RCAR_GP_PIN(1, 17), PU1, 29 }, /* A17 */ + { RCAR_GP_PIN(1, 16), PU1, 28 }, /* A16 */ + { RCAR_GP_PIN(1, 15), PU1, 27 }, /* A15 */ + { RCAR_GP_PIN(1, 14), PU1, 26 }, /* A14 */ + { RCAR_GP_PIN(1, 13), PU1, 25 }, /* A13 */ + { RCAR_GP_PIN(1, 12), PU1, 24 }, /* A12 */ + { RCAR_GP_PIN(1, 11), PU1, 23 }, /* A11 */ + { RCAR_GP_PIN(1, 10), PU1, 22 }, /* A10 */ + { RCAR_GP_PIN(1, 9), PU1, 21 }, /* A9 */ + { RCAR_GP_PIN(1, 8), PU1, 20 }, /* A8 */ + { RCAR_GP_PIN(1, 7), PU1, 19 }, /* A7 */ + { RCAR_GP_PIN(1, 6), PU1, 18 }, /* A6 */ + { RCAR_GP_PIN(1, 5), PU1, 17 }, /* A5 */ + { RCAR_GP_PIN(1, 4), PU1, 16 }, /* A4 */ + { RCAR_GP_PIN(1, 3), PU1, 15 }, /* A3 */ + { RCAR_GP_PIN(1, 2), PU1, 14 }, /* A2 */ + { RCAR_GP_PIN(1, 1), PU1, 13 }, /* A1 */ + { RCAR_GP_PIN(1, 0), PU1, 12 }, /* A0 */ + { RCAR_GP_PIN(2, 8), PU1, 11 }, /* PWM2_A */ + { RCAR_GP_PIN(2, 7), PU1, 10 }, /* PWM1_A */ + { RCAR_GP_PIN(2, 6), PU1, 9 }, /* PWM0 */ + { RCAR_GP_PIN(2, 5), PU1, 8 }, /* IRQ5 */ + { RCAR_GP_PIN(2, 4), PU1, 7 }, /* IRQ4 */ + { RCAR_GP_PIN(2, 3), PU1, 6 }, /* IRQ3 */ + { RCAR_GP_PIN(2, 2), PU1, 5 }, /* IRQ2 */ + { RCAR_GP_PIN(2, 1), PU1, 4 }, /* IRQ1 */ + { RCAR_GP_PIN(2, 0), PU1, 3 }, /* IRQ0 */ + { RCAR_GP_PIN(2, 14), PU1, 2 }, /* AVB_AVTP_CAPTURE_A */ + { RCAR_GP_PIN(2, 13), PU1, 1 }, /* AVB_AVTP_MATCH_A */ + { RCAR_GP_PIN(2, 12), PU1, 0 }, /* AVB_LINK */ + + { RCAR_GP_PIN(7, 3), PU2, 29 }, /* HDMI1_CEC */ + { RCAR_GP_PIN(7, 2), PU2, 28 }, /* HDMI0_CEC */ + { RCAR_GP_PIN(7, 1), PU2, 27 }, /* AVS2 */ + { RCAR_GP_PIN(7, 0), PU2, 26 }, /* AVS1 */ + { RCAR_GP_PIN(0, 15), PU2, 25 }, /* D15 */ + { RCAR_GP_PIN(0, 14), PU2, 24 }, /* D14 */ + { RCAR_GP_PIN(0, 13), PU2, 23 }, /* D13 */ + { RCAR_GP_PIN(0, 12), PU2, 22 }, /* D12 */ + { RCAR_GP_PIN(0, 11), PU2, 21 }, /* D11 */ + { RCAR_GP_PIN(0, 10), PU2, 20 }, /* D10 */ + { RCAR_GP_PIN(0, 9), PU2, 19 }, /* D9 */ + { RCAR_GP_PIN(0, 8), PU2, 18 }, /* D8 */ + { RCAR_GP_PIN(0, 7), PU2, 17 }, /* D7 */ + { RCAR_GP_PIN(0, 6), PU2, 16 }, /* D6 */ + { RCAR_GP_PIN(0, 5), PU2, 15 }, /* D5 */ + { RCAR_GP_PIN(0, 4), PU2, 14 }, /* D4 */ + { RCAR_GP_PIN(0, 3), PU2, 13 }, /* D3 */ + { RCAR_GP_PIN(0, 2), PU2, 12 }, /* D2 */ + { RCAR_GP_PIN(0, 1), PU2, 11 }, /* D1 */ + { RCAR_GP_PIN(0, 0), PU2, 10 }, /* D0 */ + { RCAR_GP_PIN(1, 27), PU2, 8 }, /* EX_WAIT0_A */ + { RCAR_GP_PIN(1, 26), PU2, 7 }, /* WE1_N */ + { RCAR_GP_PIN(1, 25), PU2, 6 }, /* WE0_N */ + { RCAR_GP_PIN(1, 24), PU2, 5 }, /* RD_WR_N */ + { RCAR_GP_PIN(1, 23), PU2, 4 }, /* RD_N */ + { RCAR_GP_PIN(1, 22), PU2, 3 }, /* BS_N */ + { RCAR_GP_PIN(1, 21), PU2, 2 }, /* CS1_N_A26 */ + { RCAR_GP_PIN(1, 20), PU2, 1 }, /* CS0_N */ + + { RCAR_GP_PIN(4, 9), PU3, 31 }, /* SD3_DAT0 */ + { RCAR_GP_PIN(4, 8), PU3, 30 }, /* SD3_CMD */ + { RCAR_GP_PIN(4, 7), PU3, 29 }, /* SD3_CLK */ + { RCAR_GP_PIN(4, 6), PU3, 28 }, /* SD2_DS */ + { RCAR_GP_PIN(4, 5), PU3, 27 }, /* SD2_DAT3 */ + { RCAR_GP_PIN(4, 4), PU3, 26 }, /* SD2_DAT2 */ + { RCAR_GP_PIN(4, 3), PU3, 25 }, /* SD2_DAT1 */ + { RCAR_GP_PIN(4, 2), PU3, 24 }, /* SD2_DAT0 */ + { RCAR_GP_PIN(4, 1), PU3, 23 }, /* SD2_CMD */ + { RCAR_GP_PIN(4, 0), PU3, 22 }, /* SD2_CLK */ + { RCAR_GP_PIN(3, 11), PU3, 21 }, /* SD1_DAT3 */ + { RCAR_GP_PIN(3, 10), PU3, 20 }, /* SD1_DAT2 */ + { RCAR_GP_PIN(3, 9), PU3, 19 }, /* SD1_DAT1 */ + { RCAR_GP_PIN(3, 8), PU3, 18 }, /* SD1_DAT0 */ + { RCAR_GP_PIN(3, 7), PU3, 17 }, /* SD1_CMD */ + { RCAR_GP_PIN(3, 6), PU3, 16 }, /* SD1_CLK */ + { RCAR_GP_PIN(3, 5), PU3, 15 }, /* SD0_DAT3 */ + { RCAR_GP_PIN(3, 4), PU3, 14 }, /* SD0_DAT2 */ + { RCAR_GP_PIN(3, 3), PU3, 13 }, /* SD0_DAT1 */ + { RCAR_GP_PIN(3, 2), PU3, 12 }, /* SD0_DAT0 */ + { RCAR_GP_PIN(3, 1), PU3, 11 }, /* SD0_CMD */ + { RCAR_GP_PIN(3, 0), PU3, 10 }, /* SD0_CLK */ + + { RCAR_GP_PIN(5, 19), PU4, 31 }, /* MSIOF0_SS1 */ + { RCAR_GP_PIN(5, 18), PU4, 30 }, /* MSIOF0_SYNC */ + { RCAR_GP_PIN(5, 17), PU4, 29 }, /* MSIOF0_SCK */ + { RCAR_GP_PIN(5, 16), PU4, 28 }, /* HRTS0_N */ + { RCAR_GP_PIN(5, 15), PU4, 27 }, /* HCTS0_N */ + { RCAR_GP_PIN(5, 14), PU4, 26 }, /* HTX0 */ + { RCAR_GP_PIN(5, 13), PU4, 25 }, /* HRX0 */ + { RCAR_GP_PIN(5, 12), PU4, 24 }, /* HSCK0 */ + { RCAR_GP_PIN(5, 11), PU4, 23 }, /* RX2_A */ + { RCAR_GP_PIN(5, 10), PU4, 22 }, /* TX2_A */ + { RCAR_GP_PIN(5, 9), PU4, 21 }, /* SCK2 */ + { RCAR_GP_PIN(5, 8), PU4, 20 }, /* RTS1_N_TANS */ + { RCAR_GP_PIN(5, 7), PU4, 19 }, /* CTS1_N */ + { RCAR_GP_PIN(5, 6), PU4, 18 }, /* TX1_A */ + { RCAR_GP_PIN(5, 5), PU4, 17 }, /* RX1_A */ + { RCAR_GP_PIN(5, 4), PU4, 16 }, /* RTS0_N_TANS */ + { RCAR_GP_PIN(5, 3), PU4, 15 }, /* CTS0_N */ + { RCAR_GP_PIN(5, 2), PU4, 14 }, /* TX0 */ + { RCAR_GP_PIN(5, 1), PU4, 13 }, /* RX0 */ + { RCAR_GP_PIN(5, 0), PU4, 12 }, /* SCK0 */ + { RCAR_GP_PIN(3, 15), PU4, 11 }, /* SD1_WP */ + { RCAR_GP_PIN(3, 14), PU4, 10 }, /* SD1_CD */ + { RCAR_GP_PIN(3, 13), PU4, 9 }, /* SD0_WP */ + { RCAR_GP_PIN(3, 12), PU4, 8 }, /* SD0_CD */ + { RCAR_GP_PIN(4, 17), PU4, 7 }, /* SD3_DS */ + { RCAR_GP_PIN(4, 16), PU4, 6 }, /* SD3_DAT7 */ + { RCAR_GP_PIN(4, 15), PU4, 5 }, /* SD3_DAT6 */ + { RCAR_GP_PIN(4, 14), PU4, 4 }, /* SD3_DAT5 */ + { RCAR_GP_PIN(4, 13), PU4, 3 }, /* SD3_DAT4 */ + { RCAR_GP_PIN(4, 12), PU4, 2 }, /* SD3_DAT3 */ + { RCAR_GP_PIN(4, 11), PU4, 1 }, /* SD3_DAT2 */ + { RCAR_GP_PIN(4, 10), PU4, 0 }, /* SD3_DAT1 */ + + { RCAR_GP_PIN(6, 24), PU5, 31 }, /* USB0_PWEN */ + { RCAR_GP_PIN(6, 23), PU5, 30 }, /* AUDIO_CLKB_B */ + { RCAR_GP_PIN(6, 22), PU5, 29 }, /* AUDIO_CLKA_A */ + { RCAR_GP_PIN(6, 21), PU5, 28 }, /* SSI_SDATA9_A */ + { RCAR_GP_PIN(6, 20), PU5, 27 }, /* SSI_SDATA8 */ + { RCAR_GP_PIN(6, 19), PU5, 26 }, /* SSI_SDATA7 */ + { RCAR_GP_PIN(6, 18), PU5, 25 }, /* SSI_WS78 */ + { RCAR_GP_PIN(6, 17), PU5, 24 }, /* SSI_SCK78 */ + { RCAR_GP_PIN(6, 16), PU5, 23 }, /* SSI_SDATA6 */ + { RCAR_GP_PIN(6, 15), PU5, 22 }, /* SSI_WS6 */ + { RCAR_GP_PIN(6, 14), PU5, 21 }, /* SSI_SCK6 */ + { RCAR_GP_PIN(6, 13), PU5, 20 }, /* SSI_SDATA5 */ + { RCAR_GP_PIN(6, 12), PU5, 19 }, /* SSI_WS5 */ + { RCAR_GP_PIN(6, 11), PU5, 18 }, /* SSI_SCK5 */ + { RCAR_GP_PIN(6, 10), PU5, 17 }, /* SSI_SDATA4 */ + { RCAR_GP_PIN(6, 9), PU5, 16 }, /* SSI_WS4 */ + { RCAR_GP_PIN(6, 8), PU5, 15 }, /* SSI_SCK4 */ + { RCAR_GP_PIN(6, 7), PU5, 14 }, /* SSI_SDATA3 */ + { RCAR_GP_PIN(6, 6), PU5, 13 }, /* SSI_WS34 */ + { RCAR_GP_PIN(6, 5), PU5, 12 }, /* SSI_SCK34 */ + { RCAR_GP_PIN(6, 4), PU5, 11 }, /* SSI_SDATA2_A */ + { RCAR_GP_PIN(6, 3), PU5, 10 }, /* SSI_SDATA1_A */ + { RCAR_GP_PIN(6, 2), PU5, 9 }, /* SSI_SDATA0 */ + { RCAR_GP_PIN(6, 1), PU5, 8 }, /* SSI_WS01239 */ + { RCAR_GP_PIN(6, 0), PU5, 7 }, /* SSI_SCK01239 */ + { RCAR_GP_PIN(5, 25), PU5, 5 }, /* MLB_DAT */ + { RCAR_GP_PIN(5, 24), PU5, 4 }, /* MLB_SIG */ + { RCAR_GP_PIN(5, 23), PU5, 3 }, /* MLB_CLK */ + { RCAR_GP_PIN(5, 22), PU5, 2 }, /* MSIOF0_RXD */ + { RCAR_GP_PIN(5, 21), PU5, 1 }, /* MSIOF0_SS2 */ + { RCAR_GP_PIN(5, 20), PU5, 0 }, /* MSIOF0_TXD */ + + { RCAR_GP_PIN(6, 31), PU6, 6 }, /* USB31_OVC */ + { RCAR_GP_PIN(6, 30), PU6, 5 }, /* USB31_PWEN */ + { RCAR_GP_PIN(6, 29), PU6, 4 }, /* USB30_OVC */ + { RCAR_GP_PIN(6, 28), PU6, 3 }, /* USB30_PWEN */ + { RCAR_GP_PIN(6, 27), PU6, 2 }, /* USB1_OVC */ + { RCAR_GP_PIN(6, 26), PU6, 1 }, /* USB1_PWEN */ + { RCAR_GP_PIN(6, 25), PU6, 0 }, /* USB0_OVC */ }; static unsigned int r8a7795_pinmux_get_bias(struct sh_pfc *pfc, unsigned int pin) { + const struct sh_pfc_bias_info *info; u32 reg; u32 bit; - if (WARN_ON_ONCE(!pullups[pin].reg)) + info = sh_pfc_pin_to_bias_info(bias_info, ARRAY_SIZE(bias_info), pin); + if (!info) return PIN_CONFIG_BIAS_DISABLE; - reg = pullups[pin].reg; - bit = BIT(pullups[pin].bit); + reg = info->reg; + bit = BIT(info->bit); if (sh_pfc_read_reg(pfc, PUEN + reg, 32) & bit) { if (sh_pfc_read_reg(pfc, PUD + reg, 32) & bit) @@ -5379,15 +5378,17 @@ static unsigned int r8a7795_pinmux_get_bias(struct sh_pfc *pfc, static void r8a7795_pinmux_set_bias(struct sh_pfc *pfc, unsigned int pin, unsigned int bias) { + const struct sh_pfc_bias_info *info; u32 enable, updown; u32 reg; u32 bit; - if (WARN_ON_ONCE(!pullups[pin].reg)) + info = sh_pfc_pin_to_bias_info(bias_info, ARRAY_SIZE(bias_info), pin); + if (!info) return; - reg = pullups[pin].reg; - bit = BIT(pullups[pin].bit); + reg = info->reg; + bit = BIT(info->bit); enable = sh_pfc_read_reg(pfc, PUEN + reg, 32) & ~bit; if (bias != PIN_CONFIG_BIAS_DISABLE) diff --git a/drivers/pinctrl/sh-pfc/pinctrl.c b/drivers/pinctrl/sh-pfc/pinctrl.c index c5772584594ce3..fcacfa73ef6e9b 100644 --- a/drivers/pinctrl/sh-pfc/pinctrl.c +++ b/drivers/pinctrl/sh-pfc/pinctrl.c @@ -570,7 +570,8 @@ static bool sh_pfc_pinconf_validate(struct sh_pfc *pfc, unsigned int _pin, switch (param) { case PIN_CONFIG_BIAS_DISABLE: - return true; + return pin->configs & + (SH_PFC_PIN_CFG_PULL_UP | SH_PFC_PIN_CFG_PULL_DOWN); case PIN_CONFIG_BIAS_PULL_UP: return pin->configs & SH_PFC_PIN_CFG_PULL_UP; diff --git a/drivers/pinctrl/sh-pfc/sh_pfc.h b/drivers/pinctrl/sh-pfc/sh_pfc.h index 2345421103db34..9556c172e3d208 100644 --- a/drivers/pinctrl/sh-pfc/sh_pfc.h +++ b/drivers/pinctrl/sh-pfc/sh_pfc.h @@ -189,6 +189,12 @@ struct sh_pfc_window { unsigned long size; }; +struct sh_pfc_bias_info { + u16 pin; + u16 reg : 11; + u16 bit : 5; +}; + struct sh_pfc_pin_range; struct sh_pfc { diff --git a/drivers/pinctrl/uniphier/pinctrl-uniphier-ld20.c b/drivers/pinctrl/uniphier/pinctrl-uniphier-ld20.c index aa8bd9794683b7..96686336e3a396 100644 --- a/drivers/pinctrl/uniphier/pinctrl-uniphier-ld20.c +++ b/drivers/pinctrl/uniphier/pinctrl-uniphier-ld20.c @@ -561,7 +561,7 @@ static const int ether_rgmii_muxvals[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; static const unsigned ether_rmii_pins[] = {30, 31, 32, 33, 34, 35, 36, 37, 39, 41, 42, 45}; -static const int ether_rmii_muxvals[] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; +static const int ether_rmii_muxvals[] = {0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1}; static const unsigned i2c0_pins[] = {63, 64}; static const int i2c0_muxvals[] = {0, 0}; static const unsigned i2c1_pins[] = {65, 66}; diff --git a/drivers/platform/Kconfig b/drivers/platform/Kconfig index c11db8bceea13e..6bfb6fb8637ab3 100644 --- a/drivers/platform/Kconfig +++ b/drivers/platform/Kconfig @@ -8,3 +8,5 @@ endif source "drivers/platform/goldfish/Kconfig" source "drivers/platform/chrome/Kconfig" + +source "drivers/platform/hisi/Kconfig" diff --git a/drivers/platform/Makefile b/drivers/platform/Makefile index ca2692510733df..57bc1d19ab5282 100644 --- a/drivers/platform/Makefile +++ b/drivers/platform/Makefile @@ -7,3 +7,4 @@ obj-$(CONFIG_MIPS) += mips/ obj-$(CONFIG_OLPC) += olpc/ obj-$(CONFIG_GOLDFISH) += goldfish/ obj-$(CONFIG_CHROME_PLATFORMS) += chrome/ +obj-$(CONFIG_ARCH_HISI) += hisi/ diff --git a/drivers/platform/goldfish/Makefile b/drivers/platform/goldfish/Makefile index d3487125838cdc..277a820ee4e11c 100644 --- a/drivers/platform/goldfish/Makefile +++ b/drivers/platform/goldfish/Makefile @@ -2,4 +2,5 @@ # Makefile for Goldfish platform specific drivers # obj-$(CONFIG_GOLDFISH_BUS) += pdev_bus.o -obj-$(CONFIG_GOLDFISH_PIPE) += goldfish_pipe.o +obj-$(CONFIG_GOLDFISH_PIPE) += goldfish_pipe_all.o +goldfish_pipe_all-objs := goldfish_pipe.o goldfish_pipe_v2.o diff --git a/drivers/platform/goldfish/goldfish_pipe.c b/drivers/platform/goldfish/goldfish_pipe.c index 1aba2c74160eb5..91e0a5645799a7 100644 --- a/drivers/platform/goldfish/goldfish_pipe.c +++ b/drivers/platform/goldfish/goldfish_pipe.c @@ -15,52 +15,11 @@ * */ -/* This source file contains the implementation of a special device driver - * that intends to provide a *very* fast communication channel between the - * guest system and the QEMU emulator. - * - * Usage from the guest is simply the following (error handling simplified): - * - * int fd = open("/dev/qemu_pipe",O_RDWR); - * .... write() or read() through the pipe. - * - * This driver doesn't deal with the exact protocol used during the session. - * It is intended to be as simple as something like: - * - * // do this _just_ after opening the fd to connect to a specific - * // emulator service. - * const char* msg = ""; - * if (write(fd, msg, strlen(msg)+1) < 0) { - * ... could not connect to service - * close(fd); - * } - * - * // after this, simply read() and write() to communicate with the - * // service. Exact protocol details left as an exercise to the reader. - * - * This driver is very fast because it doesn't copy any data through - * intermediate buffers, since the emulator is capable of translating - * guest user addresses into host ones. - * - * Note that we must however ensure that each user page involved in the - * exchange is properly mapped during a transfer. +/* This source file contains the implementation of the legacy version of + * a goldfish pipe device driver. See goldfish_pipe_v2.c for the current + * version. */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include "goldfish_pipe.h" /* * IMPORTANT: The following constants must match the ones used and defined @@ -110,29 +69,15 @@ #define PIPE_WAKE_READ (1 << 1) /* pipe can now be read from */ #define PIPE_WAKE_WRITE (1 << 2) /* pipe can now be written to */ -struct access_params { - unsigned long channel; - u32 size; - unsigned long address; - u32 cmd; - u32 result; - /* reserved for future extension */ - u32 flags; -}; +#define MAX_PAGES_TO_GRAB 32 -/* The global driver data. Holds a reference to the i/o page used to - * communicate with the emulator, and a wake queue for blocked tasks - * waiting to be awoken. - */ -struct goldfish_pipe_dev { - spinlock_t lock; - unsigned char __iomem *base; - struct access_params *aps; - int irq; - u32 version; -}; +#define DEBUG 0 -static struct goldfish_pipe_dev pipe_dev[1]; +#if DEBUG +#define DPRINT(...) { printk(KERN_ERR __VA_ARGS__); } +#else +#define DPRINT(...) +#endif /* This data type models a given pipe instance */ struct goldfish_pipe { @@ -142,6 +87,15 @@ struct goldfish_pipe { wait_queue_head_t wake_queue; }; +struct access_params { + unsigned long channel; + u32 size; + unsigned long address; + u32 cmd; + u32 result; + /* reserved for future extension */ + u32 flags; +}; /* Bit flags for the 'flags' field */ enum { @@ -231,8 +185,10 @@ static int setup_access_params_addr(struct platform_device *pdev, if (valid_batchbuffer_addr(dev, aps)) { dev->aps = aps; return 0; - } else + } else { + devm_kfree(&pdev->dev, aps); return -1; + } } /* A value that will not be set by qemu emulator */ @@ -269,6 +225,7 @@ static ssize_t goldfish_pipe_read_write(struct file *filp, char __user *buffer, struct goldfish_pipe *pipe = filp->private_data; struct goldfish_pipe_dev *dev = pipe->dev; unsigned long address, address_end; + struct page* pages[MAX_PAGES_TO_GRAB] = {}; int count = 0, ret = -EINVAL; /* If the emulator already closed the pipe, no need to go further */ @@ -293,45 +250,61 @@ static ssize_t goldfish_pipe_read_write(struct file *filp, char __user *buffer, while (address < address_end) { unsigned long page_end = (address & PAGE_MASK) + PAGE_SIZE; - unsigned long next = page_end < address_end ? page_end - : address_end; - unsigned long avail = next - address; - int status, wakeBit; - struct page *page; - - /* Either vaddr or paddr depending on the device version */ - unsigned long xaddr; + unsigned long next, avail; + int status, wakeBit, page_i, num_contiguous_pages; + long first_page, last_page, requested_pages; + unsigned long xaddr, xaddr_prev, xaddr_i; /* - * We grab the pages on a page-by-page basis in case user - * space gives us a potentially huge buffer but the read only - * returns a small amount, then there's no need to pin that - * much memory to the process. + * Attempt to grab multiple physically contiguous pages. */ - down_read(¤t->mm->mmap_sem); - ret = get_user_pages(address, 1, is_write ? 0 : FOLL_WRITE, - &page, NULL); - up_read(¤t->mm->mmap_sem); - if (ret < 0) - break; + first_page = address & PAGE_MASK; + last_page = (address_end - 1) & PAGE_MASK; + requested_pages = ((last_page - first_page) >> PAGE_SHIFT) + 1; + if (requested_pages > MAX_PAGES_TO_GRAB) { + requested_pages = MAX_PAGES_TO_GRAB; + } + ret = get_user_pages_fast(first_page, requested_pages, + !is_write, pages); + + DPRINT("%s: requested pages: %d %d %p\n", __FUNCTION__, + ret, requested_pages, first_page); + if (ret == 0) { + DPRINT("%s: error: (requested pages == 0) (wanted %d)\n", + __FUNCTION__, requested_pages); + mutex_unlock(&pipe->lock); + return ret; + } + if (ret < 0) { + DPRINT("%s: (requested pages < 0) %d \n", + __FUNCTION__, requested_pages); + mutex_unlock(&pipe->lock); + return ret; + } - if (dev->version) { - /* Device version 1 or newer (qemu-android) expects the - * physical address. - */ - xaddr = page_to_phys(page) | (address & ~PAGE_MASK); - } else { - /* Device version 0 (classic emulator) expects the - * virtual address. - */ - xaddr = address; + xaddr = page_to_phys(pages[0]) | (address & ~PAGE_MASK); + xaddr_prev = xaddr; + num_contiguous_pages = ret == 0 ? 0 : 1; + for (page_i = 1; page_i < ret; page_i++) { + xaddr_i = page_to_phys(pages[page_i]) | (address & ~PAGE_MASK); + if (xaddr_i == xaddr_prev + PAGE_SIZE) { + page_end += PAGE_SIZE; + xaddr_prev = xaddr_i; + num_contiguous_pages++; + } else { + DPRINT("%s: discontinuous page boundary: %d pages instead\n", + __FUNCTION__, page_i); + break; + } } + next = page_end < address_end ? page_end : address_end; + avail = next - address; /* Now, try to transfer the bytes in the current page */ spin_lock_irqsave(&dev->lock, irq_flags); if (access_with_param(dev, - is_write ? CMD_WRITE_BUFFER : CMD_READ_BUFFER, - xaddr, avail, pipe, &status)) { + is_write ? CMD_WRITE_BUFFER : CMD_READ_BUFFER, + xaddr, avail, pipe, &status)) { gf_write_ptr(pipe, dev->base + PIPE_REG_CHANNEL, dev->base + PIPE_REG_CHANNEL_HIGH); writel(avail, dev->base + PIPE_REG_SIZE); @@ -344,9 +317,13 @@ static ssize_t goldfish_pipe_read_write(struct file *filp, char __user *buffer, } spin_unlock_irqrestore(&dev->lock, irq_flags); - if (status > 0 && !is_write) - set_page_dirty(page); - put_page(page); + for (page_i = 0; page_i < ret; page_i++) { + if (status > 0 && !is_write && + page_i < num_contiguous_pages) { + set_page_dirty(pages[page_i]); + } + put_page(pages[page_i]); + } if (status > 0) { /* Correct transfer */ count += status; @@ -368,7 +345,7 @@ static ssize_t goldfish_pipe_read_write(struct file *filp, char __user *buffer, */ if (status != PIPE_ERROR_AGAIN) pr_info_ratelimited("goldfish_pipe: backend returned error %d on %s\n", - status, is_write ? "write" : "read"); + status, is_write ? "write" : "read"); ret = 0; break; } @@ -378,7 +355,7 @@ static ssize_t goldfish_pipe_read_write(struct file *filp, char __user *buffer, * non-blocking mode, just return the error code. */ if (status != PIPE_ERROR_AGAIN || - (filp->f_flags & O_NONBLOCK) != 0) { + (filp->f_flags & O_NONBLOCK) != 0) { ret = goldfish_pipe_error_convert(status); break; } @@ -392,7 +369,7 @@ static ssize_t goldfish_pipe_read_write(struct file *filp, char __user *buffer, /* Tell the emulator we're going to wait for a wake event */ goldfish_cmd(pipe, - is_write ? CMD_WAKE_ON_WRITE : CMD_WAKE_ON_READ); + is_write ? CMD_WAKE_ON_WRITE : CMD_WAKE_ON_READ); /* Unlock the pipe, then wait for the wake signal */ mutex_unlock(&pipe->lock); @@ -538,6 +515,8 @@ static int goldfish_pipe_open(struct inode *inode, struct file *file) pipe->dev = dev; mutex_init(&pipe->lock); + DPRINT("%s: call. pipe_dev pipe_dev=0x%lx new_pipe_addr=0x%lx file=0x%lx\n", __FUNCTION__, pipe_dev, pipe, file); + // spin lock init, write head of list, i guess init_waitqueue_head(&pipe->wake_queue); /* @@ -560,6 +539,7 @@ static int goldfish_pipe_release(struct inode *inode, struct file *filp) { struct goldfish_pipe *pipe = filp->private_data; + DPRINT("%s: call. pipe=0x%lx file=0x%lx\n", __FUNCTION__, pipe, filp); /* The guest is closing the channel, so tell the emulator right now */ goldfish_cmd(pipe, CMD_CLOSE); kfree(pipe); @@ -576,98 +556,33 @@ static const struct file_operations goldfish_pipe_fops = { .release = goldfish_pipe_release, }; -static struct miscdevice goldfish_pipe_device = { +static struct miscdevice goldfish_pipe_dev = { .minor = MISC_DYNAMIC_MINOR, .name = "goldfish_pipe", .fops = &goldfish_pipe_fops, }; -static int goldfish_pipe_probe(struct platform_device *pdev) +int goldfish_pipe_device_init_v1(struct platform_device *pdev) { - int err; - struct resource *r; struct goldfish_pipe_dev *dev = pipe_dev; - - /* not thread safe, but this should not happen */ - WARN_ON(dev->base != NULL); - - spin_lock_init(&dev->lock); - - r = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (r == NULL || resource_size(r) < PAGE_SIZE) { - dev_err(&pdev->dev, "can't allocate i/o page\n"); - return -EINVAL; - } - dev->base = devm_ioremap(&pdev->dev, r->start, PAGE_SIZE); - if (dev->base == NULL) { - dev_err(&pdev->dev, "ioremap failed\n"); - return -EINVAL; - } - - r = platform_get_resource(pdev, IORESOURCE_IRQ, 0); - if (r == NULL) { - err = -EINVAL; - goto error; - } - dev->irq = r->start; - - err = devm_request_irq(&pdev->dev, dev->irq, goldfish_pipe_interrupt, + int err = devm_request_irq(&pdev->dev, dev->irq, goldfish_pipe_interrupt, IRQF_SHARED, "goldfish_pipe", dev); if (err) { - dev_err(&pdev->dev, "unable to allocate IRQ\n"); - goto error; + dev_err(&pdev->dev, "unable to allocate IRQ for v1\n"); + return err; } - err = misc_register(&goldfish_pipe_device); + err = misc_register(&goldfish_pipe_dev); if (err) { - dev_err(&pdev->dev, "unable to register device\n"); - goto error; + dev_err(&pdev->dev, "unable to register v1 device\n"); + return err; } - setup_access_params_addr(pdev, dev); - /* Although the pipe device in the classic Android emulator does not - * recognize the 'version' register, it won't treat this as an error - * either and will simply return 0, which is fine. - */ - dev->version = readl(dev->base + PIPE_REG_VERSION); + setup_access_params_addr(pdev, dev); return 0; - -error: - dev->base = NULL; - return err; } -static int goldfish_pipe_remove(struct platform_device *pdev) +void goldfish_pipe_device_deinit_v1(struct platform_device *pdev) { - struct goldfish_pipe_dev *dev = pipe_dev; - misc_deregister(&goldfish_pipe_device); - dev->base = NULL; - return 0; + misc_deregister(&goldfish_pipe_dev); } - -static const struct acpi_device_id goldfish_pipe_acpi_match[] = { - { "GFSH0003", 0 }, - { }, -}; -MODULE_DEVICE_TABLE(acpi, goldfish_pipe_acpi_match); - -static const struct of_device_id goldfish_pipe_of_match[] = { - { .compatible = "google,android-pipe", }, - {}, -}; -MODULE_DEVICE_TABLE(of, goldfish_pipe_of_match); - -static struct platform_driver goldfish_pipe = { - .probe = goldfish_pipe_probe, - .remove = goldfish_pipe_remove, - .driver = { - .name = "goldfish_pipe", - .owner = THIS_MODULE, - .of_match_table = goldfish_pipe_of_match, - .acpi_match_table = ACPI_PTR(goldfish_pipe_acpi_match), - } -}; - -module_platform_driver(goldfish_pipe); -MODULE_AUTHOR("David Turner "); -MODULE_LICENSE("GPL"); diff --git a/drivers/platform/goldfish/goldfish_pipe.h b/drivers/platform/goldfish/goldfish_pipe.h new file mode 100644 index 00000000000000..6cd1b63be8c907 --- /dev/null +++ b/drivers/platform/goldfish/goldfish_pipe.h @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2016 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ +#ifndef GOLDFISH_PIPE_H +#define GOLDFISH_PIPE_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +/* Initialize the legacy version of the pipe device driver */ +int goldfish_pipe_device_init_v1(struct platform_device *pdev); + +/* Deinitialize the legacy version of the pipe device driver */ +void goldfish_pipe_device_deinit_v1(struct platform_device *pdev); + +/* Forward declarations for the device struct */ +struct goldfish_pipe; +struct goldfish_pipe_device_buffers; + +/* The global driver data. Holds a reference to the i/o page used to + * communicate with the emulator, and a wake queue for blocked tasks + * waiting to be awoken. + */ +struct goldfish_pipe_dev { + /* + * Global device spinlock. Protects the following members: + * - pipes, pipes_capacity + * - [*pipes, *pipes + pipes_capacity) - array data + * - first_signalled_pipe, + * goldfish_pipe::prev_signalled, + * goldfish_pipe::next_signalled, + * goldfish_pipe::signalled_flags - all singnalled-related fields, + * in all allocated pipes + * - open_command_params - PIPE_CMD_OPEN-related buffers + * + * It looks like a lot of different fields, but the trick is that the only + * operation that happens often is the signalled pipes array manipulation. + * That's why it's OK for now to keep the rest of the fields under the same + * lock. If we notice too much contention because of PIPE_CMD_OPEN, + * then we should add a separate lock there. + */ + spinlock_t lock; + + /* + * Array of the pipes of |pipes_capacity| elements, + * indexed by goldfish_pipe::id + */ + struct goldfish_pipe **pipes; + u32 pipes_capacity; + + /* Pointers to the buffers host uses for interaction with this driver */ + struct goldfish_pipe_dev_buffers *buffers; + + /* Head of a doubly linked list of signalled pipes */ + struct goldfish_pipe *first_signalled_pipe; + + /* Some device-specific data */ + int irq; + int version; + unsigned char __iomem *base; + + /* v1-specific access parameters */ + struct access_params *aps; +}; + +extern struct goldfish_pipe_dev pipe_dev[1]; + +#endif /* GOLDFISH_PIPE_H */ diff --git a/drivers/platform/goldfish/goldfish_pipe_v2.c b/drivers/platform/goldfish/goldfish_pipe_v2.c new file mode 100644 index 00000000000000..ad373ed3655523 --- /dev/null +++ b/drivers/platform/goldfish/goldfish_pipe_v2.c @@ -0,0 +1,889 @@ +/* + * Copyright (C) 2012 Intel, Inc. + * Copyright (C) 2013 Intel, Inc. + * Copyright (C) 2014 Linaro Limited + * Copyright (C) 2011-2016 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +/* This source file contains the implementation of a special device driver + * that intends to provide a *very* fast communication channel between the + * guest system and the QEMU emulator. + * + * Usage from the guest is simply the following (error handling simplified): + * + * int fd = open("/dev/qemu_pipe",O_RDWR); + * .... write() or read() through the pipe. + * + * This driver doesn't deal with the exact protocol used during the session. + * It is intended to be as simple as something like: + * + * // do this _just_ after opening the fd to connect to a specific + * // emulator service. + * const char* msg = ""; + * if (write(fd, msg, strlen(msg)+1) < 0) { + * ... could not connect to service + * close(fd); + * } + * + * // after this, simply read() and write() to communicate with the + * // service. Exact protocol details left as an exercise to the reader. + * + * This driver is very fast because it doesn't copy any data through + * intermediate buffers, since the emulator is capable of translating + * guest user addresses into host ones. + * + * Note that we must however ensure that each user page involved in the + * exchange is properly mapped during a transfer. + */ + +#include "goldfish_pipe.h" + + +/* + * Update this when something changes in the driver's behavior so the host + * can benefit from knowing it + */ +enum { + PIPE_DRIVER_VERSION = 2, + PIPE_CURRENT_DEVICE_VERSION = 2 +}; + +/* + * IMPORTANT: The following constants must match the ones used and defined + * in external/qemu/hw/goldfish_pipe.c in the Android source tree. + */ + +/* List of bitflags returned in status of CMD_POLL command */ +enum PipePollFlags { + PIPE_POLL_IN = 1 << 0, + PIPE_POLL_OUT = 1 << 1, + PIPE_POLL_HUP = 1 << 2 +}; + +/* Possible status values used to signal errors - see goldfish_pipe_error_convert */ +enum PipeErrors { + PIPE_ERROR_INVAL = -1, + PIPE_ERROR_AGAIN = -2, + PIPE_ERROR_NOMEM = -3, + PIPE_ERROR_IO = -4 +}; + +/* Bit-flags used to signal events from the emulator */ +enum PipeWakeFlags { + PIPE_WAKE_CLOSED = 1 << 0, /* emulator closed pipe */ + PIPE_WAKE_READ = 1 << 1, /* pipe can now be read from */ + PIPE_WAKE_WRITE = 1 << 2 /* pipe can now be written to */ +}; + +/* Bit flags for the 'flags' field */ +enum PipeFlagsBits { + BIT_CLOSED_ON_HOST = 0, /* pipe closed by host */ + BIT_WAKE_ON_WRITE = 1, /* want to be woken on writes */ + BIT_WAKE_ON_READ = 2, /* want to be woken on reads */ +}; + +enum PipeRegs { + PIPE_REG_CMD = 0, + + PIPE_REG_SIGNAL_BUFFER_HIGH = 4, + PIPE_REG_SIGNAL_BUFFER = 8, + PIPE_REG_SIGNAL_BUFFER_COUNT = 12, + + PIPE_REG_OPEN_BUFFER_HIGH = 20, + PIPE_REG_OPEN_BUFFER = 24, + + PIPE_REG_VERSION = 36, + + PIPE_REG_GET_SIGNALLED = 48, +}; + +enum PipeCmdCode { + PIPE_CMD_OPEN = 1, /* to be used by the pipe device itself */ + PIPE_CMD_CLOSE, + PIPE_CMD_POLL, + PIPE_CMD_WRITE, + PIPE_CMD_WAKE_ON_WRITE, + PIPE_CMD_READ, + PIPE_CMD_WAKE_ON_READ, + + /* + * TODO(zyy): implement a deferred read/write execution to allow parallel + * processing of pipe operations on the host. + */ + PIPE_CMD_WAKE_ON_DONE_IO, +}; + +enum { + MAX_BUFFERS_PER_COMMAND = 336, + MAX_SIGNALLED_PIPES = 64, + INITIAL_PIPES_CAPACITY = 64 +}; + +struct goldfish_pipe_dev; +struct goldfish_pipe; +struct goldfish_pipe_command; + +/* A per-pipe command structure, shared with the host */ +struct goldfish_pipe_command { + s32 cmd; /* PipeCmdCode, guest -> host */ + s32 id; /* pipe id, guest -> host */ + s32 status; /* command execution status, host -> guest */ + s32 reserved; /* to pad to 64-bit boundary */ + union { + /* Parameters for PIPE_CMD_{READ,WRITE} */ + struct { + u32 buffers_count; /* number of buffers, guest -> host */ + s32 consumed_size; /* number of consumed bytes, host -> guest */ + u64 ptrs[MAX_BUFFERS_PER_COMMAND]; /* buffer pointers, guest -> host */ + u32 sizes[MAX_BUFFERS_PER_COMMAND]; /* buffer sizes, guest -> host */ + } rw_params; + }; +}; + +/* A single signalled pipe information */ +struct signalled_pipe_buffer { + u32 id; + u32 flags; +}; + +/* Parameters for the PIPE_CMD_OPEN command */ +struct open_command_param { + u64 command_buffer_ptr; + u32 rw_params_max_count; +}; + +/* Device-level set of buffers shared with the host */ +struct goldfish_pipe_dev_buffers { + struct open_command_param open_command_params; + struct signalled_pipe_buffer signalled_pipe_buffers[MAX_SIGNALLED_PIPES]; +}; + +/* This data type models a given pipe instance */ +struct goldfish_pipe { + u32 id; /* pipe ID - index into goldfish_pipe_dev::pipes array */ + unsigned long flags; /* The wake flags pipe is waiting for + * Note: not protected with any lock, uses atomic operations + * and barriers to make it thread-safe. + */ + unsigned long signalled_flags; /* wake flags host have signalled, + * - protected by goldfish_pipe_dev::lock */ + + struct goldfish_pipe_command *command_buffer; /* A pointer to command buffer */ + + /* doubly linked list of signalled pipes, protected by goldfish_pipe_dev::lock */ + struct goldfish_pipe *prev_signalled; + struct goldfish_pipe *next_signalled; + + /* + * A pipe's own lock. Protects the following: + * - *command_buffer - makes sure a command can safely write its parameters + * to the host and read the results back. + */ + struct mutex lock; + + wait_queue_head_t wake_queue; /* A wake queue for sleeping until host signals an event */ + struct goldfish_pipe_dev *dev; /* Pointer to the parent goldfish_pipe_dev instance */ +}; + +struct goldfish_pipe_dev pipe_dev[1] = {}; + +static int goldfish_cmd_locked(struct goldfish_pipe *pipe, enum PipeCmdCode cmd) +{ + pipe->command_buffer->cmd = cmd; + pipe->command_buffer->status = PIPE_ERROR_INVAL; /* failure by default */ + writel(pipe->id, pipe->dev->base + PIPE_REG_CMD); + return pipe->command_buffer->status; +} + +static int goldfish_cmd(struct goldfish_pipe *pipe, enum PipeCmdCode cmd) +{ + int status; + if (mutex_lock_interruptible(&pipe->lock)) + return PIPE_ERROR_IO; + status = goldfish_cmd_locked(pipe, cmd); + mutex_unlock(&pipe->lock); + return status; +} + +/* + * This function converts an error code returned by the emulator through + * the PIPE_REG_STATUS i/o register into a valid negative errno value. + */ +static int goldfish_pipe_error_convert(int status) +{ + switch (status) { + case PIPE_ERROR_AGAIN: + return -EAGAIN; + case PIPE_ERROR_NOMEM: + return -ENOMEM; + case PIPE_ERROR_IO: + return -EIO; + default: + return -EINVAL; + } +} + +static int pin_user_pages(unsigned long first_page, unsigned long last_page, + unsigned last_page_size, int is_write, + struct page *pages[MAX_BUFFERS_PER_COMMAND], unsigned *iter_last_page_size) +{ + int ret; + int requested_pages = ((last_page - first_page) >> PAGE_SHIFT) + 1; + if (requested_pages > MAX_BUFFERS_PER_COMMAND) { + requested_pages = MAX_BUFFERS_PER_COMMAND; + *iter_last_page_size = PAGE_SIZE; + } else { + *iter_last_page_size = last_page_size; + } + + ret = get_user_pages_fast( + first_page, requested_pages, !is_write, pages); + if (ret <= 0) + return -EFAULT; + if (ret < requested_pages) + *iter_last_page_size = PAGE_SIZE; + return ret; + +} + +static void release_user_pages(struct page **pages, int pages_count, + int is_write, s32 consumed_size) +{ + int i; + for (i = 0; i < pages_count; i++) { + if (!is_write && consumed_size > 0) { + set_page_dirty(pages[i]); + } + put_page(pages[i]); + } +} + +/* Populate the call parameters, merging adjacent pages together */ +static void populate_rw_params( + struct page **pages, int pages_count, + unsigned long address, unsigned long address_end, + unsigned long first_page, unsigned long last_page, + unsigned iter_last_page_size, int is_write, + struct goldfish_pipe_command *command) +{ + /* + * Process the first page separately - it's the only page that + * needs special handling for its start address. + */ + unsigned long xaddr = page_to_phys(pages[0]); + unsigned long xaddr_prev = xaddr; + int buffer_idx = 0; + int i = 1; + int size_on_page = first_page == last_page + ? (int)(address_end - address) + : (PAGE_SIZE - (address & ~PAGE_MASK)); + command->rw_params.ptrs[0] = (u64)(xaddr | (address & ~PAGE_MASK)); + command->rw_params.sizes[0] = size_on_page; + for (; i < pages_count; ++i) { + xaddr = page_to_phys(pages[i]); + size_on_page = (i == pages_count - 1) ? iter_last_page_size : PAGE_SIZE; + if (xaddr == xaddr_prev + PAGE_SIZE) { + command->rw_params.sizes[buffer_idx] += size_on_page; + } else { + ++buffer_idx; + command->rw_params.ptrs[buffer_idx] = (u64)xaddr; + command->rw_params.sizes[buffer_idx] = size_on_page; + } + xaddr_prev = xaddr; + } + command->rw_params.buffers_count = buffer_idx + 1; +} + +static int transfer_max_buffers(struct goldfish_pipe* pipe, + unsigned long address, unsigned long address_end, int is_write, + unsigned long last_page, unsigned int last_page_size, + s32* consumed_size, int* status) +{ + struct page *pages[MAX_BUFFERS_PER_COMMAND]; + unsigned long first_page = address & PAGE_MASK; + unsigned int iter_last_page_size; + int pages_count = pin_user_pages(first_page, last_page, + last_page_size, is_write, + pages, &iter_last_page_size); + if (pages_count < 0) + return pages_count; + + /* Serialize access to the pipe command buffers */ + if (mutex_lock_interruptible(&pipe->lock)) + return -ERESTARTSYS; + + populate_rw_params(pages, pages_count, address, address_end, + first_page, last_page, iter_last_page_size, is_write, + pipe->command_buffer); + + /* Transfer the data */ + *status = goldfish_cmd_locked(pipe, + is_write ? PIPE_CMD_WRITE : PIPE_CMD_READ); + + *consumed_size = pipe->command_buffer->rw_params.consumed_size; + + mutex_unlock(&pipe->lock); + + release_user_pages(pages, pages_count, is_write, *consumed_size); + + return 0; +} + +static int wait_for_host_signal(struct goldfish_pipe *pipe, int is_write) +{ + u32 wakeBit = is_write ? BIT_WAKE_ON_WRITE : BIT_WAKE_ON_READ; + set_bit(wakeBit, &pipe->flags); + + /* Tell the emulator we're going to wait for a wake event */ + (void)goldfish_cmd(pipe, + is_write ? PIPE_CMD_WAKE_ON_WRITE : PIPE_CMD_WAKE_ON_READ); + + while (test_bit(wakeBit, &pipe->flags)) { + if (wait_event_interruptible( + pipe->wake_queue, + !test_bit(wakeBit, &pipe->flags))) + return -ERESTARTSYS; + + if (test_bit(BIT_CLOSED_ON_HOST, &pipe->flags)) + return -EIO; + } + + return 0; +} + +static ssize_t goldfish_pipe_read_write(struct file *filp, + char __user *buffer, size_t bufflen, int is_write) +{ + struct goldfish_pipe *pipe = filp->private_data; + int count = 0, ret = -EINVAL; + unsigned long address, address_end, last_page; + unsigned int last_page_size; + + /* If the emulator already closed the pipe, no need to go further */ + if (unlikely(test_bit(BIT_CLOSED_ON_HOST, &pipe->flags))) + return -EIO; + /* Null reads or writes succeeds */ + if (unlikely(bufflen == 0)) + return 0; + /* Check the buffer range for access */ + if (unlikely(!access_ok(is_write ? VERIFY_WRITE : VERIFY_READ, + buffer, bufflen))) + return -EFAULT; + + address = (unsigned long)buffer; + address_end = address + bufflen; + last_page = (address_end - 1) & PAGE_MASK; + last_page_size = ((address_end - 1) & ~PAGE_MASK) + 1; + + while (address < address_end) { + s32 consumed_size; + int status; + ret = transfer_max_buffers(pipe, address, address_end, is_write, + last_page, last_page_size, &consumed_size, &status); + if (ret < 0) + break; + + if (consumed_size > 0) { + /* No matter what's the status, we've transfered something */ + count += consumed_size; + address += consumed_size; + } + if (status > 0) + continue; + if (status == 0) { + /* EOF */ + ret = 0; + break; + } + if (count > 0) { + /* + * An error occured, but we already transfered + * something on one of the previous iterations. + * Just return what we already copied and log this + * err. + */ + if (status != PIPE_ERROR_AGAIN) + pr_info_ratelimited("goldfish_pipe: backend error %d on %s\n", + status, is_write ? "write" : "read"); + break; + } + + /* + * If the error is not PIPE_ERROR_AGAIN, or if we are in + * non-blocking mode, just return the error code. + */ + if (status != PIPE_ERROR_AGAIN || (filp->f_flags & O_NONBLOCK) != 0) { + ret = goldfish_pipe_error_convert(status); + break; + } + + status = wait_for_host_signal(pipe, is_write); + if (status < 0) + return status; + } + + if (count > 0) + return count; + return ret; +} + +static ssize_t goldfish_pipe_read(struct file *filp, char __user *buffer, + size_t bufflen, loff_t *ppos) +{ + return goldfish_pipe_read_write(filp, buffer, bufflen, /* is_write */ 0); +} + +static ssize_t goldfish_pipe_write(struct file *filp, + const char __user *buffer, size_t bufflen, + loff_t *ppos) +{ + return goldfish_pipe_read_write(filp, + /* cast away the const */(char __user *)buffer, bufflen, + /* is_write */ 1); +} + +static unsigned int goldfish_pipe_poll(struct file *filp, poll_table *wait) +{ + struct goldfish_pipe *pipe = filp->private_data; + unsigned int mask = 0; + int status; + + poll_wait(filp, &pipe->wake_queue, wait); + + status = goldfish_cmd(pipe, PIPE_CMD_POLL); + if (status < 0) { + return -ERESTARTSYS; + } + + if (status & PIPE_POLL_IN) + mask |= POLLIN | POLLRDNORM; + if (status & PIPE_POLL_OUT) + mask |= POLLOUT | POLLWRNORM; + if (status & PIPE_POLL_HUP) + mask |= POLLHUP; + if (test_bit(BIT_CLOSED_ON_HOST, &pipe->flags)) + mask |= POLLERR; + + return mask; +} + +static void signalled_pipes_add_locked(struct goldfish_pipe_dev *dev, + u32 id, u32 flags) +{ + struct goldfish_pipe *pipe; + + BUG_ON(id >= dev->pipes_capacity); + + pipe = dev->pipes[id]; + if (!pipe) + return; + pipe->signalled_flags |= flags; + + if (pipe->prev_signalled || pipe->next_signalled + || dev->first_signalled_pipe == pipe) + return; /* already in the list */ + pipe->next_signalled = dev->first_signalled_pipe; + if (dev->first_signalled_pipe) { + dev->first_signalled_pipe->prev_signalled = pipe; + } + dev->first_signalled_pipe = pipe; +} + +static void signalled_pipes_remove_locked(struct goldfish_pipe_dev *dev, + struct goldfish_pipe *pipe) { + if (pipe->prev_signalled) + pipe->prev_signalled->next_signalled = pipe->next_signalled; + if (pipe->next_signalled) + pipe->next_signalled->prev_signalled = pipe->prev_signalled; + if (pipe == dev->first_signalled_pipe) + dev->first_signalled_pipe = pipe->next_signalled; + pipe->prev_signalled = NULL; + pipe->next_signalled = NULL; +} + +static struct goldfish_pipe *signalled_pipes_pop_front(struct goldfish_pipe_dev *dev, + int *wakes) +{ + struct goldfish_pipe *pipe; + unsigned long flags; + spin_lock_irqsave(&dev->lock, flags); + + pipe = dev->first_signalled_pipe; + if (pipe) { + *wakes = pipe->signalled_flags; + pipe->signalled_flags = 0; + /* + * This is an optimized version of signalled_pipes_remove_locked() - + * we want to make it as fast as possible to wake the sleeping pipe + * operations faster + */ + dev->first_signalled_pipe = pipe->next_signalled; + if (dev->first_signalled_pipe) + dev->first_signalled_pipe->prev_signalled = NULL; + pipe->next_signalled = NULL; + } + + spin_unlock_irqrestore(&dev->lock, flags); + return pipe; +} + +static void goldfish_interrupt_task(unsigned long unused) +{ + struct goldfish_pipe_dev *dev = pipe_dev; + /* Iterate over the signalled pipes and wake them one by one */ + struct goldfish_pipe *pipe; + int wakes; + while ((pipe = signalled_pipes_pop_front(dev, &wakes)) != NULL) { + if (wakes & PIPE_WAKE_CLOSED) { + pipe->flags = 1 << BIT_CLOSED_ON_HOST; + } else { + if (wakes & PIPE_WAKE_READ) + clear_bit(BIT_WAKE_ON_READ, &pipe->flags); + if (wakes & PIPE_WAKE_WRITE) + clear_bit(BIT_WAKE_ON_WRITE, &pipe->flags); + } + /* + * wake_up_interruptible() implies a write barrier, so don't explicitly + * add another one here. + */ + wake_up_interruptible(&pipe->wake_queue); + } +} +DECLARE_TASKLET(goldfish_interrupt_tasklet, goldfish_interrupt_task, 0); + +/* + * The general idea of the interrupt handling: + * + * 1. device raises an interrupt if there's at least one signalled pipe + * 2. IRQ handler reads the signalled pipes and their count from the device + * 3. device writes them into a shared buffer and returns the count + * it only resets the IRQ if it has returned all signalled pipes, + * otherwise it leaves it raised, so IRQ handler will be called + * again for the next chunk + * 4. IRQ handler adds all returned pipes to the device's signalled pipes list + * 5. IRQ handler launches a tasklet to process the signalled pipes from the + * list in a separate context + */ +static irqreturn_t goldfish_pipe_interrupt(int irq, void *dev_id) +{ + u32 count; + u32 i; + unsigned long flags; + struct goldfish_pipe_dev *dev = dev_id; + if (dev != pipe_dev) + return IRQ_NONE; + + /* Request the signalled pipes from the device */ + spin_lock_irqsave(&dev->lock, flags); + + count = readl(dev->base + PIPE_REG_GET_SIGNALLED); + if (count == 0) { + spin_unlock_irqrestore(&dev->lock, flags); + return IRQ_NONE; + } + if (count > MAX_SIGNALLED_PIPES) + count = MAX_SIGNALLED_PIPES; + + for (i = 0; i < count; ++i) + signalled_pipes_add_locked(dev, + dev->buffers->signalled_pipe_buffers[i].id, + dev->buffers->signalled_pipe_buffers[i].flags); + + spin_unlock_irqrestore(&dev->lock, flags); + + tasklet_schedule(&goldfish_interrupt_tasklet); + return IRQ_HANDLED; +} + +static int get_free_pipe_id_locked(struct goldfish_pipe_dev *dev) +{ + int id; + for (id = 0; id < dev->pipes_capacity; ++id) + if (!dev->pipes[id]) + return id; + + { + /* Reallocate the array */ + u32 new_capacity = 2 * dev->pipes_capacity; + struct goldfish_pipe **pipes = + kcalloc(new_capacity, sizeof(*pipes), + GFP_ATOMIC); + if (!pipes) + return -ENOMEM; + memcpy(pipes, dev->pipes, sizeof(*pipes) * dev->pipes_capacity); + kfree(dev->pipes); + dev->pipes = pipes; + id = dev->pipes_capacity; + dev->pipes_capacity = new_capacity; + } + return id; +} + +/** + * goldfish_pipe_open - open a channel to the AVD + * @inode: inode of device + * @file: file struct of opener + * + * Create a new pipe link between the emulator and the use application. + * Each new request produces a new pipe. + * + * Note: we use the pipe ID as a mux. All goldfish emulations are 32bit + * right now so this is fine. A move to 64bit will need this addressing + */ +static int goldfish_pipe_open(struct inode *inode, struct file *file) +{ + struct goldfish_pipe_dev *dev = pipe_dev; + unsigned long flags; + int id; + int status; + + /* Allocate new pipe kernel object */ + struct goldfish_pipe *pipe = kzalloc(sizeof(*pipe), GFP_KERNEL); + if (pipe == NULL) + return -ENOMEM; + + pipe->dev = dev; + mutex_init(&pipe->lock); + init_waitqueue_head(&pipe->wake_queue); + + /* + * Command buffer needs to be allocated on its own page to make sure it is + * physically contiguous in host's address space. + */ + pipe->command_buffer = + (struct goldfish_pipe_command*)__get_free_page(GFP_KERNEL); + if (!pipe->command_buffer) { + status = -ENOMEM; + goto err_pipe; + } + + spin_lock_irqsave(&dev->lock, flags); + + id = get_free_pipe_id_locked(dev); + if (id < 0) { + status = id; + goto err_id_locked; + } + + dev->pipes[id] = pipe; + pipe->id = id; + pipe->command_buffer->id = id; + + /* Now tell the emulator we're opening a new pipe. */ + dev->buffers->open_command_params.rw_params_max_count = + MAX_BUFFERS_PER_COMMAND; + dev->buffers->open_command_params.command_buffer_ptr = + (u64)(unsigned long)__pa(pipe->command_buffer); + status = goldfish_cmd_locked(pipe, PIPE_CMD_OPEN); + spin_unlock_irqrestore(&dev->lock, flags); + if (status < 0) + goto err_cmd; + /* All is done, save the pipe into the file's private data field */ + file->private_data = pipe; + return 0; + +err_cmd: + spin_lock_irqsave(&dev->lock, flags); + dev->pipes[id] = NULL; +err_id_locked: + spin_unlock_irqrestore(&dev->lock, flags); + free_page((unsigned long)pipe->command_buffer); +err_pipe: + kfree(pipe); + return status; +} + +static int goldfish_pipe_release(struct inode *inode, struct file *filp) +{ + unsigned long flags; + struct goldfish_pipe *pipe = filp->private_data; + struct goldfish_pipe_dev *dev = pipe->dev; + + /* The guest is closing the channel, so tell the emulator right now */ + (void)goldfish_cmd(pipe, PIPE_CMD_CLOSE); + + spin_lock_irqsave(&dev->lock, flags); + dev->pipes[pipe->id] = NULL; + signalled_pipes_remove_locked(dev, pipe); + spin_unlock_irqrestore(&dev->lock, flags); + + filp->private_data = NULL; + free_page((unsigned long)pipe->command_buffer); + kfree(pipe); + return 0; +} + +static const struct file_operations goldfish_pipe_fops = { + .owner = THIS_MODULE, + .read = goldfish_pipe_read, + .write = goldfish_pipe_write, + .poll = goldfish_pipe_poll, + .open = goldfish_pipe_open, + .release = goldfish_pipe_release, +}; + +static struct miscdevice goldfish_pipe_dev = { + .minor = MISC_DYNAMIC_MINOR, + .name = "goldfish_pipe", + .fops = &goldfish_pipe_fops, +}; + +static int goldfish_pipe_device_init_v2(struct platform_device *pdev) +{ + char *page; + struct goldfish_pipe_dev *dev = pipe_dev; + int err = devm_request_irq(&pdev->dev, dev->irq, goldfish_pipe_interrupt, + IRQF_SHARED, "goldfish_pipe", dev); + if (err) { + dev_err(&pdev->dev, "unable to allocate IRQ for v2\n"); + return err; + } + + err = misc_register(&goldfish_pipe_dev); + if (err) { + dev_err(&pdev->dev, "unable to register v2 device\n"); + return err; + } + + dev->first_signalled_pipe = NULL; + dev->pipes_capacity = INITIAL_PIPES_CAPACITY; + dev->pipes = kcalloc(dev->pipes_capacity, sizeof(*dev->pipes), GFP_KERNEL); + if (!dev->pipes) + return -ENOMEM; + + /* + * We're going to pass two buffers, open_command_params and + * signalled_pipe_buffers, to the host. This means each of those buffers + * needs to be contained in a single physical page. The easiest choice is + * to just allocate a page and place the buffers in it. + */ + BUG_ON(sizeof(*dev->buffers) > PAGE_SIZE); + page = (char*)__get_free_page(GFP_KERNEL); + if (!page) { + kfree(dev->pipes); + return -ENOMEM; + } + dev->buffers = (struct goldfish_pipe_dev_buffers*)page; + + /* Send the buffer addresses to the host */ + { + u64 paddr = __pa(&dev->buffers->signalled_pipe_buffers); + writel((u32)(unsigned long)(paddr >> 32), dev->base + PIPE_REG_SIGNAL_BUFFER_HIGH); + writel((u32)(unsigned long)paddr, dev->base + PIPE_REG_SIGNAL_BUFFER); + writel((u32)MAX_SIGNALLED_PIPES, dev->base + PIPE_REG_SIGNAL_BUFFER_COUNT); + + paddr = __pa(&dev->buffers->open_command_params); + writel((u32)(unsigned long)(paddr >> 32), dev->base + PIPE_REG_OPEN_BUFFER_HIGH); + writel((u32)(unsigned long)paddr, dev->base + PIPE_REG_OPEN_BUFFER); + } + return 0; +} + +static void goldfish_pipe_device_deinit_v2(struct platform_device *pdev) { + struct goldfish_pipe_dev *dev = pipe_dev; + misc_deregister(&goldfish_pipe_dev); + kfree(dev->pipes); + free_page((unsigned long)dev->buffers); +} + +static int goldfish_pipe_probe(struct platform_device *pdev) +{ + int err; + struct resource *r; + struct goldfish_pipe_dev *dev = pipe_dev; + + BUG_ON(sizeof(struct goldfish_pipe_command) > PAGE_SIZE); + + /* not thread safe, but this should not happen */ + WARN_ON(dev->base != NULL); + + spin_lock_init(&dev->lock); + + r = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (r == NULL || resource_size(r) < PAGE_SIZE) { + dev_err(&pdev->dev, "can't allocate i/o page\n"); + return -EINVAL; + } + dev->base = devm_ioremap(&pdev->dev, r->start, PAGE_SIZE); + if (dev->base == NULL) { + dev_err(&pdev->dev, "ioremap failed\n"); + return -EINVAL; + } + + r = platform_get_resource(pdev, IORESOURCE_IRQ, 0); + if (r == NULL) { + err = -EINVAL; + goto error; + } + dev->irq = r->start; + + /* + * Exchange the versions with the host device + * + * Note: v1 driver used to not report its version, so we write it before + * reading device version back: this allows the host implementation to + * detect the old driver (if there was no version write before read). + */ + writel((u32)PIPE_DRIVER_VERSION, dev->base + PIPE_REG_VERSION); + dev->version = readl(dev->base + PIPE_REG_VERSION); + if (dev->version < PIPE_CURRENT_DEVICE_VERSION) { + /* initialize the old device version */ + err = goldfish_pipe_device_init_v1(pdev); + } else { + /* Host device supports the new interface */ + err = goldfish_pipe_device_init_v2(pdev); + } + if (!err) + return 0; + +error: + dev->base = NULL; + return err; +} + +static int goldfish_pipe_remove(struct platform_device *pdev) +{ + struct goldfish_pipe_dev *dev = pipe_dev; + if (dev->version < PIPE_CURRENT_DEVICE_VERSION) + goldfish_pipe_device_deinit_v1(pdev); + else + goldfish_pipe_device_deinit_v2(pdev); + dev->base = NULL; + return 0; +} + +static const struct acpi_device_id goldfish_pipe_acpi_match[] = { + { "GFSH0003", 0 }, + { }, +}; +MODULE_DEVICE_TABLE(acpi, goldfish_pipe_acpi_match); + +static const struct of_device_id goldfish_pipe_of_match[] = { + { .compatible = "google,android-pipe", }, + {}, +}; +MODULE_DEVICE_TABLE(of, goldfish_pipe_of_match); + +static struct platform_driver goldfish_pipe_driver = { + .probe = goldfish_pipe_probe, + .remove = goldfish_pipe_remove, + .driver = { + .name = "goldfish_pipe", + .of_match_table = goldfish_pipe_of_match, + .acpi_match_table = ACPI_PTR(goldfish_pipe_acpi_match), + } +}; + +module_platform_driver(goldfish_pipe_driver); +MODULE_AUTHOR("David Turner "); +MODULE_LICENSE("GPL"); diff --git a/drivers/platform/hisi/Kconfig b/drivers/platform/hisi/Kconfig new file mode 100644 index 00000000000000..776cfe0cc2af5c --- /dev/null +++ b/drivers/platform/hisi/Kconfig @@ -0,0 +1,6 @@ +config HISI_FIQ_DEBUGGER + bool "Enable the FIQ serial debugger on Hisi" + default n + select FIQ_DEBUGGER + help + Enables the FIQ serial debugger on Hisi diff --git a/drivers/platform/hisi/Makefile b/drivers/platform/hisi/Makefile new file mode 100644 index 00000000000000..483c227b18bc42 --- /dev/null +++ b/drivers/platform/hisi/Makefile @@ -0,0 +1,2 @@ +obj-$(CONFIG_HISI_FIQ_DEBUGGER) += hi6220_fiq_debugger.o + diff --git a/drivers/platform/hisi/hi6220_fiq_debugger.c b/drivers/platform/hisi/hi6220_fiq_debugger.c new file mode 100644 index 00000000000000..12bb5ece4156c4 --- /dev/null +++ b/drivers/platform/hisi/hi6220_fiq_debugger.c @@ -0,0 +1,260 @@ +/* + * platform/hisilicon/hi6220_fiq_debugger.c + * + * Serial Debugger Interface for Hi6220 + * + * Copyright (C) 2016 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../staging/android/fiq_debugger/fiq_debugger.h" + +struct hi6220_fiq_debugger { + struct fiq_debugger_pdata pdata; + void __iomem *debug_port_base; +}; + +static inline void uart_write(struct hi6220_fiq_debugger *t, + unsigned int val, unsigned int off) +{ + __raw_writew(val, t->debug_port_base + off); +} + +static inline unsigned int uart_read(struct hi6220_fiq_debugger *t, + unsigned int off) +{ + return __raw_readw(t->debug_port_base + off); +} + +static int debug_port_init(struct platform_device *pdev) +{ + struct hi6220_fiq_debugger *t = + container_of(dev_get_platdata(&pdev->dev), typeof(*t), pdata); + + uart_write(t, UART011_OEIS | UART011_BEIS | UART011_PEIS | + UART011_FEIS | UART011_RTIS | UART011_RXIS, UART011_ICR); + uart_write(t, UART011_RTIM | UART011_RXIM, UART011_IMSC); + + /* uart_write(t, 0x1b, UART011_FBRD); */ + + return 0; +} + +static int debug_getc(struct platform_device *pdev) +{ + int ch; + struct hi6220_fiq_debugger *t = + container_of(dev_get_platdata(&pdev->dev), typeof(*t), pdata); + + if (uart_read(t, UART01x_FR) & UART01x_FR_RXFE) + return FIQ_DEBUGGER_NO_CHAR; + ch = uart_read(t, UART01x_DR); + if (ch & UART011_DR_BE) + return FIQ_DEBUGGER_BREAK; + return ch & 0x00FF; +} + +static void debug_putc(struct platform_device *pdev, unsigned int c) +{ + struct hi6220_fiq_debugger *t = + container_of(dev_get_platdata(&pdev->dev), typeof(*t), pdata); + + while (uart_read(t, UART01x_FR) & UART01x_FR_TXFF) + cpu_relax(); + uart_write(t, c, UART01x_DR); + while (uart_read(t, UART01x_FR) & UART01x_FR_BUSY) + cpu_relax(); +} + +static void debug_flush(struct platform_device *pdev) +{ + struct hi6220_fiq_debugger *t = + container_of(dev_get_platdata(&pdev->dev), typeof(*t), pdata); + + while (uart_read(t, UART01x_FR) & UART01x_FR_BUSY) + cpu_relax(); +} + +static int debug_suspend(struct platform_device *pdev) +{ + return 0; +} + +static int debug_resume(struct platform_device *pdev) +{ + return debug_port_init(pdev); +} + +static int hi6220_fiq_debugger_id; + +static int __hi6220_serial_debug_init(unsigned int base, int fiq, int irq, + struct clk *clk, int signal_irq, int wakeup_irq) +{ + struct hi6220_fiq_debugger *t; + struct platform_device *pdev; + struct resource *res; + int res_count = 1; + int ret = -ENOMEM; + + t = kzalloc(sizeof(struct hi6220_fiq_debugger), GFP_KERNEL); + if (!t) { + pr_err("Failed to allocate for fiq debugger\n"); + return ret; + } + + t->pdata.uart_init = debug_port_init; + t->pdata.uart_getc = debug_getc; + t->pdata.uart_putc = debug_putc; + t->pdata.uart_flush = debug_flush; + t->pdata.uart_dev_suspend = debug_suspend; + t->pdata.uart_dev_resume = debug_resume; + + t->debug_port_base = ioremap(base, PAGE_SIZE); + if (!t->debug_port_base) { + pr_err("Failed to ioremap for fiq debugger\n"); + goto out1; + } + + res = kzalloc(sizeof(struct resource) * 3, GFP_KERNEL); + if (!res) { + pr_err("Failed to alloc fiq debugger resources\n"); + goto out2; + } + + pdev = kzalloc(sizeof(struct platform_device), GFP_KERNEL); + if (!pdev) { + pr_err("Failed to alloc fiq debugger platform device\n"); + goto out3; + }; + + if (fiq >= 0) { + res[0].flags = IORESOURCE_IRQ; + res[0].start = fiq; + res[0].end = fiq; + res[0].name = "fiq"; + } else { + res[0].flags = IORESOURCE_IRQ; + res[0].start = irq; + res[0].end = irq; + res[0].name = "uart_irq"; + } + + if (signal_irq >= 0) { + res[1].flags = IORESOURCE_IRQ; + res[1].start = signal_irq; + res[1].end = signal_irq; + res[1].name = "signal"; + res_count++; + } + + if (wakeup_irq >= 0) { + res[2].flags = IORESOURCE_IRQ; + res[2].start = wakeup_irq; + res[2].end = wakeup_irq; + res[2].name = "wakeup"; + res_count++; + } + + pdev->name = "fiq_debugger"; + pdev->id = hi6220_fiq_debugger_id++; + pdev->dev.platform_data = &t->pdata; + pdev->resource = res; + pdev->num_resources = res_count; + + ret = platform_device_register(pdev); + if (ret) { + pr_err("Failed to register fiq debugger\n"); + goto out4; + } + + return 0; + +out4: + kfree(pdev); +out3: + kfree(res); +out2: + iounmap(t->debug_port_base); +out1: + kfree(t); + return ret; +} + +int hi6220_serial_debug_init_irq_mode(unsigned int base, int irq, + struct clk *clk, int signal_irq, int wakeup_irq) +{ + return __hi6220_serial_debug_init(base, -1, irq, clk, signal_irq, + wakeup_irq); +} + +static int hi6220_serial_debug_probe(struct platform_device *pdev) +{ + struct resource *res; + int fiq; + int signal_irq; + int wakeup_irq; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) { + dev_err(&pdev->dev, "No mem resource\n"); + return -EINVAL; + } + + fiq = platform_get_irq_byname(pdev, "fiq"); + if (fiq < 0) { + dev_err(&pdev->dev, "No IRQ for fiq, error=%d\n", fiq); + return fiq; + } + + signal_irq = platform_get_irq_byname(pdev, "signal"); + if (signal_irq < 0) { + dev_err(&pdev->dev, "No signal IRQ, error=%d\n", signal_irq); + signal_irq = -1; + } + + wakeup_irq = platform_get_irq_byname(pdev, "wakeup"); + if (wakeup_irq < 0) { + dev_err(&pdev->dev, "No wakeup IRQ, error=%d\n", wakeup_irq); + wakeup_irq = -1; + } + + return hi6220_serial_debug_init_irq_mode(res->start, fiq, NULL, + signal_irq, wakeup_irq); +} + +static const struct of_device_id hi6220_serial_debug_match[] = { + { .compatible = "android,irq-hi6220-uart", }, + {} +}; + +static struct platform_driver hi6220_serial_debug_driver = { + .probe = hi6220_serial_debug_probe, + .driver = { + .name = "hi6220-irq-debug", + .owner = THIS_MODULE, + .of_match_table = of_match_ptr(hi6220_serial_debug_match), + }, +}; + +module_platform_driver(hi6220_serial_debug_driver); diff --git a/drivers/platform/x86/asus-nb-wmi.c b/drivers/platform/x86/asus-nb-wmi.c index 26e4cbc34db8f8..6032b7085582b4 100644 --- a/drivers/platform/x86/asus-nb-wmi.c +++ b/drivers/platform/x86/asus-nb-wmi.c @@ -173,6 +173,15 @@ static const struct dmi_system_id asus_quirks[] = { }, .driver_data = &quirk_asus_wapf4, }, + { + .callback = dmi_matched, + .ident = "ASUSTeK COMPUTER INC. X45U", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_PRODUCT_NAME, "X45U"), + }, + .driver_data = &quirk_asus_wapf4, + }, { .callback = dmi_matched, .ident = "ASUSTeK COMPUTER INC. X456UA", diff --git a/drivers/platform/x86/fujitsu-laptop.c b/drivers/platform/x86/fujitsu-laptop.c index 61f39abf5dc8f2..82d67715ce76d9 100644 --- a/drivers/platform/x86/fujitsu-laptop.c +++ b/drivers/platform/x86/fujitsu-laptop.c @@ -177,43 +177,43 @@ static void acpi_fujitsu_hotkey_notify(struct acpi_device *device, u32 event); #if IS_ENABLED(CONFIG_LEDS_CLASS) static enum led_brightness logolamp_get(struct led_classdev *cdev); -static void logolamp_set(struct led_classdev *cdev, +static int logolamp_set(struct led_classdev *cdev, enum led_brightness brightness); static struct led_classdev logolamp_led = { .name = "fujitsu::logolamp", .brightness_get = logolamp_get, - .brightness_set = logolamp_set + .brightness_set_blocking = logolamp_set }; static enum led_brightness kblamps_get(struct led_classdev *cdev); -static void kblamps_set(struct led_classdev *cdev, +static int kblamps_set(struct led_classdev *cdev, enum led_brightness brightness); static struct led_classdev kblamps_led = { .name = "fujitsu::kblamps", .brightness_get = kblamps_get, - .brightness_set = kblamps_set + .brightness_set_blocking = kblamps_set }; static enum led_brightness radio_led_get(struct led_classdev *cdev); -static void radio_led_set(struct led_classdev *cdev, +static int radio_led_set(struct led_classdev *cdev, enum led_brightness brightness); static struct led_classdev radio_led = { .name = "fujitsu::radio_led", .brightness_get = radio_led_get, - .brightness_set = radio_led_set + .brightness_set_blocking = radio_led_set }; static enum led_brightness eco_led_get(struct led_classdev *cdev); -static void eco_led_set(struct led_classdev *cdev, +static int eco_led_set(struct led_classdev *cdev, enum led_brightness brightness); static struct led_classdev eco_led = { .name = "fujitsu::eco_led", .brightness_get = eco_led_get, - .brightness_set = eco_led_set + .brightness_set_blocking = eco_led_set }; #endif @@ -267,48 +267,48 @@ static int call_fext_func(int cmd, int arg0, int arg1, int arg2) #if IS_ENABLED(CONFIG_LEDS_CLASS) /* LED class callbacks */ -static void logolamp_set(struct led_classdev *cdev, +static int logolamp_set(struct led_classdev *cdev, enum led_brightness brightness) { if (brightness >= LED_FULL) { call_fext_func(FUNC_LEDS, 0x1, LOGOLAMP_POWERON, FUNC_LED_ON); - call_fext_func(FUNC_LEDS, 0x1, LOGOLAMP_ALWAYS, FUNC_LED_ON); + return call_fext_func(FUNC_LEDS, 0x1, LOGOLAMP_ALWAYS, FUNC_LED_ON); } else if (brightness >= LED_HALF) { call_fext_func(FUNC_LEDS, 0x1, LOGOLAMP_POWERON, FUNC_LED_ON); - call_fext_func(FUNC_LEDS, 0x1, LOGOLAMP_ALWAYS, FUNC_LED_OFF); + return call_fext_func(FUNC_LEDS, 0x1, LOGOLAMP_ALWAYS, FUNC_LED_OFF); } else { - call_fext_func(FUNC_LEDS, 0x1, LOGOLAMP_POWERON, FUNC_LED_OFF); + return call_fext_func(FUNC_LEDS, 0x1, LOGOLAMP_POWERON, FUNC_LED_OFF); } } -static void kblamps_set(struct led_classdev *cdev, +static int kblamps_set(struct led_classdev *cdev, enum led_brightness brightness) { if (brightness >= LED_FULL) - call_fext_func(FUNC_LEDS, 0x1, KEYBOARD_LAMPS, FUNC_LED_ON); + return call_fext_func(FUNC_LEDS, 0x1, KEYBOARD_LAMPS, FUNC_LED_ON); else - call_fext_func(FUNC_LEDS, 0x1, KEYBOARD_LAMPS, FUNC_LED_OFF); + return call_fext_func(FUNC_LEDS, 0x1, KEYBOARD_LAMPS, FUNC_LED_OFF); } -static void radio_led_set(struct led_classdev *cdev, +static int radio_led_set(struct led_classdev *cdev, enum led_brightness brightness) { if (brightness >= LED_FULL) - call_fext_func(FUNC_RFKILL, 0x5, RADIO_LED_ON, RADIO_LED_ON); + return call_fext_func(FUNC_RFKILL, 0x5, RADIO_LED_ON, RADIO_LED_ON); else - call_fext_func(FUNC_RFKILL, 0x5, RADIO_LED_ON, 0x0); + return call_fext_func(FUNC_RFKILL, 0x5, RADIO_LED_ON, 0x0); } -static void eco_led_set(struct led_classdev *cdev, +static int eco_led_set(struct led_classdev *cdev, enum led_brightness brightness) { int curr; curr = call_fext_func(FUNC_LEDS, 0x2, ECO_LED, 0x0); if (brightness >= LED_FULL) - call_fext_func(FUNC_LEDS, 0x1, ECO_LED, curr | ECO_LED_ON); + return call_fext_func(FUNC_LEDS, 0x1, ECO_LED, curr | ECO_LED_ON); else - call_fext_func(FUNC_LEDS, 0x1, ECO_LED, curr & ~ECO_LED_ON); + return call_fext_func(FUNC_LEDS, 0x1, ECO_LED, curr & ~ECO_LED_ON); } static enum led_brightness logolamp_get(struct led_classdev *cdev) diff --git a/drivers/platform/x86/intel_mid_powerbtn.c b/drivers/platform/x86/intel_mid_powerbtn.c index 1fc0de870ff826..361770568ad03a 100644 --- a/drivers/platform/x86/intel_mid_powerbtn.c +++ b/drivers/platform/x86/intel_mid_powerbtn.c @@ -77,7 +77,7 @@ static int mfld_pb_probe(struct platform_device *pdev) input_set_capability(input, EV_KEY, KEY_POWER); - error = request_threaded_irq(irq, NULL, mfld_pb_isr, 0, + error = request_threaded_irq(irq, NULL, mfld_pb_isr, IRQF_ONESHOT, DRIVER_NAME, input); if (error) { dev_err(&pdev->dev, "Unable to request irq %d for mfld power" diff --git a/drivers/power/supply/bq24190_charger.c b/drivers/power/supply/bq24190_charger.c index f5746b9f4e8390..e9584330aeed27 100644 --- a/drivers/power/supply/bq24190_charger.c +++ b/drivers/power/supply/bq24190_charger.c @@ -1141,7 +1141,7 @@ static int bq24190_battery_set_property(struct power_supply *psy, dev_dbg(bdi->dev, "prop: %d\n", psp); - pm_runtime_put_sync(bdi->dev); + pm_runtime_get_sync(bdi->dev); switch (psp) { case POWER_SUPPLY_PROP_ONLINE: diff --git a/drivers/power/supply/bq27xxx_battery.c b/drivers/power/supply/bq27xxx_battery.c index 3b0dbc689d7299..bccb3f595ff3df 100644 --- a/drivers/power/supply/bq27xxx_battery.c +++ b/drivers/power/supply/bq27xxx_battery.c @@ -164,6 +164,25 @@ static u8 bq27xxx_regs[][BQ27XXX_REG_MAX] = { [BQ27XXX_REG_DCAP] = 0x3c, [BQ27XXX_REG_AP] = INVALID_REG_ADDR, }, + [BQ27510] = { + [BQ27XXX_REG_CTRL] = 0x00, + [BQ27XXX_REG_TEMP] = 0x06, + [BQ27XXX_REG_INT_TEMP] = 0x28, + [BQ27XXX_REG_VOLT] = 0x08, + [BQ27XXX_REG_AI] = 0x14, + [BQ27XXX_REG_FLAGS] = 0x0a, + [BQ27XXX_REG_TTE] = 0x16, + [BQ27XXX_REG_TTF] = INVALID_REG_ADDR, + [BQ27XXX_REG_TTES] = 0x1a, + [BQ27XXX_REG_TTECP] = INVALID_REG_ADDR, + [BQ27XXX_REG_NAC] = 0x0c, + [BQ27XXX_REG_FCC] = 0x12, + [BQ27XXX_REG_CYCT] = 0x1e, + [BQ27XXX_REG_AE] = INVALID_REG_ADDR, + [BQ27XXX_REG_SOC] = 0x20, + [BQ27XXX_REG_DCAP] = 0x2e, + [BQ27XXX_REG_AP] = INVALID_REG_ADDR, + }, [BQ27530] = { [BQ27XXX_REG_CTRL] = 0x00, [BQ27XXX_REG_TEMP] = 0x06, @@ -302,6 +321,24 @@ static enum power_supply_property bq27500_battery_props[] = { POWER_SUPPLY_PROP_MANUFACTURER, }; +static enum power_supply_property bq27510_battery_props[] = { + POWER_SUPPLY_PROP_STATUS, + POWER_SUPPLY_PROP_PRESENT, + POWER_SUPPLY_PROP_VOLTAGE_NOW, + POWER_SUPPLY_PROP_CURRENT_NOW, + POWER_SUPPLY_PROP_CAPACITY, + POWER_SUPPLY_PROP_CAPACITY_LEVEL, + POWER_SUPPLY_PROP_TEMP, + POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW, + POWER_SUPPLY_PROP_TECHNOLOGY, + POWER_SUPPLY_PROP_CHARGE_FULL, + POWER_SUPPLY_PROP_CHARGE_NOW, + POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN, + POWER_SUPPLY_PROP_CYCLE_COUNT, + POWER_SUPPLY_PROP_HEALTH, + POWER_SUPPLY_PROP_MANUFACTURER, +}; + static enum power_supply_property bq27530_battery_props[] = { POWER_SUPPLY_PROP_STATUS, POWER_SUPPLY_PROP_PRESENT, @@ -385,6 +422,7 @@ static struct { BQ27XXX_PROP(BQ27000, bq27000_battery_props), BQ27XXX_PROP(BQ27010, bq27010_battery_props), BQ27XXX_PROP(BQ27500, bq27500_battery_props), + BQ27XXX_PROP(BQ27510, bq27510_battery_props), BQ27XXX_PROP(BQ27530, bq27530_battery_props), BQ27XXX_PROP(BQ27541, bq27541_battery_props), BQ27XXX_PROP(BQ27545, bq27545_battery_props), @@ -635,7 +673,8 @@ static int bq27xxx_battery_read_pwr_avg(struct bq27xxx_device_info *di) */ static bool bq27xxx_battery_overtemp(struct bq27xxx_device_info *di, u16 flags) { - if (di->chip == BQ27500 || di->chip == BQ27541 || di->chip == BQ27545) + if (di->chip == BQ27500 || di->chip == BQ27510 || + di->chip == BQ27541 || di->chip == BQ27545) return flags & (BQ27XXX_FLAG_OTC | BQ27XXX_FLAG_OTD); if (di->chip == BQ27530 || di->chip == BQ27421) return flags & BQ27XXX_FLAG_OT; diff --git a/drivers/power/supply/bq27xxx_battery_i2c.c b/drivers/power/supply/bq27xxx_battery_i2c.c index 85d4ea2a9c2026..5c5c3a6f99234d 100644 --- a/drivers/power/supply/bq27xxx_battery_i2c.c +++ b/drivers/power/supply/bq27xxx_battery_i2c.c @@ -149,8 +149,8 @@ static const struct i2c_device_id bq27xxx_i2c_id_table[] = { { "bq27200", BQ27000 }, { "bq27210", BQ27010 }, { "bq27500", BQ27500 }, - { "bq27510", BQ27500 }, - { "bq27520", BQ27500 }, + { "bq27510", BQ27510 }, + { "bq27520", BQ27510 }, { "bq27530", BQ27530 }, { "bq27531", BQ27530 }, { "bq27541", BQ27541 }, diff --git a/drivers/power/supply/power_supply_sysfs.c b/drivers/power/supply/power_supply_sysfs.c index bcde8d13476af5..fdb824fdf6c13d 100644 --- a/drivers/power/supply/power_supply_sysfs.c +++ b/drivers/power/supply/power_supply_sysfs.c @@ -107,7 +107,10 @@ static ssize_t power_supply_show_property(struct device *dev, else if (off >= POWER_SUPPLY_PROP_MODEL_NAME) return sprintf(buf, "%s\n", value.strval); - return sprintf(buf, "%d\n", value.intval); + if (off == POWER_SUPPLY_PROP_CHARGE_COUNTER_EXT) + return sprintf(buf, "%lld\n", value.int64val); + else + return sprintf(buf, "%d\n", value.intval); } static ssize_t power_supply_store_property(struct device *dev, @@ -198,6 +201,12 @@ static struct device_attribute power_supply_attrs[] = { POWER_SUPPLY_ATTR(scope), POWER_SUPPLY_ATTR(charge_term_current), POWER_SUPPLY_ATTR(calibrate), + /* Local extensions */ + POWER_SUPPLY_ATTR(usb_hc), + POWER_SUPPLY_ATTR(usb_otg), + POWER_SUPPLY_ATTR(charge_enabled), + /* Local extensions of type int64_t */ + POWER_SUPPLY_ATTR(charge_counter_ext), /* Properties of type `const char *' */ POWER_SUPPLY_ATTR(model_name), POWER_SUPPLY_ATTR(manufacturer), diff --git a/drivers/powercap/intel_rapl.c b/drivers/powercap/intel_rapl.c index 243b233ff31bff..3c71f608b444b1 100644 --- a/drivers/powercap/intel_rapl.c +++ b/drivers/powercap/intel_rapl.c @@ -442,6 +442,7 @@ static int contraint_to_pl(struct rapl_domain *rd, int cid) return i; } } + pr_err("Cannot find matching power limit for constraint %d\n", cid); return -EINVAL; } @@ -457,6 +458,10 @@ static int set_power_limit(struct powercap_zone *power_zone, int cid, get_online_cpus(); rd = power_zone_to_rapl_domain(power_zone); id = contraint_to_pl(rd, cid); + if (id < 0) { + ret = id; + goto set_exit; + } rp = rd->rp; @@ -496,6 +501,11 @@ static int get_current_power_limit(struct powercap_zone *power_zone, int cid, get_online_cpus(); rd = power_zone_to_rapl_domain(power_zone); id = contraint_to_pl(rd, cid); + if (id < 0) { + ret = id; + goto get_exit; + } + switch (rd->rpl[id].prim_id) { case PL1_ENABLE: prim = POWER_LIMIT1; @@ -512,6 +522,7 @@ static int get_current_power_limit(struct powercap_zone *power_zone, int cid, else *data = val; +get_exit: put_online_cpus(); return ret; @@ -527,6 +538,10 @@ static int set_time_window(struct powercap_zone *power_zone, int cid, get_online_cpus(); rd = power_zone_to_rapl_domain(power_zone); id = contraint_to_pl(rd, cid); + if (id < 0) { + ret = id; + goto set_time_exit; + } switch (rd->rpl[id].prim_id) { case PL1_ENABLE: @@ -538,6 +553,8 @@ static int set_time_window(struct powercap_zone *power_zone, int cid, default: ret = -EINVAL; } + +set_time_exit: put_online_cpus(); return ret; } @@ -552,6 +569,10 @@ static int get_time_window(struct powercap_zone *power_zone, int cid, u64 *data) get_online_cpus(); rd = power_zone_to_rapl_domain(power_zone); id = contraint_to_pl(rd, cid); + if (id < 0) { + ret = id; + goto get_time_exit; + } switch (rd->rpl[id].prim_id) { case PL1_ENABLE: @@ -566,6 +587,8 @@ static int get_time_window(struct powercap_zone *power_zone, int cid, u64 *data) } if (!ret) *data = val; + +get_time_exit: put_online_cpus(); return ret; @@ -707,7 +730,7 @@ static u64 rapl_unit_xlate(struct rapl_domain *rd, enum unit_type type, case ENERGY_UNIT: scale = ENERGY_UNIT_SCALE; /* per domain unit takes precedence */ - if (rd && rd->domain_energy_unit) + if (rd->domain_energy_unit) units = rd->domain_energy_unit; else units = rp->energy_unit; diff --git a/drivers/regulator/axp20x-regulator.c b/drivers/regulator/axp20x-regulator.c index 54382ef902c67b..a3ade9e4ef478e 100644 --- a/drivers/regulator/axp20x-regulator.c +++ b/drivers/regulator/axp20x-regulator.c @@ -272,7 +272,7 @@ static const struct regulator_desc axp806_regulators[] = { 64, AXP806_DCDCD_V_CTRL, 0x3f, AXP806_PWR_OUT_CTRL1, BIT(3)), AXP_DESC(AXP806, DCDCE, "dcdce", "vine", 1100, 3400, 100, - AXP806_DCDCB_V_CTRL, 0x1f, AXP806_PWR_OUT_CTRL1, BIT(4)), + AXP806_DCDCE_V_CTRL, 0x1f, AXP806_PWR_OUT_CTRL1, BIT(4)), AXP_DESC(AXP806, ALDO1, "aldo1", "aldoin", 700, 3300, 100, AXP806_ALDO1_V_CTRL, 0x1f, AXP806_PWR_OUT_CTRL1, BIT(5)), AXP_DESC(AXP806, ALDO2, "aldo2", "aldoin", 700, 3400, 100, @@ -337,10 +337,18 @@ static const struct regulator_desc axp809_regulators[] = { AXP22X_ELDO2_V_OUT, 0x1f, AXP22X_PWR_OUT_CTRL2, BIT(1)), AXP_DESC(AXP809, ELDO3, "eldo3", "eldoin", 700, 3300, 100, AXP22X_ELDO3_V_OUT, 0x1f, AXP22X_PWR_OUT_CTRL2, BIT(2)), - AXP_DESC_IO(AXP809, LDO_IO0, "ldo_io0", "ips", 700, 3300, 100, + /* + * Note the datasheet only guarantees reliable operation up to + * 3.3V, this needs to be enforced via dts provided constraints + */ + AXP_DESC_IO(AXP809, LDO_IO0, "ldo_io0", "ips", 700, 3800, 100, AXP22X_LDO_IO0_V_OUT, 0x1f, AXP20X_GPIO0_CTRL, 0x07, AXP22X_IO_ENABLED, AXP22X_IO_DISABLED), - AXP_DESC_IO(AXP809, LDO_IO1, "ldo_io1", "ips", 700, 3300, 100, + /* + * Note the datasheet only guarantees reliable operation up to + * 3.3V, this needs to be enforced via dts provided constraints + */ + AXP_DESC_IO(AXP809, LDO_IO1, "ldo_io1", "ips", 700, 3800, 100, AXP22X_LDO_IO1_V_OUT, 0x1f, AXP20X_GPIO1_CTRL, 0x07, AXP22X_IO_ENABLED, AXP22X_IO_DISABLED), AXP_DESC_FIXED(AXP809, RTC_LDO, "rtc_ldo", "ips", 1800), diff --git a/drivers/regulator/helpers.c b/drivers/regulator/helpers.c index bcf38fd5106a23..379cdacc05d8ec 100644 --- a/drivers/regulator/helpers.c +++ b/drivers/regulator/helpers.c @@ -454,13 +454,17 @@ EXPORT_SYMBOL_GPL(regulator_set_bypass_regmap); int regulator_get_bypass_regmap(struct regulator_dev *rdev, bool *enable) { unsigned int val; + unsigned int val_on = rdev->desc->bypass_val_on; int ret; ret = regmap_read(rdev->regmap, rdev->desc->bypass_reg, &val); if (ret != 0) return ret; - *enable = (val & rdev->desc->bypass_mask) == rdev->desc->bypass_val_on; + if (!val_on) + val_on = rdev->desc->bypass_mask; + + *enable = (val & rdev->desc->bypass_mask) == val_on; return 0; } diff --git a/drivers/regulator/stw481x-vmmc.c b/drivers/regulator/stw481x-vmmc.c index 7d2ae3e9e942f2..342f5da799758e 100644 --- a/drivers/regulator/stw481x-vmmc.c +++ b/drivers/regulator/stw481x-vmmc.c @@ -47,7 +47,8 @@ static struct regulator_desc vmmc_regulator = { .volt_table = stw481x_vmmc_voltages, .enable_time = 200, /* FIXME: look this up */ .enable_reg = STW_CONF1, - .enable_mask = STW_CONF1_PDN_VMMC, + .enable_mask = STW_CONF1_PDN_VMMC | STW_CONF1_MMC_LS_STATUS, + .enable_val = STW_CONF1_PDN_VMMC, .vsel_reg = STW_CONF1, .vsel_mask = STW_CONF1_VMMC_MASK, }; diff --git a/drivers/regulator/tps65086-regulator.c b/drivers/regulator/tps65086-regulator.c index 33f389d583ef11..caf174ffa316fa 100644 --- a/drivers/regulator/tps65086-regulator.c +++ b/drivers/regulator/tps65086-regulator.c @@ -71,18 +71,17 @@ struct tps65086_regulator { unsigned int decay_mask; }; -static const struct regulator_linear_range tps65086_buck126_10mv_ranges[] = { +static const struct regulator_linear_range tps65086_10mv_ranges[] = { REGULATOR_LINEAR_RANGE(0, 0x0, 0x0, 0), REGULATOR_LINEAR_RANGE(410000, 0x1, 0x7F, 10000), }; static const struct regulator_linear_range tps65086_buck126_25mv_ranges[] = { - REGULATOR_LINEAR_RANGE(0, 0x0, 0x0, 0), - REGULATOR_LINEAR_RANGE(1000000, 0x1, 0x18, 0), + REGULATOR_LINEAR_RANGE(1000000, 0x0, 0x18, 0), REGULATOR_LINEAR_RANGE(1025000, 0x19, 0x7F, 25000), }; -static const struct regulator_linear_range tps65086_buck345_ranges[] = { +static const struct regulator_linear_range tps65086_buck345_25mv_ranges[] = { REGULATOR_LINEAR_RANGE(0, 0x0, 0x0, 0), REGULATOR_LINEAR_RANGE(425000, 0x1, 0x7F, 25000), }; @@ -125,27 +124,27 @@ static int tps65086_of_parse_cb(struct device_node *dev, static struct tps65086_regulator regulators[] = { TPS65086_REGULATOR("BUCK1", "buck1", BUCK1, 0x80, TPS65086_BUCK1CTRL, BUCK_VID_MASK, TPS65086_BUCK123CTRL, BIT(0), - tps65086_buck126_10mv_ranges, TPS65086_BUCK1CTRL, + tps65086_10mv_ranges, TPS65086_BUCK1CTRL, BIT(0)), TPS65086_REGULATOR("BUCK2", "buck2", BUCK2, 0x80, TPS65086_BUCK2CTRL, BUCK_VID_MASK, TPS65086_BUCK123CTRL, BIT(1), - tps65086_buck126_10mv_ranges, TPS65086_BUCK2CTRL, + tps65086_10mv_ranges, TPS65086_BUCK2CTRL, BIT(0)), TPS65086_REGULATOR("BUCK3", "buck3", BUCK3, 0x80, TPS65086_BUCK3VID, BUCK_VID_MASK, TPS65086_BUCK123CTRL, BIT(2), - tps65086_buck345_ranges, TPS65086_BUCK3DECAY, + tps65086_10mv_ranges, TPS65086_BUCK3DECAY, BIT(0)), TPS65086_REGULATOR("BUCK4", "buck4", BUCK4, 0x80, TPS65086_BUCK4VID, BUCK_VID_MASK, TPS65086_BUCK4CTRL, BIT(0), - tps65086_buck345_ranges, TPS65086_BUCK4VID, + tps65086_10mv_ranges, TPS65086_BUCK4VID, BIT(0)), TPS65086_REGULATOR("BUCK5", "buck5", BUCK5, 0x80, TPS65086_BUCK5VID, BUCK_VID_MASK, TPS65086_BUCK5CTRL, BIT(0), - tps65086_buck345_ranges, TPS65086_BUCK5CTRL, + tps65086_10mv_ranges, TPS65086_BUCK5CTRL, BIT(0)), TPS65086_REGULATOR("BUCK6", "buck6", BUCK6, 0x80, TPS65086_BUCK6VID, BUCK_VID_MASK, TPS65086_BUCK6CTRL, BIT(0), - tps65086_buck126_10mv_ranges, TPS65086_BUCK6CTRL, + tps65086_10mv_ranges, TPS65086_BUCK6CTRL, BIT(0)), TPS65086_REGULATOR("LDOA1", "ldoa1", LDOA1, 0xF, TPS65086_LDOA1CTRL, VDOA1_VID_MASK, TPS65086_LDOA1CTRL, BIT(0), @@ -162,18 +161,6 @@ static struct tps65086_regulator regulators[] = { TPS65086_SWITCH("VTT", "vtt", VTT, TPS65086_SWVTT_EN, BIT(4)), }; -static inline bool has_25mv_mode(int id) -{ - switch (id) { - case BUCK1: - case BUCK2: - case BUCK6: - return true; - default: - return false; - } -} - static int tps65086_of_parse_cb(struct device_node *dev, const struct regulator_desc *desc, struct regulator_config *config) @@ -181,12 +168,27 @@ static int tps65086_of_parse_cb(struct device_node *dev, int ret; /* Check for 25mV step mode */ - if (has_25mv_mode(desc->id) && - of_property_read_bool(config->of_node, "ti,regulator-step-size-25mv")) { - regulators[desc->id].desc.linear_ranges = + if (of_property_read_bool(config->of_node, "ti,regulator-step-size-25mv")) { + switch (desc->id) { + case BUCK1: + case BUCK2: + case BUCK6: + regulators[desc->id].desc.linear_ranges = tps65086_buck126_25mv_ranges; - regulators[desc->id].desc.n_linear_ranges = + regulators[desc->id].desc.n_linear_ranges = ARRAY_SIZE(tps65086_buck126_25mv_ranges); + break; + case BUCK3: + case BUCK4: + case BUCK5: + regulators[desc->id].desc.linear_ranges = + tps65086_buck345_25mv_ranges; + regulators[desc->id].desc.n_linear_ranges = + ARRAY_SIZE(tps65086_buck345_25mv_ranges); + break; + default: + dev_warn(config->dev, "25mV step mode only valid for BUCK regulators\n"); + } } /* Check for decay mode */ diff --git a/drivers/remoteproc/Kconfig b/drivers/remoteproc/Kconfig index f396bfef5d4286..5fcbefcb86369b 100644 --- a/drivers/remoteproc/Kconfig +++ b/drivers/remoteproc/Kconfig @@ -91,17 +91,12 @@ config QCOM_Q6V5_PIL Say y here to support the Qualcomm Peripherial Image Loader for the Hexagon V5 based remote processors. -config QCOM_WCNSS_IRIS - tristate - depends on OF && ARCH_QCOM - config QCOM_WCNSS_PIL tristate "Qualcomm WCNSS Peripheral Image Loader" depends on OF && ARCH_QCOM depends on QCOM_SMEM select QCOM_MDT_LOADER select QCOM_SCM - select QCOM_WCNSS_IRIS select REMOTEPROC help Say y here to support the Peripheral Image Loader for the Qualcomm diff --git a/drivers/remoteproc/Makefile b/drivers/remoteproc/Makefile index 6dfb62ed643f8e..034b6f3563a79f 100644 --- a/drivers/remoteproc/Makefile +++ b/drivers/remoteproc/Makefile @@ -13,6 +13,7 @@ obj-$(CONFIG_WKUP_M3_RPROC) += wkup_m3_rproc.o obj-$(CONFIG_DA8XX_REMOTEPROC) += da8xx_remoteproc.o obj-$(CONFIG_QCOM_MDT_LOADER) += qcom_mdt_loader.o obj-$(CONFIG_QCOM_Q6V5_PIL) += qcom_q6v5_pil.o -obj-$(CONFIG_QCOM_WCNSS_IRIS) += qcom_wcnss_iris.o -obj-$(CONFIG_QCOM_WCNSS_PIL) += qcom_wcnss.o +obj-$(CONFIG_QCOM_WCNSS_PIL) += qcom_wcnss_pil.o +qcom_wcnss_pil-y += qcom_wcnss.o +qcom_wcnss_pil-y += qcom_wcnss_iris.o obj-$(CONFIG_ST_REMOTEPROC) += st_remoteproc.o diff --git a/drivers/remoteproc/qcom_wcnss.c b/drivers/remoteproc/qcom_wcnss.c index f5cedeaafba189..323b629474a6b5 100644 --- a/drivers/remoteproc/qcom_wcnss.c +++ b/drivers/remoteproc/qcom_wcnss.c @@ -143,7 +143,6 @@ void qcom_wcnss_assign_iris(struct qcom_wcnss *wcnss, mutex_unlock(&wcnss->iris_lock); } -EXPORT_SYMBOL_GPL(qcom_wcnss_assign_iris); static int wcnss_load(struct rproc *rproc, const struct firmware *fw) { @@ -619,6 +618,28 @@ static struct platform_driver wcnss_driver = { }, }; -module_platform_driver(wcnss_driver); +static int __init wcnss_init(void) +{ + int ret; + + ret = platform_driver_register(&wcnss_driver); + if (ret) + return ret; + + ret = platform_driver_register(&qcom_iris_driver); + if (ret) + platform_driver_unregister(&wcnss_driver); + + return ret; +} +module_init(wcnss_init); + +static void __exit wcnss_exit(void) +{ + platform_driver_unregister(&qcom_iris_driver); + platform_driver_unregister(&wcnss_driver); +} +module_exit(wcnss_exit); + MODULE_DESCRIPTION("Qualcomm Peripherial Image Loader for Wireless Subsystem"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/remoteproc/qcom_wcnss.h b/drivers/remoteproc/qcom_wcnss.h index 9dc4a9fe41e19a..25fb7f62a4576d 100644 --- a/drivers/remoteproc/qcom_wcnss.h +++ b/drivers/remoteproc/qcom_wcnss.h @@ -4,6 +4,8 @@ struct qcom_iris; struct qcom_wcnss; +extern struct platform_driver qcom_iris_driver; + struct wcnss_vreg_info { const char * const name; int min_voltage; diff --git a/drivers/remoteproc/qcom_wcnss_iris.c b/drivers/remoteproc/qcom_wcnss_iris.c index f0ca24a8dd0b77..05d6e175411a9b 100644 --- a/drivers/remoteproc/qcom_wcnss_iris.c +++ b/drivers/remoteproc/qcom_wcnss_iris.c @@ -94,14 +94,12 @@ int qcom_iris_enable(struct qcom_iris *iris) return ret; } -EXPORT_SYMBOL_GPL(qcom_iris_enable); void qcom_iris_disable(struct qcom_iris *iris) { clk_disable_unprepare(iris->xo_clk); regulator_bulk_disable(iris->num_vregs, iris->vregs); } -EXPORT_SYMBOL_GPL(qcom_iris_disable); static int qcom_iris_probe(struct platform_device *pdev) { @@ -174,7 +172,7 @@ static const struct of_device_id iris_of_match[] = { {} }; -static struct platform_driver wcnss_driver = { +struct platform_driver qcom_iris_driver = { .probe = qcom_iris_probe, .remove = qcom_iris_remove, .driver = { @@ -182,7 +180,3 @@ static struct platform_driver wcnss_driver = { .of_match_table = iris_of_match, }, }; - -module_platform_driver(wcnss_driver); -MODULE_DESCRIPTION("Qualcomm Wireless Subsystem Iris driver"); -MODULE_LICENSE("GPL v2"); diff --git a/drivers/remoteproc/st_remoteproc.c b/drivers/remoteproc/st_remoteproc.c index ae8963fcc8c8f8..da4e152e97331f 100644 --- a/drivers/remoteproc/st_remoteproc.c +++ b/drivers/remoteproc/st_remoteproc.c @@ -245,8 +245,10 @@ static int st_rproc_probe(struct platform_device *pdev) goto free_rproc; enabled = st_rproc_state(pdev); - if (enabled < 0) + if (enabled < 0) { + ret = enabled; goto free_rproc; + } if (enabled) { atomic_inc(&rproc->power); diff --git a/drivers/rpmsg/qcom_smd.c b/drivers/rpmsg/qcom_smd.c index 06fef2b4c81408..1d4770c02e57e2 100644 --- a/drivers/rpmsg/qcom_smd.c +++ b/drivers/rpmsg/qcom_smd.c @@ -739,7 +739,7 @@ static int __qcom_smd_send(struct qcom_smd_channel *channel, const void *data, while (qcom_smd_get_tx_avail(channel) < tlen) { if (!wait) { - ret = -ENOMEM; + ret = -EAGAIN; goto out; } diff --git a/drivers/rpmsg/rpmsg_core.c b/drivers/rpmsg/rpmsg_core.c index b6ea9ffa73816d..e0a629eaceab83 100644 --- a/drivers/rpmsg/rpmsg_core.c +++ b/drivers/rpmsg/rpmsg_core.c @@ -411,8 +411,8 @@ int rpmsg_register_device(struct rpmsg_device *rpdev) struct device *dev = &rpdev->dev; int ret; - dev_set_name(&rpdev->dev, "%s:%s", - dev_name(dev->parent), rpdev->id.name); + dev_set_name(&rpdev->dev, "%s.%s.%d.%d", dev_name(dev->parent), + rpdev->id.name, rpdev->src, rpdev->dst); rpdev->dev.bus = &rpmsg_bus; rpdev->dev.release = rpmsg_release_device; diff --git a/drivers/rtc/rtc-palmas.c b/drivers/rtc/rtc-palmas.c index 4bcfb88674d38b..34aea38ebfa625 100644 --- a/drivers/rtc/rtc-palmas.c +++ b/drivers/rtc/rtc-palmas.c @@ -45,6 +45,42 @@ struct palmas_rtc { /* Total number of RTC registers needed to set time*/ #define PALMAS_NUM_TIME_REGS (PALMAS_YEARS_REG - PALMAS_SECONDS_REG + 1) +/* + * Special bin2bcd mapping to deal with bcd storage of year. + * + * 0-69 -> 0xD0 + * 70-99 (1970 - 1999) -> 0xD0 - 0xF9 (correctly rolls to 0x00) + * 100-199 (2000 - 2099) -> 0x00 - 0x99 (does not roll to 0xA0 :-( ) + * 200-229 (2100 - 2129) -> 0xA0 - 0xC9 (really for completeness) + * 230- -> 0xC9 + * + * Confirmed: the only transition that does not work correctly for this rtc + * clock is the transition from 2099 to 2100, it proceeds to 2000. We will + * accept this issue since the clock retains and transitions the year correctly + * in all other conditions. + */ +static unsigned char year_bin2bcd(int val) +{ + if (val < 70) + return 0xD0; + if (val < 100) + return bin2bcd(val - 20) | 0x80; /* KISS leverage of bin2bcd */ + if (val >= 230) + return 0xC9; + if (val >= 200) + return bin2bcd(val - 180) | 0x80; + return bin2bcd(val - 100); +} + +static int year_bcd2bin(unsigned char val) +{ + if (val >= 0xD0) + return bcd2bin(val & 0x7F) + 20; + if (val >= 0xA0) + return bcd2bin(val & 0x7F) + 180; + return bcd2bin(val) + 100; +} + static int palmas_rtc_read_time(struct device *dev, struct rtc_time *tm) { unsigned char rtc_data[PALMAS_NUM_TIME_REGS]; @@ -71,7 +107,7 @@ static int palmas_rtc_read_time(struct device *dev, struct rtc_time *tm) tm->tm_hour = bcd2bin(rtc_data[2]); tm->tm_mday = bcd2bin(rtc_data[3]); tm->tm_mon = bcd2bin(rtc_data[4]) - 1; - tm->tm_year = bcd2bin(rtc_data[5]) + 100; + tm->tm_year = year_bcd2bin(rtc_data[5]); return ret; } @@ -87,7 +123,7 @@ static int palmas_rtc_set_time(struct device *dev, struct rtc_time *tm) rtc_data[2] = bin2bcd(tm->tm_hour); rtc_data[3] = bin2bcd(tm->tm_mday); rtc_data[4] = bin2bcd(tm->tm_mon + 1); - rtc_data[5] = bin2bcd(tm->tm_year - 100); + rtc_data[5] = year_bin2bcd(tm->tm_year); /* Stop RTC while updating the RTC time registers */ ret = palmas_update_bits(palmas, PALMAS_RTC_BASE, PALMAS_RTC_CTRL_REG, @@ -142,7 +178,7 @@ static int palmas_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alm) alm->time.tm_hour = bcd2bin(alarm_data[2]); alm->time.tm_mday = bcd2bin(alarm_data[3]); alm->time.tm_mon = bcd2bin(alarm_data[4]) - 1; - alm->time.tm_year = bcd2bin(alarm_data[5]) + 100; + alm->time.tm_year = year_bcd2bin(alarm_data[5]); ret = palmas_read(palmas, PALMAS_RTC_BASE, PALMAS_RTC_INTERRUPTS_REG, &int_val); @@ -173,7 +209,7 @@ static int palmas_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alm) alarm_data[2] = bin2bcd(alm->time.tm_hour); alarm_data[3] = bin2bcd(alm->time.tm_mday); alarm_data[4] = bin2bcd(alm->time.tm_mon + 1); - alarm_data[5] = bin2bcd(alm->time.tm_year - 100); + alarm_data[5] = year_bin2bcd(alm->time.tm_year); ret = palmas_bulk_write(palmas, PALMAS_RTC_BASE, PALMAS_ALARM_SECONDS_REG, alarm_data, PALMAS_NUM_TIME_REGS); diff --git a/drivers/s390/char/vmlogrdr.c b/drivers/s390/char/vmlogrdr.c index e883063c72581b..3167e85819941c 100644 --- a/drivers/s390/char/vmlogrdr.c +++ b/drivers/s390/char/vmlogrdr.c @@ -870,7 +870,7 @@ static int __init vmlogrdr_init(void) goto cleanup; for (i=0; i < MAXMINOR; ++i ) { - sys_ser[i].buffer = (char *) get_zeroed_page(GFP_KERNEL); + sys_ser[i].buffer = (char *) get_zeroed_page(GFP_KERNEL | GFP_DMA); if (!sys_ser[i].buffer) { rc = -ENOMEM; break; diff --git a/drivers/s390/scsi/zfcp_dbf.c b/drivers/s390/scsi/zfcp_dbf.c index 581001989937ce..d5bf36ec8a7513 100644 --- a/drivers/s390/scsi/zfcp_dbf.c +++ b/drivers/s390/scsi/zfcp_dbf.c @@ -289,11 +289,12 @@ void zfcp_dbf_rec_trig(char *tag, struct zfcp_adapter *adapter, /** - * zfcp_dbf_rec_run - trace event related to running recovery + * zfcp_dbf_rec_run_lvl - trace event related to running recovery + * @level: trace level to be used for event * @tag: identifier for event * @erp: erp_action running */ -void zfcp_dbf_rec_run(char *tag, struct zfcp_erp_action *erp) +void zfcp_dbf_rec_run_lvl(int level, char *tag, struct zfcp_erp_action *erp) { struct zfcp_dbf *dbf = erp->adapter->dbf; struct zfcp_dbf_rec *rec = &dbf->rec_buf; @@ -319,10 +320,20 @@ void zfcp_dbf_rec_run(char *tag, struct zfcp_erp_action *erp) else rec->u.run.rec_count = atomic_read(&erp->adapter->erp_counter); - debug_event(dbf->rec, 1, rec, sizeof(*rec)); + debug_event(dbf->rec, level, rec, sizeof(*rec)); spin_unlock_irqrestore(&dbf->rec_lock, flags); } +/** + * zfcp_dbf_rec_run - trace event related to running recovery + * @tag: identifier for event + * @erp: erp_action running + */ +void zfcp_dbf_rec_run(char *tag, struct zfcp_erp_action *erp) +{ + zfcp_dbf_rec_run_lvl(1, tag, erp); +} + /** * zfcp_dbf_rec_run_wka - trace wka port event with info like running recovery * @tag: identifier for event diff --git a/drivers/s390/scsi/zfcp_dbf.h b/drivers/s390/scsi/zfcp_dbf.h index 36d07584271d56..db186d44cfafb6 100644 --- a/drivers/s390/scsi/zfcp_dbf.h +++ b/drivers/s390/scsi/zfcp_dbf.h @@ -2,7 +2,7 @@ * zfcp device driver * debug feature declarations * - * Copyright IBM Corp. 2008, 2015 + * Copyright IBM Corp. 2008, 2016 */ #ifndef ZFCP_DBF_H @@ -283,6 +283,30 @@ struct zfcp_dbf { struct zfcp_dbf_scsi scsi_buf; }; +/** + * zfcp_dbf_hba_fsf_resp_suppress - true if we should not trace by default + * @req: request that has been completed + * + * Returns true if FCP response with only benign residual under count. + */ +static inline +bool zfcp_dbf_hba_fsf_resp_suppress(struct zfcp_fsf_req *req) +{ + struct fsf_qtcb *qtcb = req->qtcb; + u32 fsf_stat = qtcb->header.fsf_status; + struct fcp_resp *fcp_rsp; + u8 rsp_flags, fr_status; + + if (qtcb->prefix.qtcb_type != FSF_IO_COMMAND) + return false; /* not an FCP response */ + fcp_rsp = (struct fcp_resp *)&qtcb->bottom.io.fcp_rsp; + rsp_flags = fcp_rsp->fr_flags; + fr_status = fcp_rsp->fr_status; + return (fsf_stat == FSF_FCP_RSP_AVAILABLE) && + (rsp_flags == FCP_RESID_UNDER) && + (fr_status == SAM_STAT_GOOD); +} + static inline void zfcp_dbf_hba_fsf_resp(char *tag, int level, struct zfcp_fsf_req *req) { @@ -304,7 +328,9 @@ void zfcp_dbf_hba_fsf_response(struct zfcp_fsf_req *req) zfcp_dbf_hba_fsf_resp("fs_perr", 1, req); } else if (qtcb->header.fsf_status != FSF_GOOD) { - zfcp_dbf_hba_fsf_resp("fs_ferr", 1, req); + zfcp_dbf_hba_fsf_resp("fs_ferr", + zfcp_dbf_hba_fsf_resp_suppress(req) + ? 5 : 1, req); } else if ((req->fsf_command == FSF_QTCB_OPEN_PORT_WITH_DID) || (req->fsf_command == FSF_QTCB_OPEN_LUN)) { @@ -388,4 +414,15 @@ void zfcp_dbf_scsi_devreset(char *tag, struct scsi_cmnd *scmnd, u8 flag) _zfcp_dbf_scsi(tmp_tag, 1, scmnd, NULL); } +/** + * zfcp_dbf_scsi_nullcmnd() - trace NULLify of SCSI command in dev/tgt-reset. + * @scmnd: SCSI command that was NULLified. + * @fsf_req: request that owned @scmnd. + */ +static inline void zfcp_dbf_scsi_nullcmnd(struct scsi_cmnd *scmnd, + struct zfcp_fsf_req *fsf_req) +{ + _zfcp_dbf_scsi("scfc__1", 3, scmnd, fsf_req); +} + #endif /* ZFCP_DBF_H */ diff --git a/drivers/s390/scsi/zfcp_erp.c b/drivers/s390/scsi/zfcp_erp.c index a59d678125bd0e..7ccfce55903423 100644 --- a/drivers/s390/scsi/zfcp_erp.c +++ b/drivers/s390/scsi/zfcp_erp.c @@ -3,7 +3,7 @@ * * Error Recovery Procedures (ERP). * - * Copyright IBM Corp. 2002, 2015 + * Copyright IBM Corp. 2002, 2016 */ #define KMSG_COMPONENT "zfcp" @@ -1204,6 +1204,62 @@ static void zfcp_erp_action_dequeue(struct zfcp_erp_action *erp_action) } } +/** + * zfcp_erp_try_rport_unblock - unblock rport if no more/new recovery + * @port: zfcp_port whose fc_rport we should try to unblock + */ +static void zfcp_erp_try_rport_unblock(struct zfcp_port *port) +{ + unsigned long flags; + struct zfcp_adapter *adapter = port->adapter; + int port_status; + struct Scsi_Host *shost = adapter->scsi_host; + struct scsi_device *sdev; + + write_lock_irqsave(&adapter->erp_lock, flags); + port_status = atomic_read(&port->status); + if ((port_status & ZFCP_STATUS_COMMON_UNBLOCKED) == 0 || + (port_status & (ZFCP_STATUS_COMMON_ERP_INUSE | + ZFCP_STATUS_COMMON_ERP_FAILED)) != 0) { + /* new ERP of severity >= port triggered elsewhere meanwhile or + * local link down (adapter erp_failed but not clear unblock) + */ + zfcp_dbf_rec_run_lvl(4, "ertru_p", &port->erp_action); + write_unlock_irqrestore(&adapter->erp_lock, flags); + return; + } + spin_lock(shost->host_lock); + __shost_for_each_device(sdev, shost) { + struct zfcp_scsi_dev *zsdev = sdev_to_zfcp(sdev); + int lun_status; + + if (zsdev->port != port) + continue; + /* LUN under port of interest */ + lun_status = atomic_read(&zsdev->status); + if ((lun_status & ZFCP_STATUS_COMMON_ERP_FAILED) != 0) + continue; /* unblock rport despite failed LUNs */ + /* LUN recovery not given up yet [maybe follow-up pending] */ + if ((lun_status & ZFCP_STATUS_COMMON_UNBLOCKED) == 0 || + (lun_status & ZFCP_STATUS_COMMON_ERP_INUSE) != 0) { + /* LUN blocked: + * not yet unblocked [LUN recovery pending] + * or meanwhile blocked [new LUN recovery triggered] + */ + zfcp_dbf_rec_run_lvl(4, "ertru_l", &zsdev->erp_action); + spin_unlock(shost->host_lock); + write_unlock_irqrestore(&adapter->erp_lock, flags); + return; + } + } + /* now port has no child or all children have completed recovery, + * and no ERP of severity >= port was meanwhile triggered elsewhere + */ + zfcp_scsi_schedule_rport_register(port); + spin_unlock(shost->host_lock); + write_unlock_irqrestore(&adapter->erp_lock, flags); +} + static void zfcp_erp_action_cleanup(struct zfcp_erp_action *act, int result) { struct zfcp_adapter *adapter = act->adapter; @@ -1214,6 +1270,7 @@ static void zfcp_erp_action_cleanup(struct zfcp_erp_action *act, int result) case ZFCP_ERP_ACTION_REOPEN_LUN: if (!(act->status & ZFCP_STATUS_ERP_NO_REF)) scsi_device_put(sdev); + zfcp_erp_try_rport_unblock(port); break; case ZFCP_ERP_ACTION_REOPEN_PORT: @@ -1224,7 +1281,7 @@ static void zfcp_erp_action_cleanup(struct zfcp_erp_action *act, int result) */ if (act->step != ZFCP_ERP_STEP_UNINITIALIZED) if (result == ZFCP_ERP_SUCCEEDED) - zfcp_scsi_schedule_rport_register(port); + zfcp_erp_try_rport_unblock(port); /* fall through */ case ZFCP_ERP_ACTION_REOPEN_PORT_FORCED: put_device(&port->dev); diff --git a/drivers/s390/scsi/zfcp_ext.h b/drivers/s390/scsi/zfcp_ext.h index c8fed9fa1cca36..21c8c689b02bb1 100644 --- a/drivers/s390/scsi/zfcp_ext.h +++ b/drivers/s390/scsi/zfcp_ext.h @@ -3,7 +3,7 @@ * * External function declarations. * - * Copyright IBM Corp. 2002, 2015 + * Copyright IBM Corp. 2002, 2016 */ #ifndef ZFCP_EXT_H @@ -35,6 +35,8 @@ extern void zfcp_dbf_adapter_unregister(struct zfcp_adapter *); extern void zfcp_dbf_rec_trig(char *, struct zfcp_adapter *, struct zfcp_port *, struct scsi_device *, u8, u8); extern void zfcp_dbf_rec_run(char *, struct zfcp_erp_action *); +extern void zfcp_dbf_rec_run_lvl(int level, char *tag, + struct zfcp_erp_action *erp); extern void zfcp_dbf_rec_run_wka(char *, struct zfcp_fc_wka_port *, u64); extern void zfcp_dbf_hba_fsf_uss(char *, struct zfcp_fsf_req *); extern void zfcp_dbf_hba_fsf_res(char *, int, struct zfcp_fsf_req *); diff --git a/drivers/s390/scsi/zfcp_fsf.c b/drivers/s390/scsi/zfcp_fsf.c index 75f820ca17b79b..27ff38f839fc1c 100644 --- a/drivers/s390/scsi/zfcp_fsf.c +++ b/drivers/s390/scsi/zfcp_fsf.c @@ -1583,7 +1583,7 @@ static void zfcp_fsf_open_wka_port_handler(struct zfcp_fsf_req *req) int zfcp_fsf_open_wka_port(struct zfcp_fc_wka_port *wka_port) { struct zfcp_qdio *qdio = wka_port->adapter->qdio; - struct zfcp_fsf_req *req = NULL; + struct zfcp_fsf_req *req; int retval = -EIO; spin_lock_irq(&qdio->req_q_lock); @@ -1612,7 +1612,7 @@ int zfcp_fsf_open_wka_port(struct zfcp_fc_wka_port *wka_port) zfcp_fsf_req_free(req); out: spin_unlock_irq(&qdio->req_q_lock); - if (req && !IS_ERR(req)) + if (!retval) zfcp_dbf_rec_run_wka("fsowp_1", wka_port, req->req_id); return retval; } @@ -1638,7 +1638,7 @@ static void zfcp_fsf_close_wka_port_handler(struct zfcp_fsf_req *req) int zfcp_fsf_close_wka_port(struct zfcp_fc_wka_port *wka_port) { struct zfcp_qdio *qdio = wka_port->adapter->qdio; - struct zfcp_fsf_req *req = NULL; + struct zfcp_fsf_req *req; int retval = -EIO; spin_lock_irq(&qdio->req_q_lock); @@ -1667,7 +1667,7 @@ int zfcp_fsf_close_wka_port(struct zfcp_fc_wka_port *wka_port) zfcp_fsf_req_free(req); out: spin_unlock_irq(&qdio->req_q_lock); - if (req && !IS_ERR(req)) + if (!retval) zfcp_dbf_rec_run_wka("fscwp_1", wka_port, req->req_id); return retval; } diff --git a/drivers/s390/scsi/zfcp_fsf.h b/drivers/s390/scsi/zfcp_fsf.h index be1c04b334c51f..ea3c76ac0de14d 100644 --- a/drivers/s390/scsi/zfcp_fsf.h +++ b/drivers/s390/scsi/zfcp_fsf.h @@ -3,7 +3,7 @@ * * Interface to the FSF support functions. * - * Copyright IBM Corp. 2002, 2015 + * Copyright IBM Corp. 2002, 2016 */ #ifndef FSF_H @@ -78,6 +78,7 @@ #define FSF_APP_TAG_CHECK_FAILURE 0x00000082 #define FSF_REF_TAG_CHECK_FAILURE 0x00000083 #define FSF_ADAPTER_STATUS_AVAILABLE 0x000000AD +#define FSF_FCP_RSP_AVAILABLE 0x000000AF #define FSF_UNKNOWN_COMMAND 0x000000E2 #define FSF_UNKNOWN_OP_SUBTYPE 0x000000E3 #define FSF_INVALID_COMMAND_OPTION 0x000000E5 diff --git a/drivers/s390/scsi/zfcp_reqlist.h b/drivers/s390/scsi/zfcp_reqlist.h index 7c2c6194dfca58..703fce59befef0 100644 --- a/drivers/s390/scsi/zfcp_reqlist.h +++ b/drivers/s390/scsi/zfcp_reqlist.h @@ -4,7 +4,7 @@ * Data structure and helper functions for tracking pending FSF * requests. * - * Copyright IBM Corp. 2009 + * Copyright IBM Corp. 2009, 2016 */ #ifndef ZFCP_REQLIST_H @@ -180,4 +180,32 @@ static inline void zfcp_reqlist_move(struct zfcp_reqlist *rl, spin_unlock_irqrestore(&rl->lock, flags); } +/** + * zfcp_reqlist_apply_for_all() - apply a function to every request. + * @rl: the requestlist that contains the target requests. + * @f: the function to apply to each request; the first parameter of the + * function will be the target-request; the second parameter is the same + * pointer as given with the argument @data. + * @data: freely chosen argument; passed through to @f as second parameter. + * + * Uses :c:macro:`list_for_each_entry` to iterate over the lists in the hash- + * table (not a 'safe' variant, so don't modify the list). + * + * Holds @rl->lock over the entire request-iteration. + */ +static inline void +zfcp_reqlist_apply_for_all(struct zfcp_reqlist *rl, + void (*f)(struct zfcp_fsf_req *, void *), void *data) +{ + struct zfcp_fsf_req *req; + unsigned long flags; + unsigned int i; + + spin_lock_irqsave(&rl->lock, flags); + for (i = 0; i < ZFCP_REQ_LIST_BUCKETS; i++) + list_for_each_entry(req, &rl->buckets[i], list) + f(req, data); + spin_unlock_irqrestore(&rl->lock, flags); +} + #endif /* ZFCP_REQLIST_H */ diff --git a/drivers/s390/scsi/zfcp_scsi.c b/drivers/s390/scsi/zfcp_scsi.c index 9069f98a18172e..07ffdbb5107f73 100644 --- a/drivers/s390/scsi/zfcp_scsi.c +++ b/drivers/s390/scsi/zfcp_scsi.c @@ -3,7 +3,7 @@ * * Interface to Linux SCSI midlayer. * - * Copyright IBM Corp. 2002, 2015 + * Copyright IBM Corp. 2002, 2016 */ #define KMSG_COMPONENT "zfcp" @@ -88,9 +88,7 @@ int zfcp_scsi_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scpnt) } if (unlikely(!(status & ZFCP_STATUS_COMMON_UNBLOCKED))) { - /* This could be either - * open LUN pending: this is temporary, will result in - * open LUN or ERP_FAILED, so retry command + /* This could be * call to rport_delete pending: mimic retry from * fc_remote_port_chkready until rport is BLOCKED */ @@ -209,6 +207,57 @@ static int zfcp_scsi_eh_abort_handler(struct scsi_cmnd *scpnt) return retval; } +struct zfcp_scsi_req_filter { + u8 tmf_scope; + u32 lun_handle; + u32 port_handle; +}; + +static void zfcp_scsi_forget_cmnd(struct zfcp_fsf_req *old_req, void *data) +{ + struct zfcp_scsi_req_filter *filter = + (struct zfcp_scsi_req_filter *)data; + + /* already aborted - prevent side-effects - or not a SCSI command */ + if (old_req->data == NULL || old_req->fsf_command != FSF_QTCB_FCP_CMND) + return; + + /* (tmf_scope == FCP_TMF_TGT_RESET || tmf_scope == FCP_TMF_LUN_RESET) */ + if (old_req->qtcb->header.port_handle != filter->port_handle) + return; + + if (filter->tmf_scope == FCP_TMF_LUN_RESET && + old_req->qtcb->header.lun_handle != filter->lun_handle) + return; + + zfcp_dbf_scsi_nullcmnd((struct scsi_cmnd *)old_req->data, old_req); + old_req->data = NULL; +} + +static void zfcp_scsi_forget_cmnds(struct zfcp_scsi_dev *zsdev, u8 tm_flags) +{ + struct zfcp_adapter *adapter = zsdev->port->adapter; + struct zfcp_scsi_req_filter filter = { + .tmf_scope = FCP_TMF_TGT_RESET, + .port_handle = zsdev->port->handle, + }; + unsigned long flags; + + if (tm_flags == FCP_TMF_LUN_RESET) { + filter.tmf_scope = FCP_TMF_LUN_RESET; + filter.lun_handle = zsdev->lun_handle; + } + + /* + * abort_lock secures against other processings - in the abort-function + * and normal cmnd-handler - of (struct zfcp_fsf_req *)->data + */ + write_lock_irqsave(&adapter->abort_lock, flags); + zfcp_reqlist_apply_for_all(adapter->req_list, zfcp_scsi_forget_cmnd, + &filter); + write_unlock_irqrestore(&adapter->abort_lock, flags); +} + static int zfcp_task_mgmt_function(struct scsi_cmnd *scpnt, u8 tm_flags) { struct zfcp_scsi_dev *zfcp_sdev = sdev_to_zfcp(scpnt->device); @@ -241,8 +290,10 @@ static int zfcp_task_mgmt_function(struct scsi_cmnd *scpnt, u8 tm_flags) if (fsf_req->status & ZFCP_STATUS_FSFREQ_TMFUNCFAILED) { zfcp_dbf_scsi_devreset("fail", scpnt, tm_flags); retval = FAILED; - } else + } else { zfcp_dbf_scsi_devreset("okay", scpnt, tm_flags); + zfcp_scsi_forget_cmnds(zfcp_sdev, tm_flags); + } zfcp_fsf_req_free(fsf_req); return retval; diff --git a/drivers/scsi/aacraid/comminit.c b/drivers/scsi/aacraid/comminit.c index 341ea327ae7946..792d3e7e35e2ee 100644 --- a/drivers/scsi/aacraid/comminit.c +++ b/drivers/scsi/aacraid/comminit.c @@ -50,9 +50,13 @@ struct aac_common aac_config = { static inline int aac_is_msix_mode(struct aac_dev *dev) { - u32 status; + u32 status = 0; - status = src_readl(dev, MUnit.OMR); + if (dev->pdev->device == PMC_DEVICE_S6 || + dev->pdev->device == PMC_DEVICE_S7 || + dev->pdev->device == PMC_DEVICE_S8) { + status = src_readl(dev, MUnit.OMR); + } return (status & AAC_INT_MODE_MSIX); } diff --git a/drivers/scsi/aacraid/linit.c b/drivers/scsi/aacraid/linit.c index 79871f3519ffc9..d5b26fa541d367 100644 --- a/drivers/scsi/aacraid/linit.c +++ b/drivers/scsi/aacraid/linit.c @@ -160,7 +160,6 @@ static const struct pci_device_id aac_pci_tbl[] = { { 0x9005, 0x028b, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 62 }, /* Adaptec PMC Series 6 (Tupelo) */ { 0x9005, 0x028c, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 63 }, /* Adaptec PMC Series 7 (Denali) */ { 0x9005, 0x028d, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 64 }, /* Adaptec PMC Series 8 */ - { 0x9005, 0x028f, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 65 }, /* Adaptec PMC Series 9 */ { 0,} }; MODULE_DEVICE_TABLE(pci, aac_pci_tbl); @@ -239,7 +238,6 @@ static struct aac_driver_ident aac_drivers[] = { { aac_src_init, "aacraid", "ADAPTEC ", "RAID ", 2, AAC_QUIRK_SRC }, /* Adaptec PMC Series 6 (Tupelo) */ { aac_srcv_init, "aacraid", "ADAPTEC ", "RAID ", 2, AAC_QUIRK_SRC }, /* Adaptec PMC Series 7 (Denali) */ { aac_srcv_init, "aacraid", "ADAPTEC ", "RAID ", 2, AAC_QUIRK_SRC }, /* Adaptec PMC Series 8 */ - { aac_srcv_init, "aacraid", "ADAPTEC ", "RAID ", 2, AAC_QUIRK_SRC } /* Adaptec PMC Series 9 */ }; /** diff --git a/drivers/scsi/g_NCR5380.c b/drivers/scsi/g_NCR5380.c index cbf010324c1875..596a75924d9021 100644 --- a/drivers/scsi/g_NCR5380.c +++ b/drivers/scsi/g_NCR5380.c @@ -170,12 +170,12 @@ static int generic_NCR5380_init_one(struct scsi_host_template *tpnt, if (ports[i]) { /* At this point we have our region reserved */ magic_configure(i, 0, magic); /* no IRQ yet */ - outb(0xc0, ports[i] + 9); - if (inb(ports[i] + 9) != 0x80) { + base = ports[i]; + outb(0xc0, base + 9); + if (inb(base + 9) != 0x80) { ret = -ENODEV; goto out_release; } - base = ports[i]; port_idx = i; } else return -EINVAL; diff --git a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c index 642b739ad0da33..91dfd58b175dc3 100644 --- a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c +++ b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c @@ -45,6 +45,7 @@ #define INITIAL_SRP_LIMIT 800 #define DEFAULT_MAX_SECTORS 256 +#define MAX_TXU 1024 * 1024 static uint max_vdma_size = MAX_H_COPY_RDMA; @@ -1239,7 +1240,7 @@ static long ibmvscsis_adapter_info(struct scsi_info *vscsi, } info = dma_alloc_coherent(&vscsi->dma_dev->dev, sizeof(*info), &token, - GFP_KERNEL); + GFP_ATOMIC); if (!info) { dev_err(&vscsi->dev, "bad dma_alloc_coherent %p\n", iue->target); @@ -1291,7 +1292,7 @@ static long ibmvscsis_adapter_info(struct scsi_info *vscsi, info->mad_version = cpu_to_be32(MAD_VERSION_1); info->os_type = cpu_to_be32(LINUX); memset(&info->port_max_txu[0], 0, sizeof(info->port_max_txu)); - info->port_max_txu[0] = cpu_to_be32(128 * PAGE_SIZE); + info->port_max_txu[0] = cpu_to_be32(MAX_TXU); dma_wmb(); rc = h_copy_rdma(sizeof(*info), vscsi->dds.window[LOCAL].liobn, @@ -1357,7 +1358,7 @@ static int ibmvscsis_cap_mad(struct scsi_info *vscsi, struct iu_entry *iue) } cap = dma_alloc_coherent(&vscsi->dma_dev->dev, olen, &token, - GFP_KERNEL); + GFP_ATOMIC); if (!cap) { dev_err(&vscsi->dev, "bad dma_alloc_coherent %p\n", iue->target); @@ -3702,7 +3703,7 @@ static int ibmvscsis_write_pending(struct se_cmd *se_cmd) 1, 1); if (rc) { pr_err("srp_transfer_data() failed: %d\n", rc); - return -EAGAIN; + return -EIO; } /* * We now tell TCM to add this WRITE CDB directly into the TCM storage @@ -3928,6 +3929,7 @@ static struct configfs_attribute *ibmvscsis_tpg_attrs[] = { static const struct target_core_fabric_ops ibmvscsis_ops = { .module = THIS_MODULE, .name = "ibmvscsis", + .max_data_sg_nents = MAX_TXU / PAGE_SIZE, .get_fabric_name = ibmvscsis_get_fabric_name, .tpg_get_wwn = ibmvscsis_get_fabric_wwn, .tpg_get_tag = ibmvscsis_get_tag, diff --git a/drivers/scsi/megaraid/megaraid_sas_fusion.c b/drivers/scsi/megaraid/megaraid_sas_fusion.c index 52d8bbf7feb5c5..bd04bd01d34aad 100644 --- a/drivers/scsi/megaraid/megaraid_sas_fusion.c +++ b/drivers/scsi/megaraid/megaraid_sas_fusion.c @@ -2000,6 +2000,8 @@ megasas_build_syspd_fusion(struct megasas_instance *instance, io_request->DevHandle = pd_sync->seq[pd_index].devHandle; pRAID_Context->regLockFlags |= (MR_RL_FLAGS_SEQ_NUM_ENABLE|MR_RL_FLAGS_GRANT_DESTINATION_CUDA); + pRAID_Context->Type = MPI2_TYPE_CUDA; + pRAID_Context->nseg = 0x1; } else if (fusion->fast_path_io) { pRAID_Context->VirtualDiskTgtId = cpu_to_le16(device_id); pRAID_Context->configSeqNum = 0; @@ -2035,12 +2037,10 @@ megasas_build_syspd_fusion(struct megasas_instance *instance, pRAID_Context->timeoutValue = cpu_to_le16((os_timeout_value > timeout_limit) ? timeout_limit : os_timeout_value); - if (fusion->adapter_type == INVADER_SERIES) { - pRAID_Context->Type = MPI2_TYPE_CUDA; - pRAID_Context->nseg = 0x1; + if (fusion->adapter_type == INVADER_SERIES) io_request->IoFlags |= cpu_to_le16(MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH); - } + cmd->request_desc->SCSIIO.RequestFlags = (MPI2_REQ_DESCRIPT_FLAGS_FP_IO << MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT); @@ -2823,6 +2823,7 @@ int megasas_wait_for_outstanding_fusion(struct megasas_instance *instance, dev_err(&instance->pdev->dev, "pending commands remain after waiting, " "will reset adapter scsi%d.\n", instance->host->host_no); + *convert = 1; retval = 1; } out: diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.h b/drivers/scsi/mpt3sas/mpt3sas_base.h index 3e71bc1b4a8060..7008061c4b5bbd 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_base.h +++ b/drivers/scsi/mpt3sas/mpt3sas_base.h @@ -393,6 +393,7 @@ struct MPT3SAS_TARGET { * @eedp_enable: eedp support enable bit * @eedp_type: 0(type_1), 1(type_2), 2(type_3) * @eedp_block_length: block size + * @ata_command_pending: SATL passthrough outstanding for device */ struct MPT3SAS_DEVICE { struct MPT3SAS_TARGET *sas_target; @@ -402,6 +403,17 @@ struct MPT3SAS_DEVICE { u8 block; u8 tlr_snoop_check; u8 ignore_delay_remove; + /* + * Bug workaround for SATL handling: the mpt2/3sas firmware + * doesn't return BUSY or TASK_SET_FULL for subsequent + * commands while a SATL pass through is in operation as the + * spec requires, it simply does nothing with them until the + * pass through completes, causing them possibly to timeout if + * the passthrough is a long executing command (like format or + * secure erase). This variable allows us to do the right + * thing while a SATL command is pending. + */ + unsigned long ata_command_pending; }; #define MPT3_CMD_NOT_USED 0x8000 /* free */ diff --git a/drivers/scsi/mpt3sas/mpt3sas_scsih.c b/drivers/scsi/mpt3sas/mpt3sas_scsih.c index 1c4744e78173bc..8a7941b8189f8f 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_scsih.c +++ b/drivers/scsi/mpt3sas/mpt3sas_scsih.c @@ -51,6 +51,7 @@ #include #include #include +#include #include #include #include @@ -3885,9 +3886,18 @@ _scsih_temp_threshold_events(struct MPT3SAS_ADAPTER *ioc, } } -static inline bool ata_12_16_cmd(struct scsi_cmnd *scmd) +static int _scsih_set_satl_pending(struct scsi_cmnd *scmd, bool pending) { - return (scmd->cmnd[0] == ATA_12 || scmd->cmnd[0] == ATA_16); + struct MPT3SAS_DEVICE *priv = scmd->device->hostdata; + + if (scmd->cmnd[0] != ATA_12 && scmd->cmnd[0] != ATA_16) + return 0; + + if (pending) + return test_and_set_bit(0, &priv->ata_command_pending); + + clear_bit(0, &priv->ata_command_pending); + return 0; } /** @@ -3911,9 +3921,7 @@ _scsih_flush_running_cmds(struct MPT3SAS_ADAPTER *ioc) if (!scmd) continue; count++; - if (ata_12_16_cmd(scmd)) - scsi_internal_device_unblock(scmd->device, - SDEV_RUNNING); + _scsih_set_satl_pending(scmd, false); mpt3sas_base_free_smid(ioc, smid); scsi_dma_unmap(scmd); if (ioc->pci_error_recovery) @@ -4044,13 +4052,6 @@ scsih_qcmd(struct Scsi_Host *shost, struct scsi_cmnd *scmd) if (ioc->logging_level & MPT_DEBUG_SCSI) scsi_print_command(scmd); - /* - * Lock the device for any subsequent command until command is - * done. - */ - if (ata_12_16_cmd(scmd)) - scsi_internal_device_block(scmd->device); - sas_device_priv_data = scmd->device->hostdata; if (!sas_device_priv_data || !sas_device_priv_data->sas_target) { scmd->result = DID_NO_CONNECT << 16; @@ -4064,6 +4065,19 @@ scsih_qcmd(struct Scsi_Host *shost, struct scsi_cmnd *scmd) return 0; } + /* + * Bug work around for firmware SATL handling. The loop + * is based on atomic operations and ensures consistency + * since we're lockless at this point + */ + do { + if (test_bit(0, &sas_device_priv_data->ata_command_pending)) { + scmd->result = SAM_STAT_BUSY; + scmd->scsi_done(scmd); + return 0; + } + } while (_scsih_set_satl_pending(scmd, true)); + sas_target_priv_data = sas_device_priv_data->sas_target; /* invalid device handle */ @@ -4626,8 +4640,7 @@ _scsih_io_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index, u32 reply) if (scmd == NULL) return 1; - if (ata_12_16_cmd(scmd)) - scsi_internal_device_unblock(scmd->device, SDEV_RUNNING); + _scsih_set_satl_pending(scmd, false); mpi_request = mpt3sas_base_get_msg_frame(ioc, smid); @@ -8694,6 +8707,8 @@ _scsih_probe(struct pci_dev *pdev, const struct pci_device_id *id) switch (hba_mpi_version) { case MPI2_VERSION: + pci_disable_link_state(pdev, PCIE_LINK_STATE_L0S | + PCIE_LINK_STATE_L1 | PCIE_LINK_STATE_CLKPM); /* Use mpt2sas driver host template for SAS 2.0 HBA's */ shost = scsi_host_alloc(&mpt2sas_driver_template, sizeof(struct MPT3SAS_ADAPTER)); diff --git a/drivers/scsi/mvsas/mv_94xx.c b/drivers/scsi/mvsas/mv_94xx.c index 4c57d9abce7b77..7de5d8d75480f0 100644 --- a/drivers/scsi/mvsas/mv_94xx.c +++ b/drivers/scsi/mvsas/mv_94xx.c @@ -668,7 +668,7 @@ static void mvs_94xx_command_active(struct mvs_info *mvi, u32 slot_idx) { u32 tmp; tmp = mvs_cr32(mvi, MVS_COMMAND_ACTIVE+(slot_idx >> 3)); - if (tmp && 1 << (slot_idx % 32)) { + if (tmp & 1 << (slot_idx % 32)) { mv_printk("command active %08X, slot [%x].\n", tmp, slot_idx); mvs_cw32(mvi, MVS_COMMAND_ACTIVE + (slot_idx >> 3), 1 << (slot_idx % 32)); diff --git a/drivers/scsi/qla2xxx/qla_os.c b/drivers/scsi/qla2xxx/qla_os.c index 56d6142852a553..bea819e5336d14 100644 --- a/drivers/scsi/qla2xxx/qla_os.c +++ b/drivers/scsi/qla2xxx/qla_os.c @@ -1459,7 +1459,7 @@ qla2x00_abort_all_cmds(scsi_qla_host_t *vha, int res) /* Don't abort commands in adapter during EEH * recovery as it's not accessible/responding. */ - if (!ha->flags.eeh_busy) { + if (GET_CMD_SP(sp) && !ha->flags.eeh_busy) { /* Get a reference to the sp and drop the lock. * The reference ensures this sp->done() call * - and not the call in qla2xxx_eh_abort() - @@ -3489,7 +3489,7 @@ qla2x00_mem_alloc(struct qla_hw_data *ha, uint16_t req_len, uint16_t rsp_len, sizeof(struct ct6_dsd), 0, SLAB_HWCACHE_ALIGN, NULL); if (!ctx_cachep) - goto fail_free_gid_list; + goto fail_free_srb_mempool; } ha->ctx_mempool = mempool_create_slab_pool(SRB_MIN_REQ, ctx_cachep); @@ -3642,7 +3642,7 @@ qla2x00_mem_alloc(struct qla_hw_data *ha, uint16_t req_len, uint16_t rsp_len, ha->loop_id_map = kzalloc(BITS_TO_LONGS(LOOPID_MAP_SIZE) * sizeof(long), GFP_KERNEL); if (!ha->loop_id_map) - goto fail_async_pd; + goto fail_loop_id_map; else { qla2x00_set_reserved_loop_ids(ha); ql_dbg_pci(ql_dbg_init, ha->pdev, 0x0123, @@ -3651,6 +3651,8 @@ qla2x00_mem_alloc(struct qla_hw_data *ha, uint16_t req_len, uint16_t rsp_len, return 0; +fail_loop_id_map: + dma_pool_free(ha->s_dma_pool, ha->async_pd, ha->async_pd_dma); fail_async_pd: dma_pool_free(ha->s_dma_pool, ha->ex_init_cb, ha->ex_init_cb_dma); fail_ex_init_cb: @@ -3678,6 +3680,10 @@ qla2x00_mem_alloc(struct qla_hw_data *ha, uint16_t req_len, uint16_t rsp_len, dma_pool_free(ha->s_dma_pool, ha->ms_iocb, ha->ms_iocb_dma); ha->ms_iocb = NULL; ha->ms_iocb_dma = 0; + + if (ha->sns_cmd) + dma_free_coherent(&ha->pdev->dev, sizeof(struct sns_cmd_pkt), + ha->sns_cmd, ha->sns_cmd_dma); fail_dma_pool: if (IS_QLA82XX(ha) || ql2xenabledif) { dma_pool_destroy(ha->fcp_cmnd_dma_pool); @@ -3695,10 +3701,12 @@ qla2x00_mem_alloc(struct qla_hw_data *ha, uint16_t req_len, uint16_t rsp_len, kfree(ha->nvram); ha->nvram = NULL; fail_free_ctx_mempool: - mempool_destroy(ha->ctx_mempool); + if (ha->ctx_mempool) + mempool_destroy(ha->ctx_mempool); ha->ctx_mempool = NULL; fail_free_srb_mempool: - mempool_destroy(ha->srb_mempool); + if (ha->srb_mempool) + mempool_destroy(ha->srb_mempool); ha->srb_mempool = NULL; fail_free_gid_list: dma_free_coherent(&ha->pdev->dev, qla2x00_gid_list_size(ha), diff --git a/drivers/scsi/scsi_sysfs.c b/drivers/scsi/scsi_sysfs.c index 07349270535d19..82dfe07b1d47f7 100644 --- a/drivers/scsi/scsi_sysfs.c +++ b/drivers/scsi/scsi_sysfs.c @@ -1204,10 +1204,6 @@ int scsi_sysfs_add_sdev(struct scsi_device *sdev) struct request_queue *rq = sdev->request_queue; struct scsi_target *starget = sdev->sdev_target; - error = scsi_device_set_state(sdev, SDEV_RUNNING); - if (error) - return error; - error = scsi_target_add(starget); if (error) return error; diff --git a/drivers/scsi/ses.c b/drivers/scsi/ses.c index 8c9a35c91705e4..50adabbb580890 100644 --- a/drivers/scsi/ses.c +++ b/drivers/scsi/ses.c @@ -587,7 +587,7 @@ static void ses_match_to_enclosure(struct enclosure_device *edev, ses_enclosure_data_process(edev, to_scsi_device(edev->edev.parent), 0); - if (scsi_is_sas_rphy(&sdev->sdev_gendev)) + if (scsi_is_sas_rphy(sdev->sdev_target->dev.parent)) efd.addr = sas_get_address(sdev); if (efd.addr) { diff --git a/drivers/scsi/sg.c b/drivers/scsi/sg.c index 070332eb41f33d..dbe5b4b95df0d9 100644 --- a/drivers/scsi/sg.c +++ b/drivers/scsi/sg.c @@ -581,6 +581,9 @@ sg_write(struct file *filp, const char __user *buf, size_t count, loff_t * ppos) sg_io_hdr_t *hp; unsigned char cmnd[SG_MAX_CDB_SIZE]; + if (unlikely(segment_eq(get_fs(), KERNEL_DS))) + return -EINVAL; + if ((!(sfp = (Sg_fd *) filp->private_data)) || (!(sdp = sfp->parentdp))) return -ENXIO; SCSI_LOG_TIMEOUT(3, sg_printk(KERN_INFO, sdp, diff --git a/drivers/scsi/ufs/ufshcd.c b/drivers/scsi/ufs/ufshcd.c index 05c745663c103a..9e3177b9cda54d 100644 --- a/drivers/scsi/ufs/ufshcd.c +++ b/drivers/scsi/ufs/ufshcd.c @@ -41,6 +41,7 @@ #include #include #include +#include #include "ufshcd.h" #include "ufs_quirks.h" #include "unipro.h" @@ -1451,6 +1452,17 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) clear_bit_unlock(tag, &hba->lrb_in_use); goto out; } + + /* IO svc time latency histogram */ + if (hba != NULL && cmd->request != NULL) { + if (hba->latency_hist_enabled && + (cmd->request->cmd_type == REQ_TYPE_FS)) { + cmd->request->lat_hist_io_start = ktime_get(); + cmd->request->lat_hist_enabled = 1; + } else + cmd->request->lat_hist_enabled = 0; + } + WARN_ON(hba->clk_gating.state != CLKS_ON); lrbp = &hba->lrb[tag]; @@ -3533,6 +3545,7 @@ static void __ufshcd_transfer_req_compl(struct ufs_hba *hba, struct scsi_cmnd *cmd; int result; int index; + struct request *req; for_each_set_bit(index, &completed_reqs, hba->nutrs) { lrbp = &hba->lrb[index]; @@ -3544,6 +3557,22 @@ static void __ufshcd_transfer_req_compl(struct ufs_hba *hba, /* Mark completed command as NULL in LRB */ lrbp->cmd = NULL; clear_bit_unlock(index, &hba->lrb_in_use); + req = cmd->request; + if (req) { + /* Update IO svc time latency histogram */ + if (req->lat_hist_enabled) { + ktime_t completion; + u_int64_t delta_us; + + completion = ktime_get(); + delta_us = ktime_us_delta(completion, + req->lat_hist_io_start); + /* rq_data_dir() => true if WRITE */ + blk_update_latency_hist(&hba->io_lat_s, + (rq_data_dir(req) == READ), + delta_us); + } + } /* Do not touch lrbp after scsi done */ cmd->scsi_done(cmd); __ufshcd_release(hba); @@ -6186,6 +6215,54 @@ int ufshcd_shutdown(struct ufs_hba *hba) } EXPORT_SYMBOL(ufshcd_shutdown); +/* + * Values permitted 0, 1, 2. + * 0 -> Disable IO latency histograms (default) + * 1 -> Enable IO latency histograms + * 2 -> Zero out IO latency histograms + */ +static ssize_t +latency_hist_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + long value; + + if (kstrtol(buf, 0, &value)) + return -EINVAL; + if (value == BLK_IO_LAT_HIST_ZERO) + blk_zero_latency_hist(&hba->io_lat_s); + else if (value == BLK_IO_LAT_HIST_ENABLE || + value == BLK_IO_LAT_HIST_DISABLE) + hba->latency_hist_enabled = value; + return count; +} + +ssize_t +latency_hist_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + + return blk_latency_hist_show(&hba->io_lat_s, buf); +} + +static DEVICE_ATTR(latency_hist, S_IRUGO | S_IWUSR, + latency_hist_show, latency_hist_store); + +static void +ufshcd_init_latency_hist(struct ufs_hba *hba) +{ + if (device_create_file(hba->dev, &dev_attr_latency_hist)) + dev_err(hba->dev, "Failed to create latency_hist sysfs entry\n"); +} + +static void +ufshcd_exit_latency_hist(struct ufs_hba *hba) +{ + device_create_file(hba->dev, &dev_attr_latency_hist); +} + /** * ufshcd_remove - de-allocate SCSI host and host memory space * data structure memory @@ -6201,6 +6278,7 @@ void ufshcd_remove(struct ufs_hba *hba) scsi_host_put(hba->host); ufshcd_exit_clk_gating(hba); + ufshcd_exit_latency_hist(hba); if (ufshcd_is_clkscaling_enabled(hba)) devfreq_remove_device(hba->devfreq); ufshcd_hba_exit(hba); @@ -6514,6 +6592,8 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) /* Hold auto suspend until async scan completes */ pm_runtime_get_sync(dev); + ufshcd_init_latency_hist(hba); + /* * The device-initialize-sequence hasn't been invoked yet. * Set the device to power-off state @@ -6528,6 +6608,7 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) scsi_remove_host(hba->host); exit_gating: ufshcd_exit_clk_gating(hba); + ufshcd_exit_latency_hist(hba); out_disable: hba->is_irq_enabled = false; scsi_host_put(host); diff --git a/drivers/scsi/ufs/ufshcd.h b/drivers/scsi/ufs/ufshcd.h index 430bef11129324..06c4027971e75d 100644 --- a/drivers/scsi/ufs/ufshcd.h +++ b/drivers/scsi/ufs/ufshcd.h @@ -547,6 +547,8 @@ struct ufs_hba { enum bkops_status urgent_bkops_lvl; bool is_urgent_bkops_lvl_checked; + int latency_hist_enabled; + struct io_latency_state io_lat_s; }; /* Returns true if clocks can be gated. Otherwise false */ diff --git a/drivers/soc/ti/wkup_m3_ipc.c b/drivers/soc/ti/wkup_m3_ipc.c index 8823cc81ae4534..5bb376009d98b7 100644 --- a/drivers/soc/ti/wkup_m3_ipc.c +++ b/drivers/soc/ti/wkup_m3_ipc.c @@ -459,6 +459,7 @@ static int wkup_m3_ipc_probe(struct platform_device *pdev) if (IS_ERR(task)) { dev_err(dev, "can't create rproc_boot thread\n"); + ret = PTR_ERR(task); goto err_put_rproc; } diff --git a/drivers/spi/spi-orion.c b/drivers/spi/spi-orion.c index ded37025b44583..6b001c4a564086 100644 --- a/drivers/spi/spi-orion.c +++ b/drivers/spi/spi-orion.c @@ -138,37 +138,62 @@ static int orion_spi_baudrate_set(struct spi_device *spi, unsigned int speed) tclk_hz = clk_get_rate(orion_spi->clk); if (devdata->typ == ARMADA_SPI) { - unsigned int clk, spr, sppr, sppr2, err; - unsigned int best_spr, best_sppr, best_err; - - best_err = speed; - best_spr = 0; - best_sppr = 0; - - /* Iterate over the valid range looking for best fit */ - for (sppr = 0; sppr < 8; sppr++) { - sppr2 = 0x1 << sppr; - - spr = tclk_hz / sppr2; - spr = DIV_ROUND_UP(spr, speed); - if ((spr == 0) || (spr > 15)) - continue; - - clk = tclk_hz / (spr * sppr2); - err = speed - clk; - - if (err < best_err) { - best_spr = spr; - best_sppr = sppr; - best_err = err; - } - } + /* + * Given the core_clk (tclk_hz) and the target rate (speed) we + * determine the best values for SPR (in [0 .. 15]) and SPPR (in + * [0..7]) such that + * + * core_clk / (SPR * 2 ** SPPR) + * + * is as big as possible but not bigger than speed. + */ - if ((best_sppr == 0) && (best_spr == 0)) - return -EINVAL; + /* best integer divider: */ + unsigned divider = DIV_ROUND_UP(tclk_hz, speed); + unsigned spr, sppr; + + if (divider < 16) { + /* This is the easy case, divider is less than 16 */ + spr = divider; + sppr = 0; + + } else { + unsigned two_pow_sppr; + /* + * Find the highest bit set in divider. This and the + * three next bits define SPR (apart from rounding). + * SPPR is then the number of zero bits that must be + * appended: + */ + sppr = fls(divider) - 4; + + /* + * As SPR only has 4 bits, we have to round divider up + * to the next multiple of 2 ** sppr. + */ + two_pow_sppr = 1 << sppr; + divider = (divider + two_pow_sppr - 1) & -two_pow_sppr; + + /* + * recalculate sppr as rounding up divider might have + * increased it enough to change the position of the + * highest set bit. In this case the bit that now + * doesn't make it into SPR is 0, so there is no need to + * round again. + */ + sppr = fls(divider) - 4; + spr = divider >> sppr; + + /* + * Now do range checking. SPR is constructed to have a + * width of 4 bits, so this is fine for sure. So we + * still need to check for sppr to fit into 3 bits: + */ + if (sppr > 7) + return -EINVAL; + } - prescale = ((best_sppr & 0x6) << 5) | - ((best_sppr & 0x1) << 4) | best_spr; + prescale = ((sppr & 0x6) << 5) | ((sppr & 0x1) << 4) | spr; } else { /* * the supported rates are: 4,6,8...30 diff --git a/drivers/spi/spi-pxa2xx.c b/drivers/spi/spi-pxa2xx.c index dd7b5b47291d55..d6239fa718be9e 100644 --- a/drivers/spi/spi-pxa2xx.c +++ b/drivers/spi/spi-pxa2xx.c @@ -1690,6 +1690,7 @@ static int pxa2xx_spi_probe(struct platform_device *pdev) pxa2xx_spi_write(drv_data, SSCR1, tmp); tmp = SSCR0_SCR(2) | SSCR0_Motorola | SSCR0_DataSize(8); pxa2xx_spi_write(drv_data, SSCR0, tmp); + break; default: tmp = SSCR1_RxTresh(RX_THRESH_DFLT) | SSCR1_TxTresh(TX_THRESH_DFLT); diff --git a/drivers/spi/spidev.c b/drivers/spi/spidev.c index 2e05046f866bd0..d77267ea0196c5 100644 --- a/drivers/spi/spidev.c +++ b/drivers/spi/spidev.c @@ -753,8 +753,8 @@ static int spidev_probe(struct spi_device *spi) */ if (spi->dev.of_node && !of_match_device(spidev_dt_ids, &spi->dev)) { dev_err(&spi->dev, "buggy DT: spidev listed directly in DT\n"); - WARN_ON(spi->dev.of_node && - !of_match_device(spidev_dt_ids, &spi->dev)); +// WARN_ON(spi->dev.of_node && +// !of_match_device(spidev_dt_ids, &spi->dev)); } spidev_probe_acpi(spi); diff --git a/drivers/ssb/pci.c b/drivers/ssb/pci.c index 0f28c08fcb3c76..77b551da57288c 100644 --- a/drivers/ssb/pci.c +++ b/drivers/ssb/pci.c @@ -909,6 +909,7 @@ static int ssb_pci_sprom_get(struct ssb_bus *bus, if (err) { ssb_warn("WARNING: Using fallback SPROM failed (err %d)\n", err); + goto out_free; } else { ssb_dbg("Using SPROM revision %d provided by platform\n", sprom->revision); diff --git a/drivers/staging/android/Kconfig b/drivers/staging/android/Kconfig index 6c00d6f765c669..0c0c092ce354ac 100644 --- a/drivers/staging/android/Kconfig +++ b/drivers/staging/android/Kconfig @@ -14,18 +14,71 @@ config ASHMEM It is, in theory, a good memory allocator for low-memory devices, because it can discard shared memory units when under memory pressure. +config ANDROID_TIMED_OUTPUT + bool "Timed output class driver" + default y + +config ANDROID_TIMED_GPIO + tristate "Android timed gpio driver" + depends on GPIOLIB || COMPILE_TEST + depends on ANDROID_TIMED_OUTPUT + default n + ---help--- + Unlike generic gpio is to allow programs to access and manipulate gpio + registers from user space, timed output/gpio is a system to allow changing + a gpio pin and restore it automatically after a specified timeout. + config ANDROID_LOW_MEMORY_KILLER bool "Android Low Memory Killer" ---help--- Registers processes to be killed when low memory conditions, this is useful as there is no particular swap space on android. - The registered process will kill according to the priorities in android init + The registered process will kills according to the priorities in android init scripts (/init.rc), and it defines priority values with minimum free memory size for each priority. +config ANDROID_LOW_MEMORY_KILLER_AUTODETECT_OOM_ADJ_VALUES + bool "Android Low Memory Killer: detect oom_adj values" + depends on ANDROID_LOW_MEMORY_KILLER + default y + ---help--- + Detect oom_adj values written to + /sys/module/lowmemorykiller/parameters/adj and convert them + to oom_score_adj values. + +config SYNC + bool "Synchronization framework" + default n + select ANON_INODES + select DMA_SHARED_BUFFER + ---help--- + This option enables the framework for synchronization between multiple + drivers. Sync implementations can take advantage of hardware + synchronization built into devices like GPUs. + +config SW_SYNC + bool "Software synchronization objects" + default n + depends on SYNC + ---help--- + A sync object driver that uses a 32bit counter to coordinate + synchronization. Useful when there is no hardware primitive backing + the synchronization. + +config SW_SYNC_USER + bool "Userspace API for SW_SYNC" + default n + depends on SW_SYNC + ---help--- + Provides a user space API to the sw sync object. + *WARNING* improper use of this can result in deadlocking kernel + drivers from userspace. + source "drivers/staging/android/ion/Kconfig" +source "drivers/staging/android/fiq_debugger/Kconfig" + endif # if ANDROID endmenu diff --git a/drivers/staging/android/Makefile b/drivers/staging/android/Makefile index 7ed1be7989097d..b2a046f869b089 100644 --- a/drivers/staging/android/Makefile +++ b/drivers/staging/android/Makefile @@ -1,6 +1,9 @@ ccflags-y += -I$(src) # needed for trace events obj-y += ion/ +obj-$(CONFIG_FIQ_DEBUGGER) += fiq_debugger/ obj-$(CONFIG_ASHMEM) += ashmem.o obj-$(CONFIG_ANDROID_LOW_MEMORY_KILLER) += lowmemorykiller.o +obj-$(CONFIG_SYNC) += sync.o sync_debug.o +obj-$(CONFIG_SW_SYNC) += sw_sync.o diff --git a/drivers/staging/android/ashmem.c b/drivers/staging/android/ashmem.c index ca9a53c03f0f9a..3a52b29efab999 100644 --- a/drivers/staging/android/ashmem.c +++ b/drivers/staging/android/ashmem.c @@ -409,22 +409,14 @@ static int ashmem_mmap(struct file *file, struct vm_area_struct *vma) } get_file(asma->file); - /* - * XXX - Reworked to use shmem_zero_setup() instead of - * shmem_set_file while we're in staging. -jstultz - */ - if (vma->vm_flags & VM_SHARED) { - ret = shmem_zero_setup(vma); - if (ret) { - fput(asma->file); - goto out; - } + if (vma->vm_flags & VM_SHARED) + shmem_set_file(vma, asma->file); + else { + if (vma->vm_file) + fput(vma->vm_file); + vma->vm_file = asma->file; } - if (vma->vm_file) - fput(vma->vm_file); - vma->vm_file = asma->file; - out: mutex_unlock(&ashmem_mutex); return ret; @@ -461,9 +453,9 @@ ashmem_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) loff_t start = range->pgstart * PAGE_SIZE; loff_t end = (range->pgend + 1) * PAGE_SIZE; - vfs_fallocate(range->asma->file, - FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, - start, end - start); + range->asma->file->f_op->fallocate(range->asma->file, + FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + start, end - start); range->purged = ASHMEM_WAS_PURGED; lru_del(range); diff --git a/drivers/staging/android/fiq_debugger/Kconfig b/drivers/staging/android/fiq_debugger/Kconfig new file mode 100644 index 00000000000000..60fc224d4efc78 --- /dev/null +++ b/drivers/staging/android/fiq_debugger/Kconfig @@ -0,0 +1,58 @@ +config FIQ_DEBUGGER + bool "FIQ Mode Serial Debugger" + default n + depends on ARM || ARM64 + help + The FIQ serial debugger can accept commands even when the + kernel is unresponsive due to being stuck with interrupts + disabled. + +config FIQ_DEBUGGER_NO_SLEEP + bool "Keep serial debugger active" + depends on FIQ_DEBUGGER + default n + help + Enables the serial debugger at boot. Passing + fiq_debugger.no_sleep on the kernel commandline will + override this config option. + +config FIQ_DEBUGGER_WAKEUP_IRQ_ALWAYS_ON + bool "Don't disable wakeup IRQ when debugger is active" + depends on FIQ_DEBUGGER + default n + help + Don't disable the wakeup irq when enabling the uart clock. This will + cause extra interrupts, but it makes the serial debugger usable with + on some MSM radio builds that ignore the uart clock request in power + collapse. + +config FIQ_DEBUGGER_CONSOLE + bool "Console on FIQ Serial Debugger port" + depends on FIQ_DEBUGGER + default n + help + Enables a console so that printk messages are displayed on + the debugger serial port as the occur. + +config FIQ_DEBUGGER_CONSOLE_DEFAULT_ENABLE + bool "Put the FIQ debugger into console mode by default" + depends on FIQ_DEBUGGER_CONSOLE + default n + help + If enabled, this puts the fiq debugger into console mode by default. + Otherwise, the fiq debugger will start out in debug mode. + +config FIQ_DEBUGGER_UART_OVERLAY + bool "Install uart DT overlay" + depends on FIQ_DEBUGGER + select OF_OVERLAY + default n + help + If enabled, fiq debugger is calling fiq_debugger_uart_overlay() + that will apply overlay uart_overlay@0 to disable proper uart. + +config FIQ_WATCHDOG + bool + select FIQ_DEBUGGER + select PSTORE_RAM + default n diff --git a/drivers/staging/android/fiq_debugger/Makefile b/drivers/staging/android/fiq_debugger/Makefile new file mode 100644 index 00000000000000..a7ca4871cad31e --- /dev/null +++ b/drivers/staging/android/fiq_debugger/Makefile @@ -0,0 +1,4 @@ +obj-y += fiq_debugger.o +obj-$(CONFIG_ARM) += fiq_debugger_arm.o +obj-$(CONFIG_ARM64) += fiq_debugger_arm64.o +obj-$(CONFIG_FIQ_WATCHDOG) += fiq_watchdog.o diff --git a/drivers/staging/android/fiq_debugger/fiq_debugger.c b/drivers/staging/android/fiq_debugger/fiq_debugger.c new file mode 100644 index 00000000000000..675b974b2a6ec8 --- /dev/null +++ b/drivers/staging/android/fiq_debugger/fiq_debugger.c @@ -0,0 +1,1246 @@ +/* + * drivers/staging/android/fiq_debugger.c + * + * Serial Debugger Interface accessed through an FIQ interrupt. + * + * Copyright (C) 2008 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CONFIG_FIQ_GLUE +#include +#endif + +#ifdef CONFIG_FIQ_DEBUGGER_UART_OVERLAY +#include +#endif + +#include + +#include "fiq_debugger.h" +#include "fiq_debugger_priv.h" +#include "fiq_debugger_ringbuf.h" + +#define DEBUG_MAX 64 +#define MAX_UNHANDLED_FIQ_COUNT 1000000 + +#define MAX_FIQ_DEBUGGER_PORTS 4 + +struct fiq_debugger_state { +#ifdef CONFIG_FIQ_GLUE + struct fiq_glue_handler handler; +#endif + struct fiq_debugger_output output; + + int fiq; + int uart_irq; + int signal_irq; + int wakeup_irq; + bool wakeup_irq_no_set_wake; + struct clk *clk; + struct fiq_debugger_pdata *pdata; + struct platform_device *pdev; + + char debug_cmd[DEBUG_MAX]; + int debug_busy; + int debug_abort; + + char debug_buf[DEBUG_MAX]; + int debug_count; + + bool no_sleep; + bool debug_enable; + bool ignore_next_wakeup_irq; + struct timer_list sleep_timer; + spinlock_t sleep_timer_lock; + bool uart_enabled; + struct wakeup_source debugger_wake_src; + bool console_enable; + int current_cpu; + atomic_t unhandled_fiq_count; + bool in_fiq; + + struct work_struct work; + spinlock_t work_lock; + char work_cmd[DEBUG_MAX]; + +#ifdef CONFIG_FIQ_DEBUGGER_CONSOLE + spinlock_t console_lock; + struct console console; + struct tty_port tty_port; + struct fiq_debugger_ringbuf *tty_rbuf; + bool syslog_dumping; +#endif + + unsigned int last_irqs[NR_IRQS]; + unsigned int last_local_timer_irqs[NR_CPUS]; +}; + +#ifdef CONFIG_FIQ_DEBUGGER_CONSOLE +struct tty_driver *fiq_tty_driver; +#endif + +#ifdef CONFIG_FIQ_DEBUGGER_NO_SLEEP +static bool initial_no_sleep = true; +#else +static bool initial_no_sleep; +#endif + +#ifdef CONFIG_FIQ_DEBUGGER_CONSOLE_DEFAULT_ENABLE +static bool initial_debug_enable = true; +static bool initial_console_enable = true; +#else +static bool initial_debug_enable; +static bool initial_console_enable; +#endif + +static bool fiq_kgdb_enable; +static bool fiq_debugger_disable; + +module_param_named(no_sleep, initial_no_sleep, bool, 0644); +module_param_named(debug_enable, initial_debug_enable, bool, 0644); +module_param_named(console_enable, initial_console_enable, bool, 0644); +module_param_named(kgdb_enable, fiq_kgdb_enable, bool, 0644); +module_param_named(disable, fiq_debugger_disable, bool, 0644); + +#ifdef CONFIG_FIQ_DEBUGGER_WAKEUP_IRQ_ALWAYS_ON +static inline +void fiq_debugger_enable_wakeup_irq(struct fiq_debugger_state *state) {} +static inline +void fiq_debugger_disable_wakeup_irq(struct fiq_debugger_state *state) {} +#else +static inline +void fiq_debugger_enable_wakeup_irq(struct fiq_debugger_state *state) +{ + if (state->wakeup_irq < 0) + return; + enable_irq(state->wakeup_irq); + if (!state->wakeup_irq_no_set_wake) + enable_irq_wake(state->wakeup_irq); +} +static inline +void fiq_debugger_disable_wakeup_irq(struct fiq_debugger_state *state) +{ + if (state->wakeup_irq < 0) + return; + disable_irq_nosync(state->wakeup_irq); + if (!state->wakeup_irq_no_set_wake) + disable_irq_wake(state->wakeup_irq); +} +#endif + +static inline bool fiq_debugger_have_fiq(struct fiq_debugger_state *state) +{ + return (state->fiq >= 0); +} + +#ifdef CONFIG_FIQ_GLUE +static void fiq_debugger_force_irq(struct fiq_debugger_state *state) +{ + unsigned int irq = state->signal_irq; + + if (WARN_ON(!fiq_debugger_have_fiq(state))) + return; + if (state->pdata->force_irq) { + state->pdata->force_irq(state->pdev, irq); + } else { + struct irq_chip *chip = irq_get_chip(irq); + if (chip && chip->irq_retrigger) + chip->irq_retrigger(irq_get_irq_data(irq)); + } +} +#endif + +static void fiq_debugger_uart_enable(struct fiq_debugger_state *state) +{ + if (state->clk) + clk_enable(state->clk); + if (state->pdata->uart_enable) + state->pdata->uart_enable(state->pdev); +} + +static void fiq_debugger_uart_disable(struct fiq_debugger_state *state) +{ + if (state->pdata->uart_disable) + state->pdata->uart_disable(state->pdev); + if (state->clk) + clk_disable(state->clk); +} + +static void fiq_debugger_uart_flush(struct fiq_debugger_state *state) +{ + if (state->pdata->uart_flush) + state->pdata->uart_flush(state->pdev); +} + +static void fiq_debugger_putc(struct fiq_debugger_state *state, char c) +{ + state->pdata->uart_putc(state->pdev, c); +} + +static void fiq_debugger_puts(struct fiq_debugger_state *state, char *s) +{ + unsigned c; + while ((c = *s++)) { + if (c == '\n') + fiq_debugger_putc(state, '\r'); + fiq_debugger_putc(state, c); + } +} + +static void fiq_debugger_prompt(struct fiq_debugger_state *state) +{ + fiq_debugger_puts(state, "debug> "); +} + +static void fiq_debugger_dump_kernel_log(struct fiq_debugger_state *state) +{ + char buf[512]; + size_t len; + struct kmsg_dumper dumper = { .active = true }; + + + kmsg_dump_rewind_nolock(&dumper); + while (kmsg_dump_get_line_nolock(&dumper, true, buf, + sizeof(buf) - 1, &len)) { + buf[len] = 0; + fiq_debugger_puts(state, buf); + } +} + +static void fiq_debugger_printf(struct fiq_debugger_output *output, + const char *fmt, ...) +{ + struct fiq_debugger_state *state; + char buf[256]; + va_list ap; + + state = container_of(output, struct fiq_debugger_state, output); + va_start(ap, fmt); + vsnprintf(buf, sizeof(buf), fmt, ap); + va_end(ap); + + fiq_debugger_puts(state, buf); +} + +/* Safe outside fiq context */ +static int fiq_debugger_printf_nfiq(void *cookie, const char *fmt, ...) +{ + struct fiq_debugger_state *state = cookie; + char buf[256]; + va_list ap; + unsigned long irq_flags; + + va_start(ap, fmt); + vsnprintf(buf, 128, fmt, ap); + va_end(ap); + + local_irq_save(irq_flags); + fiq_debugger_puts(state, buf); + fiq_debugger_uart_flush(state); + local_irq_restore(irq_flags); + return state->debug_abort; +} + +static void fiq_debugger_dump_irqs(struct fiq_debugger_state *state) +{ + int n; + struct irq_desc *desc; + + fiq_debugger_printf(&state->output, + "irqnr total since-last status name\n"); + for_each_irq_desc(n, desc) { + struct irqaction *act = desc->action; + if (!act && !kstat_irqs(n)) + continue; + fiq_debugger_printf(&state->output, "%5d: %10u %11u %8x %s\n", n, + kstat_irqs(n), + kstat_irqs(n) - state->last_irqs[n], + desc->status_use_accessors, + (act && act->name) ? act->name : "???"); + state->last_irqs[n] = kstat_irqs(n); + } +} + +static void fiq_debugger_do_ps(struct fiq_debugger_state *state) +{ + struct task_struct *g; + struct task_struct *p; + unsigned task_state; + static const char stat_nam[] = "RSDTtZX"; + + fiq_debugger_printf(&state->output, "pid ppid prio task pc\n"); + read_lock(&tasklist_lock); + do_each_thread(g, p) { + task_state = p->state ? __ffs(p->state) + 1 : 0; + fiq_debugger_printf(&state->output, + "%5d %5d %4d ", p->pid, p->parent->pid, p->prio); + fiq_debugger_printf(&state->output, "%-13.13s %c", p->comm, + task_state >= sizeof(stat_nam) ? '?' : stat_nam[task_state]); + if (task_state == TASK_RUNNING) + fiq_debugger_printf(&state->output, " running\n"); + else + fiq_debugger_printf(&state->output, " %08lx\n", + thread_saved_pc(p)); + } while_each_thread(g, p); + read_unlock(&tasklist_lock); +} + +#ifdef CONFIG_FIQ_DEBUGGER_CONSOLE +static void fiq_debugger_begin_syslog_dump(struct fiq_debugger_state *state) +{ + state->syslog_dumping = true; +} + +static void fiq_debugger_end_syslog_dump(struct fiq_debugger_state *state) +{ + state->syslog_dumping = false; +} +#else +extern int do_syslog(int type, char __user *bug, int count); +static void fiq_debugger_begin_syslog_dump(struct fiq_debugger_state *state) +{ + do_syslog(5 /* clear */, NULL, 0); +} + +static void fiq_debugger_end_syslog_dump(struct fiq_debugger_state *state) +{ + fiq_debugger_dump_kernel_log(state); +} +#endif + +static void fiq_debugger_do_sysrq(struct fiq_debugger_state *state, char rq) +{ + if ((rq == 'g' || rq == 'G') && !fiq_kgdb_enable) { + fiq_debugger_printf(&state->output, "sysrq-g blocked\n"); + return; + } + fiq_debugger_begin_syslog_dump(state); + handle_sysrq(rq); + fiq_debugger_end_syslog_dump(state); +} + +#ifdef CONFIG_KGDB +static void fiq_debugger_do_kgdb(struct fiq_debugger_state *state) +{ + if (!fiq_kgdb_enable) { + fiq_debugger_printf(&state->output, "kgdb through fiq debugger not enabled\n"); + return; + } + + fiq_debugger_printf(&state->output, "enabling console and triggering kgdb\n"); + state->console_enable = true; + handle_sysrq('g'); +} +#endif + +static void fiq_debugger_schedule_work(struct fiq_debugger_state *state, + char *cmd) +{ + unsigned long flags; + + spin_lock_irqsave(&state->work_lock, flags); + if (state->work_cmd[0] != '\0') { + fiq_debugger_printf(&state->output, "work command processor busy\n"); + spin_unlock_irqrestore(&state->work_lock, flags); + return; + } + + strlcpy(state->work_cmd, cmd, sizeof(state->work_cmd)); + spin_unlock_irqrestore(&state->work_lock, flags); + + schedule_work(&state->work); +} + +static void fiq_debugger_work(struct work_struct *work) +{ + struct fiq_debugger_state *state; + char work_cmd[DEBUG_MAX]; + char *cmd; + unsigned long flags; + + state = container_of(work, struct fiq_debugger_state, work); + + spin_lock_irqsave(&state->work_lock, flags); + + strlcpy(work_cmd, state->work_cmd, sizeof(work_cmd)); + state->work_cmd[0] = '\0'; + + spin_unlock_irqrestore(&state->work_lock, flags); + + cmd = work_cmd; + if (!strncmp(cmd, "reboot", 6)) { + cmd += 6; + while (*cmd == ' ') + cmd++; + if (cmd != '\0') + kernel_restart(cmd); + else + kernel_restart(NULL); + } else { + fiq_debugger_printf(&state->output, "unknown work command '%s'\n", + work_cmd); + } +} + +/* This function CANNOT be called in FIQ context */ +static void fiq_debugger_irq_exec(struct fiq_debugger_state *state, char *cmd) +{ + if (!strcmp(cmd, "ps")) + fiq_debugger_do_ps(state); + if (!strcmp(cmd, "sysrq")) + fiq_debugger_do_sysrq(state, 'h'); + if (!strncmp(cmd, "sysrq ", 6)) + fiq_debugger_do_sysrq(state, cmd[6]); +#ifdef CONFIG_KGDB + if (!strcmp(cmd, "kgdb")) + fiq_debugger_do_kgdb(state); +#endif + if (!strncmp(cmd, "reboot", 6)) + fiq_debugger_schedule_work(state, cmd); +} + +static void fiq_debugger_help(struct fiq_debugger_state *state) +{ + fiq_debugger_printf(&state->output, + "FIQ Debugger commands:\n" + " pc PC status\n" + " regs Register dump\n" + " allregs Extended Register dump\n" + " bt Stack trace\n" + " reboot [] Reboot with command \n" + " reset [] Hard reset with command \n" + " irqs Interupt status\n" + " kmsg Kernel log\n" + " version Kernel version\n"); + fiq_debugger_printf(&state->output, + " sleep Allow sleep while in FIQ\n" + " nosleep Disable sleep while in FIQ\n" + " console Switch terminal to console\n" + " cpu Current CPU\n" + " cpu Switch to CPU\n"); + fiq_debugger_printf(&state->output, + " ps Process list\n" + " sysrq sysrq options\n" + " sysrq Execute sysrq with \n"); +#ifdef CONFIG_KGDB + fiq_debugger_printf(&state->output, + " kgdb Enter kernel debugger\n"); +#endif +} + +static void fiq_debugger_take_affinity(void *info) +{ + struct fiq_debugger_state *state = info; + struct cpumask cpumask; + + cpumask_clear(&cpumask); + cpumask_set_cpu(get_cpu(), &cpumask); + + irq_set_affinity(state->uart_irq, &cpumask); +} + +static void fiq_debugger_switch_cpu(struct fiq_debugger_state *state, int cpu) +{ + if (!fiq_debugger_have_fiq(state)) + smp_call_function_single(cpu, fiq_debugger_take_affinity, state, + false); + state->current_cpu = cpu; +} + +static bool fiq_debugger_fiq_exec(struct fiq_debugger_state *state, + const char *cmd, const struct pt_regs *regs, + void *svc_sp) +{ + bool signal_helper = false; + + if (!strcmp(cmd, "help") || !strcmp(cmd, "?")) { + fiq_debugger_help(state); + } else if (!strcmp(cmd, "pc")) { + fiq_debugger_dump_pc(&state->output, regs); + } else if (!strcmp(cmd, "regs")) { + fiq_debugger_dump_regs(&state->output, regs); + } else if (!strcmp(cmd, "allregs")) { + fiq_debugger_dump_allregs(&state->output, regs); + } else if (!strcmp(cmd, "bt")) { + fiq_debugger_dump_stacktrace(&state->output, regs, 100, svc_sp); + } else if (!strncmp(cmd, "reset", 5)) { + cmd += 5; + while (*cmd == ' ') + cmd++; + if (*cmd) { + char tmp_cmd[32]; + strlcpy(tmp_cmd, cmd, sizeof(tmp_cmd)); + machine_restart(tmp_cmd); + } else { + machine_restart(NULL); + } + } else if (!strcmp(cmd, "irqs")) { + fiq_debugger_dump_irqs(state); + } else if (!strcmp(cmd, "kmsg")) { + fiq_debugger_dump_kernel_log(state); + } else if (!strcmp(cmd, "version")) { + fiq_debugger_printf(&state->output, "%s\n", linux_banner); + } else if (!strcmp(cmd, "sleep")) { + state->no_sleep = false; + fiq_debugger_printf(&state->output, "enabling sleep\n"); + } else if (!strcmp(cmd, "nosleep")) { + state->no_sleep = true; + fiq_debugger_printf(&state->output, "disabling sleep\n"); + } else if (!strcmp(cmd, "console")) { + fiq_debugger_printf(&state->output, "console mode\n"); + fiq_debugger_uart_flush(state); + state->console_enable = true; + } else if (!strcmp(cmd, "cpu")) { + fiq_debugger_printf(&state->output, "cpu %d\n", state->current_cpu); + } else if (!strncmp(cmd, "cpu ", 4)) { + unsigned long cpu = 0; + if (kstrtoul(cmd + 4, 10, &cpu) == 0) + fiq_debugger_switch_cpu(state, cpu); + else + fiq_debugger_printf(&state->output, "invalid cpu\n"); + fiq_debugger_printf(&state->output, "cpu %d\n", state->current_cpu); + } else { + if (state->debug_busy) { + fiq_debugger_printf(&state->output, + "command processor busy. trying to abort.\n"); + state->debug_abort = -1; + } else { + strcpy(state->debug_cmd, cmd); + state->debug_busy = 1; + } + + return true; + } + if (!state->console_enable) + fiq_debugger_prompt(state); + + return signal_helper; +} + +static void fiq_debugger_sleep_timer_expired(unsigned long data) +{ + struct fiq_debugger_state *state = (struct fiq_debugger_state *)data; + unsigned long flags; + + spin_lock_irqsave(&state->sleep_timer_lock, flags); + if (state->uart_enabled && !state->no_sleep) { + if (state->debug_enable && !state->console_enable) { + state->debug_enable = false; + fiq_debugger_printf_nfiq(state, + "suspending fiq debugger\n"); + } + state->ignore_next_wakeup_irq = true; + fiq_debugger_uart_disable(state); + state->uart_enabled = false; + fiq_debugger_enable_wakeup_irq(state); + } + __pm_relax(&state->debugger_wake_src); + spin_unlock_irqrestore(&state->sleep_timer_lock, flags); +} + +static void fiq_debugger_handle_wakeup(struct fiq_debugger_state *state) +{ + unsigned long flags; + + spin_lock_irqsave(&state->sleep_timer_lock, flags); + if (state->wakeup_irq >= 0 && state->ignore_next_wakeup_irq) { + state->ignore_next_wakeup_irq = false; + } else if (!state->uart_enabled) { + __pm_stay_awake(&state->debugger_wake_src); + fiq_debugger_uart_enable(state); + state->uart_enabled = true; + fiq_debugger_disable_wakeup_irq(state); + mod_timer(&state->sleep_timer, jiffies + HZ / 2); + } + spin_unlock_irqrestore(&state->sleep_timer_lock, flags); +} + +static irqreturn_t fiq_debugger_wakeup_irq_handler(int irq, void *dev) +{ + struct fiq_debugger_state *state = dev; + + if (!state->no_sleep) + fiq_debugger_puts(state, "WAKEUP\n"); + fiq_debugger_handle_wakeup(state); + + return IRQ_HANDLED; +} + +static +void fiq_debugger_handle_console_irq_context(struct fiq_debugger_state *state) +{ +#if defined(CONFIG_FIQ_DEBUGGER_CONSOLE) + if (state->tty_port.ops) { + int i; + int count = fiq_debugger_ringbuf_level(state->tty_rbuf); + for (i = 0; i < count; i++) { + int c = fiq_debugger_ringbuf_peek(state->tty_rbuf, 0); + tty_insert_flip_char(&state->tty_port, c, TTY_NORMAL); + if (!fiq_debugger_ringbuf_consume(state->tty_rbuf, 1)) + pr_warn("fiq tty failed to consume byte\n"); + } + tty_flip_buffer_push(&state->tty_port); + } +#endif +} + +static void fiq_debugger_handle_irq_context(struct fiq_debugger_state *state) +{ + if (!state->no_sleep) { + unsigned long flags; + + spin_lock_irqsave(&state->sleep_timer_lock, flags); + __pm_stay_awake(&state->debugger_wake_src); + mod_timer(&state->sleep_timer, jiffies + HZ * 5); + spin_unlock_irqrestore(&state->sleep_timer_lock, flags); + } + fiq_debugger_handle_console_irq_context(state); + if (state->debug_busy) { + fiq_debugger_irq_exec(state, state->debug_cmd); + if (!state->console_enable) + fiq_debugger_prompt(state); + state->debug_busy = 0; + } +} + +static int fiq_debugger_getc(struct fiq_debugger_state *state) +{ + return state->pdata->uart_getc(state->pdev); +} + +static bool fiq_debugger_handle_uart_interrupt(struct fiq_debugger_state *state, + int this_cpu, const struct pt_regs *regs, void *svc_sp) +{ + int c; + static int last_c; + int count = 0; + bool signal_helper = false; + + if (this_cpu != state->current_cpu) { + if (state->in_fiq) + return false; + + if (atomic_inc_return(&state->unhandled_fiq_count) != + MAX_UNHANDLED_FIQ_COUNT) + return false; + + fiq_debugger_printf(&state->output, + "fiq_debugger: cpu %d not responding, " + "reverting to cpu %d\n", state->current_cpu, + this_cpu); + + atomic_set(&state->unhandled_fiq_count, 0); + fiq_debugger_switch_cpu(state, this_cpu); + return false; + } + + state->in_fiq = true; + + while ((c = fiq_debugger_getc(state)) != FIQ_DEBUGGER_NO_CHAR) { + count++; + if (!state->debug_enable) { + if ((c == 13) || (c == 10)) { + state->debug_enable = true; + state->debug_count = 0; + fiq_debugger_prompt(state); + } + } else if (c == FIQ_DEBUGGER_BREAK) { + state->console_enable = false; + fiq_debugger_puts(state, "fiq debugger mode\n"); + state->debug_count = 0; + fiq_debugger_prompt(state); +#ifdef CONFIG_FIQ_DEBUGGER_CONSOLE + } else if (state->console_enable && state->tty_rbuf) { + fiq_debugger_ringbuf_push(state->tty_rbuf, c); + signal_helper = true; +#endif + } else if ((c >= ' ') && (c < 127)) { + if (state->debug_count < (DEBUG_MAX - 1)) { + state->debug_buf[state->debug_count++] = c; + fiq_debugger_putc(state, c); + } + } else if ((c == 8) || (c == 127)) { + if (state->debug_count > 0) { + state->debug_count--; + fiq_debugger_putc(state, 8); + fiq_debugger_putc(state, ' '); + fiq_debugger_putc(state, 8); + } + } else if ((c == 13) || (c == 10)) { + if (c == '\r' || (c == '\n' && last_c != '\r')) { + fiq_debugger_putc(state, '\r'); + fiq_debugger_putc(state, '\n'); + } + if (state->debug_count) { + state->debug_buf[state->debug_count] = 0; + state->debug_count = 0; + signal_helper |= + fiq_debugger_fiq_exec(state, + state->debug_buf, + regs, svc_sp); + } else { + fiq_debugger_prompt(state); + } + } + last_c = c; + } + if (!state->console_enable) + fiq_debugger_uart_flush(state); + if (state->pdata->fiq_ack) + state->pdata->fiq_ack(state->pdev, state->fiq); + + /* poke sleep timer if necessary */ + if (state->debug_enable && !state->no_sleep) + signal_helper = true; + + atomic_set(&state->unhandled_fiq_count, 0); + state->in_fiq = false; + + return signal_helper; +} + +#ifdef CONFIG_FIQ_GLUE +static void fiq_debugger_fiq(struct fiq_glue_handler *h, + const struct pt_regs *regs, void *svc_sp) +{ + struct fiq_debugger_state *state = + container_of(h, struct fiq_debugger_state, handler); + unsigned int this_cpu = THREAD_INFO(svc_sp)->cpu; + bool need_irq; + + need_irq = fiq_debugger_handle_uart_interrupt(state, this_cpu, regs, + svc_sp); + if (need_irq) + fiq_debugger_force_irq(state); +} +#endif + +/* + * When not using FIQs, we only use this single interrupt as an entry point. + * This just effectively takes over the UART interrupt and does all the work + * in this context. + */ +static irqreturn_t fiq_debugger_uart_irq(int irq, void *dev) +{ + struct fiq_debugger_state *state = dev; + bool not_done; + + fiq_debugger_handle_wakeup(state); + + /* handle the debugger irq in regular context */ + not_done = fiq_debugger_handle_uart_interrupt(state, smp_processor_id(), + get_irq_regs(), + current_thread_info()); + if (not_done) + fiq_debugger_handle_irq_context(state); + + return IRQ_HANDLED; +} + +/* + * If FIQs are used, not everything can happen in fiq context. + * FIQ handler does what it can and then signals this interrupt to finish the + * job in irq context. + */ +static irqreturn_t fiq_debugger_signal_irq(int irq, void *dev) +{ + struct fiq_debugger_state *state = dev; + + if (state->pdata->force_irq_ack) + state->pdata->force_irq_ack(state->pdev, state->signal_irq); + + fiq_debugger_handle_irq_context(state); + + return IRQ_HANDLED; +} + +#ifdef CONFIG_FIQ_GLUE +static void fiq_debugger_resume(struct fiq_glue_handler *h) +{ + struct fiq_debugger_state *state = + container_of(h, struct fiq_debugger_state, handler); + if (state->pdata->uart_resume) + state->pdata->uart_resume(state->pdev); +} +#endif + +#if defined(CONFIG_FIQ_DEBUGGER_CONSOLE) +struct tty_driver *fiq_debugger_console_device(struct console *co, int *index) +{ + *index = co->index; + return fiq_tty_driver; +} + +static void fiq_debugger_console_write(struct console *co, + const char *s, unsigned int count) +{ + struct fiq_debugger_state *state; + unsigned long flags; + + state = container_of(co, struct fiq_debugger_state, console); + + if (!state->console_enable && !state->syslog_dumping) + return; + + fiq_debugger_uart_enable(state); + spin_lock_irqsave(&state->console_lock, flags); + while (count--) { + if (*s == '\n') + fiq_debugger_putc(state, '\r'); + fiq_debugger_putc(state, *s++); + } + fiq_debugger_uart_flush(state); + spin_unlock_irqrestore(&state->console_lock, flags); + fiq_debugger_uart_disable(state); +} + +static struct console fiq_debugger_console = { + .name = "ttyFIQ", + .device = fiq_debugger_console_device, + .write = fiq_debugger_console_write, + .flags = CON_PRINTBUFFER | CON_ANYTIME | CON_ENABLED, +}; + +int fiq_tty_open(struct tty_struct *tty, struct file *filp) +{ + int line = tty->index; + struct fiq_debugger_state **states = tty->driver->driver_state; + struct fiq_debugger_state *state = states[line]; + + return tty_port_open(&state->tty_port, tty, filp); +} + +void fiq_tty_close(struct tty_struct *tty, struct file *filp) +{ + tty_port_close(tty->port, tty, filp); +} + +int fiq_tty_write(struct tty_struct *tty, const unsigned char *buf, int count) +{ + int i; + int line = tty->index; + struct fiq_debugger_state **states = tty->driver->driver_state; + struct fiq_debugger_state *state = states[line]; + + if (!state->console_enable) + return count; + + fiq_debugger_uart_enable(state); + spin_lock_irq(&state->console_lock); + for (i = 0; i < count; i++) + fiq_debugger_putc(state, *buf++); + spin_unlock_irq(&state->console_lock); + fiq_debugger_uart_disable(state); + + return count; +} + +int fiq_tty_write_room(struct tty_struct *tty) +{ + return 16; +} + +#ifdef CONFIG_CONSOLE_POLL +static int fiq_tty_poll_init(struct tty_driver *driver, int line, char *options) +{ + return 0; +} + +static int fiq_tty_poll_get_char(struct tty_driver *driver, int line) +{ + struct fiq_debugger_state **states = driver->driver_state; + struct fiq_debugger_state *state = states[line]; + int c = NO_POLL_CHAR; + + fiq_debugger_uart_enable(state); + if (fiq_debugger_have_fiq(state)) { + int count = fiq_debugger_ringbuf_level(state->tty_rbuf); + if (count > 0) { + c = fiq_debugger_ringbuf_peek(state->tty_rbuf, 0); + fiq_debugger_ringbuf_consume(state->tty_rbuf, 1); + } + } else { + c = fiq_debugger_getc(state); + if (c == FIQ_DEBUGGER_NO_CHAR) + c = NO_POLL_CHAR; + } + fiq_debugger_uart_disable(state); + + return c; +} + +static void fiq_tty_poll_put_char(struct tty_driver *driver, int line, char ch) +{ + struct fiq_debugger_state **states = driver->driver_state; + struct fiq_debugger_state *state = states[line]; + fiq_debugger_uart_enable(state); + fiq_debugger_putc(state, ch); + fiq_debugger_uart_disable(state); +} +#endif + +static const struct tty_port_operations fiq_tty_port_ops; + +static const struct tty_operations fiq_tty_driver_ops = { + .write = fiq_tty_write, + .write_room = fiq_tty_write_room, + .open = fiq_tty_open, + .close = fiq_tty_close, +#ifdef CONFIG_CONSOLE_POLL + .poll_init = fiq_tty_poll_init, + .poll_get_char = fiq_tty_poll_get_char, + .poll_put_char = fiq_tty_poll_put_char, +#endif +}; + +static int fiq_debugger_tty_init(void) +{ + int ret; + struct fiq_debugger_state **states = NULL; + + states = kzalloc(sizeof(*states) * MAX_FIQ_DEBUGGER_PORTS, GFP_KERNEL); + if (!states) { + pr_err("Failed to allocate fiq debugger state structres\n"); + return -ENOMEM; + } + + fiq_tty_driver = alloc_tty_driver(MAX_FIQ_DEBUGGER_PORTS); + if (!fiq_tty_driver) { + pr_err("Failed to allocate fiq debugger tty\n"); + ret = -ENOMEM; + goto err_free_state; + } + + fiq_tty_driver->owner = THIS_MODULE; + fiq_tty_driver->driver_name = "fiq-debugger"; + fiq_tty_driver->name = "ttyFIQ"; + fiq_tty_driver->type = TTY_DRIVER_TYPE_SERIAL; + fiq_tty_driver->subtype = SERIAL_TYPE_NORMAL; + fiq_tty_driver->init_termios = tty_std_termios; + fiq_tty_driver->flags = TTY_DRIVER_REAL_RAW | + TTY_DRIVER_DYNAMIC_DEV; + fiq_tty_driver->driver_state = states; + + fiq_tty_driver->init_termios.c_cflag = + B115200 | CS8 | CREAD | HUPCL | CLOCAL; + fiq_tty_driver->init_termios.c_ispeed = 115200; + fiq_tty_driver->init_termios.c_ospeed = 115200; + + tty_set_operations(fiq_tty_driver, &fiq_tty_driver_ops); + + ret = tty_register_driver(fiq_tty_driver); + if (ret) { + pr_err("Failed to register fiq tty: %d\n", ret); + goto err_free_tty; + } + + pr_info("Registered FIQ tty driver\n"); + return 0; + +err_free_tty: + put_tty_driver(fiq_tty_driver); + fiq_tty_driver = NULL; +err_free_state: + kfree(states); + return ret; +} + +static int fiq_debugger_tty_init_one(struct fiq_debugger_state *state) +{ + int ret; + struct device *tty_dev; + struct fiq_debugger_state **states = fiq_tty_driver->driver_state; + + states[state->pdev->id] = state; + + state->tty_rbuf = fiq_debugger_ringbuf_alloc(1024); + if (!state->tty_rbuf) { + pr_err("Failed to allocate fiq debugger ringbuf\n"); + ret = -ENOMEM; + goto err; + } + + tty_port_init(&state->tty_port); + state->tty_port.ops = &fiq_tty_port_ops; + + tty_dev = tty_port_register_device(&state->tty_port, fiq_tty_driver, + state->pdev->id, &state->pdev->dev); + if (IS_ERR(tty_dev)) { + pr_err("Failed to register fiq debugger tty device\n"); + ret = PTR_ERR(tty_dev); + goto err; + } + + device_set_wakeup_capable(tty_dev, 1); + + pr_info("Registered fiq debugger ttyFIQ%d\n", state->pdev->id); + + return 0; + +err: + fiq_debugger_ringbuf_free(state->tty_rbuf); + state->tty_rbuf = NULL; + return ret; +} +#endif + +static int fiq_debugger_dev_suspend(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct fiq_debugger_state *state = platform_get_drvdata(pdev); + + if (state->pdata->uart_dev_suspend) + return state->pdata->uart_dev_suspend(pdev); + return 0; +} + +static int fiq_debugger_dev_resume(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct fiq_debugger_state *state = platform_get_drvdata(pdev); + + if (state->pdata->uart_dev_resume) + return state->pdata->uart_dev_resume(pdev); + return 0; +} + +static int fiq_debugger_probe(struct platform_device *pdev) +{ + int ret; + struct fiq_debugger_pdata *pdata = dev_get_platdata(&pdev->dev); + struct fiq_debugger_state *state; + int fiq; + int uart_irq; + + if (pdev->id >= MAX_FIQ_DEBUGGER_PORTS) + return -EINVAL; + + if (!pdata->uart_getc || !pdata->uart_putc) + return -EINVAL; + if ((pdata->uart_enable && !pdata->uart_disable) || + (!pdata->uart_enable && pdata->uart_disable)) + return -EINVAL; + + fiq = platform_get_irq_byname(pdev, "fiq"); + uart_irq = platform_get_irq_byname(pdev, "uart_irq"); + + /* uart_irq mode and fiq mode are mutually exclusive, but one of them + * is required */ + if ((uart_irq < 0 && fiq < 0) || (uart_irq >= 0 && fiq >= 0)) + return -EINVAL; + if (fiq >= 0 && !pdata->fiq_enable) + return -EINVAL; + + state = kzalloc(sizeof(*state), GFP_KERNEL); + state->output.printf = fiq_debugger_printf; + setup_timer(&state->sleep_timer, fiq_debugger_sleep_timer_expired, + (unsigned long)state); + state->pdata = pdata; + state->pdev = pdev; + state->no_sleep = initial_no_sleep; + state->debug_enable = initial_debug_enable; + state->console_enable = initial_console_enable; + + state->fiq = fiq; + state->uart_irq = uart_irq; + state->signal_irq = platform_get_irq_byname(pdev, "signal"); + state->wakeup_irq = platform_get_irq_byname(pdev, "wakeup"); + + INIT_WORK(&state->work, fiq_debugger_work); + spin_lock_init(&state->work_lock); + + platform_set_drvdata(pdev, state); + + spin_lock_init(&state->sleep_timer_lock); + + if (state->wakeup_irq < 0 && fiq_debugger_have_fiq(state)) + state->no_sleep = true; + state->ignore_next_wakeup_irq = !state->no_sleep; + + wakeup_source_init(&state->debugger_wake_src, "serial-debug"); + + state->clk = clk_get(&pdev->dev, NULL); + if (IS_ERR(state->clk)) + state->clk = NULL; + + /* do not call pdata->uart_enable here since uart_init may still + * need to do some initialization before uart_enable can work. + * So, only try to manage the clock during init. + */ + if (state->clk) + clk_enable(state->clk); + + if (pdata->uart_init) { + ret = pdata->uart_init(pdev); + if (ret) + goto err_uart_init; + } + + fiq_debugger_printf_nfiq(state, + "\n", + state->no_sleep ? "" : "twice "); + +#ifdef CONFIG_FIQ_GLUE + if (fiq_debugger_have_fiq(state)) { + state->handler.fiq = fiq_debugger_fiq; + state->handler.resume = fiq_debugger_resume; + ret = fiq_glue_register_handler(&state->handler); + if (ret) { + pr_err("%s: could not install fiq handler\n", __func__); + goto err_register_irq; + } + + pdata->fiq_enable(pdev, state->fiq, 1); + } else +#endif + { + ret = request_irq(state->uart_irq, fiq_debugger_uart_irq, + IRQF_NO_SUSPEND, "debug", state); + if (ret) { + pr_err("%s: could not install irq handler\n", __func__); + goto err_register_irq; + } + + /* for irq-only mode, we want this irq to wake us up, if it + * can. + */ + enable_irq_wake(state->uart_irq); + } + + if (state->clk) + clk_disable(state->clk); + + if (state->signal_irq >= 0) { + ret = request_irq(state->signal_irq, fiq_debugger_signal_irq, + IRQF_TRIGGER_RISING, "debug-signal", state); + if (ret) + pr_err("serial_debugger: could not install signal_irq"); + } + + if (state->wakeup_irq >= 0) { + ret = request_irq(state->wakeup_irq, + fiq_debugger_wakeup_irq_handler, + IRQF_TRIGGER_FALLING, + "debug-wakeup", state); + if (ret) { + pr_err("serial_debugger: " + "could not install wakeup irq\n"); + state->wakeup_irq = -1; + } else { + ret = enable_irq_wake(state->wakeup_irq); + if (ret) { + pr_err("serial_debugger: " + "could not enable wakeup\n"); + state->wakeup_irq_no_set_wake = true; + } + } + } + if (state->no_sleep) + fiq_debugger_handle_wakeup(state); + +#if defined(CONFIG_FIQ_DEBUGGER_CONSOLE) + spin_lock_init(&state->console_lock); + state->console = fiq_debugger_console; + state->console.index = pdev->id; + if (!console_set_on_cmdline) + add_preferred_console(state->console.name, + state->console.index, NULL); + register_console(&state->console); + fiq_debugger_tty_init_one(state); +#endif + return 0; + +err_register_irq: + if (pdata->uart_free) + pdata->uart_free(pdev); +err_uart_init: + if (state->clk) + clk_disable(state->clk); + if (state->clk) + clk_put(state->clk); + wakeup_source_trash(&state->debugger_wake_src); + platform_set_drvdata(pdev, NULL); + kfree(state); + return ret; +} + +static const struct dev_pm_ops fiq_debugger_dev_pm_ops = { + .suspend = fiq_debugger_dev_suspend, + .resume = fiq_debugger_dev_resume, +}; + +static struct platform_driver fiq_debugger_driver = { + .probe = fiq_debugger_probe, + .driver = { + .name = "fiq_debugger", + .pm = &fiq_debugger_dev_pm_ops, + }, +}; + +#if defined(CONFIG_FIQ_DEBUGGER_UART_OVERLAY) +int fiq_debugger_uart_overlay(void) +{ + struct device_node *onp = of_find_node_by_path("/uart_overlay@0"); + int ret; + + if (!onp) { + pr_err("serial_debugger: uart overlay not found\n"); + return -ENODEV; + } + + ret = of_overlay_create(onp); + if (ret < 0) { + pr_err("serial_debugger: fail to create overlay: %d\n", ret); + of_node_put(onp); + return ret; + } + + pr_info("serial_debugger: uart overlay applied\n"); + return 0; +} +#endif + +static int __init fiq_debugger_init(void) +{ + if (fiq_debugger_disable) { + pr_err("serial_debugger: disabled\n"); + return -ENODEV; + } +#if defined(CONFIG_FIQ_DEBUGGER_CONSOLE) + fiq_debugger_tty_init(); +#endif +#if defined(CONFIG_FIQ_DEBUGGER_UART_OVERLAY) + fiq_debugger_uart_overlay(); +#endif + return platform_driver_register(&fiq_debugger_driver); +} + +postcore_initcall(fiq_debugger_init); diff --git a/drivers/staging/android/fiq_debugger/fiq_debugger.h b/drivers/staging/android/fiq_debugger/fiq_debugger.h new file mode 100644 index 00000000000000..c9ec4f8db086f4 --- /dev/null +++ b/drivers/staging/android/fiq_debugger/fiq_debugger.h @@ -0,0 +1,64 @@ +/* + * drivers/staging/android/fiq_debugger/fiq_debugger.h + * + * Copyright (C) 2010 Google, Inc. + * Author: Colin Cross + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#ifndef _ARCH_ARM_MACH_TEGRA_FIQ_DEBUGGER_H_ +#define _ARCH_ARM_MACH_TEGRA_FIQ_DEBUGGER_H_ + +#include + +#define FIQ_DEBUGGER_NO_CHAR NO_POLL_CHAR +#define FIQ_DEBUGGER_BREAK 0x00ff0100 + +#define FIQ_DEBUGGER_FIQ_IRQ_NAME "fiq" +#define FIQ_DEBUGGER_SIGNAL_IRQ_NAME "signal" +#define FIQ_DEBUGGER_WAKEUP_IRQ_NAME "wakeup" + +/** + * struct fiq_debugger_pdata - fiq debugger platform data + * @uart_resume: used to restore uart state right before enabling + * the fiq. + * @uart_enable: Do the work necessary to communicate with the uart + * hw (enable clocks, etc.). This must be ref-counted. + * @uart_disable: Do the work necessary to disable the uart hw + * (disable clocks, etc.). This must be ref-counted. + * @uart_dev_suspend: called during PM suspend, generally not needed + * for real fiq mode debugger. + * @uart_dev_resume: called during PM resume, generally not needed + * for real fiq mode debugger. + */ +struct fiq_debugger_pdata { + int (*uart_init)(struct platform_device *pdev); + void (*uart_free)(struct platform_device *pdev); + int (*uart_resume)(struct platform_device *pdev); + int (*uart_getc)(struct platform_device *pdev); + void (*uart_putc)(struct platform_device *pdev, unsigned int c); + void (*uart_flush)(struct platform_device *pdev); + void (*uart_enable)(struct platform_device *pdev); + void (*uart_disable)(struct platform_device *pdev); + + int (*uart_dev_suspend)(struct platform_device *pdev); + int (*uart_dev_resume)(struct platform_device *pdev); + + void (*fiq_enable)(struct platform_device *pdev, unsigned int fiq, + bool enable); + void (*fiq_ack)(struct platform_device *pdev, unsigned int fiq); + + void (*force_irq)(struct platform_device *pdev, unsigned int irq); + void (*force_irq_ack)(struct platform_device *pdev, unsigned int irq); +}; + +#endif diff --git a/drivers/staging/android/fiq_debugger/fiq_debugger_arm.c b/drivers/staging/android/fiq_debugger/fiq_debugger_arm.c new file mode 100644 index 00000000000000..8b3e0137be1a0b --- /dev/null +++ b/drivers/staging/android/fiq_debugger/fiq_debugger_arm.c @@ -0,0 +1,240 @@ +/* + * Copyright (C) 2014 Google, Inc. + * Author: Colin Cross + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include + +#include + +#include "fiq_debugger_priv.h" + +static char *mode_name(unsigned cpsr) +{ + switch (cpsr & MODE_MASK) { + case USR_MODE: return "USR"; + case FIQ_MODE: return "FIQ"; + case IRQ_MODE: return "IRQ"; + case SVC_MODE: return "SVC"; + case ABT_MODE: return "ABT"; + case UND_MODE: return "UND"; + case SYSTEM_MODE: return "SYS"; + default: return "???"; + } +} + +void fiq_debugger_dump_pc(struct fiq_debugger_output *output, + const struct pt_regs *regs) +{ + output->printf(output, " pc %08x cpsr %08x mode %s\n", + regs->ARM_pc, regs->ARM_cpsr, mode_name(regs->ARM_cpsr)); +} + +void fiq_debugger_dump_regs(struct fiq_debugger_output *output, + const struct pt_regs *regs) +{ + output->printf(output, + " r0 %08x r1 %08x r2 %08x r3 %08x\n", + regs->ARM_r0, regs->ARM_r1, regs->ARM_r2, regs->ARM_r3); + output->printf(output, + " r4 %08x r5 %08x r6 %08x r7 %08x\n", + regs->ARM_r4, regs->ARM_r5, regs->ARM_r6, regs->ARM_r7); + output->printf(output, + " r8 %08x r9 %08x r10 %08x r11 %08x mode %s\n", + regs->ARM_r8, regs->ARM_r9, regs->ARM_r10, regs->ARM_fp, + mode_name(regs->ARM_cpsr)); + output->printf(output, + " ip %08x sp %08x lr %08x pc %08x cpsr %08x\n", + regs->ARM_ip, regs->ARM_sp, regs->ARM_lr, regs->ARM_pc, + regs->ARM_cpsr); +} + +struct mode_regs { + unsigned long sp_svc; + unsigned long lr_svc; + unsigned long spsr_svc; + + unsigned long sp_abt; + unsigned long lr_abt; + unsigned long spsr_abt; + + unsigned long sp_und; + unsigned long lr_und; + unsigned long spsr_und; + + unsigned long sp_irq; + unsigned long lr_irq; + unsigned long spsr_irq; + + unsigned long r8_fiq; + unsigned long r9_fiq; + unsigned long r10_fiq; + unsigned long r11_fiq; + unsigned long r12_fiq; + unsigned long sp_fiq; + unsigned long lr_fiq; + unsigned long spsr_fiq; +}; + +static void __naked get_mode_regs(struct mode_regs *regs) +{ + asm volatile ( + "mrs r1, cpsr\n" + "msr cpsr_c, #0xd3 @(SVC_MODE | PSR_I_BIT | PSR_F_BIT)\n" + "stmia r0!, {r13 - r14}\n" + "mrs r2, spsr\n" + "msr cpsr_c, #0xd7 @(ABT_MODE | PSR_I_BIT | PSR_F_BIT)\n" + "stmia r0!, {r2, r13 - r14}\n" + "mrs r2, spsr\n" + "msr cpsr_c, #0xdb @(UND_MODE | PSR_I_BIT | PSR_F_BIT)\n" + "stmia r0!, {r2, r13 - r14}\n" + "mrs r2, spsr\n" + "msr cpsr_c, #0xd2 @(IRQ_MODE | PSR_I_BIT | PSR_F_BIT)\n" + "stmia r0!, {r2, r13 - r14}\n" + "mrs r2, spsr\n" + "msr cpsr_c, #0xd1 @(FIQ_MODE | PSR_I_BIT | PSR_F_BIT)\n" + "stmia r0!, {r2, r8 - r14}\n" + "mrs r2, spsr\n" + "stmia r0!, {r2}\n" + "msr cpsr_c, r1\n" + "bx lr\n"); +} + + +void fiq_debugger_dump_allregs(struct fiq_debugger_output *output, + const struct pt_regs *regs) +{ + struct mode_regs mode_regs; + unsigned long mode = regs->ARM_cpsr & MODE_MASK; + + fiq_debugger_dump_regs(output, regs); + get_mode_regs(&mode_regs); + + output->printf(output, + "%csvc: sp %08x lr %08x spsr %08x\n", + mode == SVC_MODE ? '*' : ' ', + mode_regs.sp_svc, mode_regs.lr_svc, mode_regs.spsr_svc); + output->printf(output, + "%cabt: sp %08x lr %08x spsr %08x\n", + mode == ABT_MODE ? '*' : ' ', + mode_regs.sp_abt, mode_regs.lr_abt, mode_regs.spsr_abt); + output->printf(output, + "%cund: sp %08x lr %08x spsr %08x\n", + mode == UND_MODE ? '*' : ' ', + mode_regs.sp_und, mode_regs.lr_und, mode_regs.spsr_und); + output->printf(output, + "%cirq: sp %08x lr %08x spsr %08x\n", + mode == IRQ_MODE ? '*' : ' ', + mode_regs.sp_irq, mode_regs.lr_irq, mode_regs.spsr_irq); + output->printf(output, + "%cfiq: r8 %08x r9 %08x r10 %08x r11 %08x r12 %08x\n", + mode == FIQ_MODE ? '*' : ' ', + mode_regs.r8_fiq, mode_regs.r9_fiq, mode_regs.r10_fiq, + mode_regs.r11_fiq, mode_regs.r12_fiq); + output->printf(output, + " fiq: sp %08x lr %08x spsr %08x\n", + mode_regs.sp_fiq, mode_regs.lr_fiq, mode_regs.spsr_fiq); +} + +struct stacktrace_state { + struct fiq_debugger_output *output; + unsigned int depth; +}; + +static int report_trace(struct stackframe *frame, void *d) +{ + struct stacktrace_state *sts = d; + + if (sts->depth) { + sts->output->printf(sts->output, + " pc: %p (%pF), lr %p (%pF), sp %p, fp %p\n", + frame->pc, frame->pc, frame->lr, frame->lr, + frame->sp, frame->fp); + sts->depth--; + return 0; + } + sts->output->printf(sts->output, " ...\n"); + + return sts->depth == 0; +} + +struct frame_tail { + struct frame_tail *fp; + unsigned long sp; + unsigned long lr; +} __attribute__((packed)); + +static struct frame_tail *user_backtrace(struct fiq_debugger_output *output, + struct frame_tail *tail) +{ + struct frame_tail buftail[2]; + + /* Also check accessibility of one struct frame_tail beyond */ + if (!access_ok(VERIFY_READ, tail, sizeof(buftail))) { + output->printf(output, " invalid frame pointer %p\n", + tail); + return NULL; + } + if (__copy_from_user_inatomic(buftail, tail, sizeof(buftail))) { + output->printf(output, + " failed to copy frame pointer %p\n", tail); + return NULL; + } + + output->printf(output, " %p\n", buftail[0].lr); + + /* frame pointers should strictly progress back up the stack + * (towards higher addresses) */ + if (tail >= buftail[0].fp) + return NULL; + + return buftail[0].fp-1; +} + +void fiq_debugger_dump_stacktrace(struct fiq_debugger_output *output, + const struct pt_regs *regs, unsigned int depth, void *ssp) +{ + struct frame_tail *tail; + struct thread_info *real_thread_info = THREAD_INFO(ssp); + struct stacktrace_state sts; + + sts.depth = depth; + sts.output = output; + *current_thread_info() = *real_thread_info; + + if (!current) + output->printf(output, "current NULL\n"); + else + output->printf(output, "pid: %d comm: %s\n", + current->pid, current->comm); + fiq_debugger_dump_regs(output, regs); + + if (!user_mode(regs)) { + struct stackframe frame; + frame.fp = regs->ARM_fp; + frame.sp = regs->ARM_sp; + frame.lr = regs->ARM_lr; + frame.pc = regs->ARM_pc; + output->printf(output, + " pc: %p (%pF), lr %p (%pF), sp %p, fp %p\n", + regs->ARM_pc, regs->ARM_pc, regs->ARM_lr, regs->ARM_lr, + regs->ARM_sp, regs->ARM_fp); + walk_stackframe(&frame, report_trace, &sts); + return; + } + + tail = ((struct frame_tail *) regs->ARM_fp) - 1; + while (depth-- && tail && !((unsigned long) tail & 3)) + tail = user_backtrace(output, tail); +} diff --git a/drivers/staging/android/fiq_debugger/fiq_debugger_arm64.c b/drivers/staging/android/fiq_debugger/fiq_debugger_arm64.c new file mode 100644 index 00000000000000..97246bcbcd6254 --- /dev/null +++ b/drivers/staging/android/fiq_debugger/fiq_debugger_arm64.c @@ -0,0 +1,202 @@ +/* + * Copyright (C) 2014 Google, Inc. + * Author: Colin Cross + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include + +#include "fiq_debugger_priv.h" + +static char *mode_name(const struct pt_regs *regs) +{ + if (compat_user_mode(regs)) { + return "USR"; + } else { + switch (processor_mode(regs)) { + case PSR_MODE_EL0t: return "EL0t"; + case PSR_MODE_EL1t: return "EL1t"; + case PSR_MODE_EL1h: return "EL1h"; + case PSR_MODE_EL2t: return "EL2t"; + case PSR_MODE_EL2h: return "EL2h"; + default: return "???"; + } + } +} + +void fiq_debugger_dump_pc(struct fiq_debugger_output *output, + const struct pt_regs *regs) +{ + output->printf(output, " pc %016lx cpsr %08lx mode %s\n", + regs->pc, regs->pstate, mode_name(regs)); +} + +void fiq_debugger_dump_regs_aarch32(struct fiq_debugger_output *output, + const struct pt_regs *regs) +{ + output->printf(output, " r0 %08x r1 %08x r2 %08x r3 %08x\n", + regs->compat_usr(0), regs->compat_usr(1), + regs->compat_usr(2), regs->compat_usr(3)); + output->printf(output, " r4 %08x r5 %08x r6 %08x r7 %08x\n", + regs->compat_usr(4), regs->compat_usr(5), + regs->compat_usr(6), regs->compat_usr(7)); + output->printf(output, " r8 %08x r9 %08x r10 %08x r11 %08x\n", + regs->compat_usr(8), regs->compat_usr(9), + regs->compat_usr(10), regs->compat_usr(11)); + output->printf(output, " ip %08x sp %08x lr %08x pc %08x\n", + regs->compat_usr(12), regs->compat_sp, + regs->compat_lr, regs->pc); + output->printf(output, " cpsr %08x (%s)\n", + regs->pstate, mode_name(regs)); +} + +void fiq_debugger_dump_regs_aarch64(struct fiq_debugger_output *output, + const struct pt_regs *regs) +{ + + output->printf(output, " x0 %016lx x1 %016lx\n", + regs->regs[0], regs->regs[1]); + output->printf(output, " x2 %016lx x3 %016lx\n", + regs->regs[2], regs->regs[3]); + output->printf(output, " x4 %016lx x5 %016lx\n", + regs->regs[4], regs->regs[5]); + output->printf(output, " x6 %016lx x7 %016lx\n", + regs->regs[6], regs->regs[7]); + output->printf(output, " x8 %016lx x9 %016lx\n", + regs->regs[8], regs->regs[9]); + output->printf(output, " x10 %016lx x11 %016lx\n", + regs->regs[10], regs->regs[11]); + output->printf(output, " x12 %016lx x13 %016lx\n", + regs->regs[12], regs->regs[13]); + output->printf(output, " x14 %016lx x15 %016lx\n", + regs->regs[14], regs->regs[15]); + output->printf(output, " x16 %016lx x17 %016lx\n", + regs->regs[16], regs->regs[17]); + output->printf(output, " x18 %016lx x19 %016lx\n", + regs->regs[18], regs->regs[19]); + output->printf(output, " x20 %016lx x21 %016lx\n", + regs->regs[20], regs->regs[21]); + output->printf(output, " x22 %016lx x23 %016lx\n", + regs->regs[22], regs->regs[23]); + output->printf(output, " x24 %016lx x25 %016lx\n", + regs->regs[24], regs->regs[25]); + output->printf(output, " x26 %016lx x27 %016lx\n", + regs->regs[26], regs->regs[27]); + output->printf(output, " x28 %016lx x29 %016lx\n", + regs->regs[28], regs->regs[29]); + output->printf(output, " x30 %016lx sp %016lx\n", + regs->regs[30], regs->sp); + output->printf(output, " pc %016lx cpsr %08x (%s)\n", + regs->pc, regs->pstate, mode_name(regs)); +} + +void fiq_debugger_dump_regs(struct fiq_debugger_output *output, + const struct pt_regs *regs) +{ + if (compat_user_mode(regs)) + fiq_debugger_dump_regs_aarch32(output, regs); + else + fiq_debugger_dump_regs_aarch64(output, regs); +} + +#define READ_SPECIAL_REG(x) ({ \ + u64 val; \ + asm volatile ("mrs %0, " # x : "=r"(val)); \ + val; \ +}) + +void fiq_debugger_dump_allregs(struct fiq_debugger_output *output, + const struct pt_regs *regs) +{ + u32 pstate = READ_SPECIAL_REG(CurrentEl); + bool in_el2 = (pstate & PSR_MODE_MASK) >= PSR_MODE_EL2t; + + fiq_debugger_dump_regs(output, regs); + + output->printf(output, " sp_el0 %016lx\n", + READ_SPECIAL_REG(sp_el0)); + + if (in_el2) + output->printf(output, " sp_el1 %016lx\n", + READ_SPECIAL_REG(sp_el1)); + + output->printf(output, " elr_el1 %016lx\n", + READ_SPECIAL_REG(elr_el1)); + + output->printf(output, " spsr_el1 %08lx\n", + READ_SPECIAL_REG(spsr_el1)); + + if (in_el2) { + output->printf(output, " spsr_irq %08lx\n", + READ_SPECIAL_REG(spsr_irq)); + output->printf(output, " spsr_abt %08lx\n", + READ_SPECIAL_REG(spsr_abt)); + output->printf(output, " spsr_und %08lx\n", + READ_SPECIAL_REG(spsr_und)); + output->printf(output, " spsr_fiq %08lx\n", + READ_SPECIAL_REG(spsr_fiq)); + output->printf(output, " spsr_el2 %08lx\n", + READ_SPECIAL_REG(elr_el2)); + output->printf(output, " spsr_el2 %08lx\n", + READ_SPECIAL_REG(spsr_el2)); + } +} + +struct stacktrace_state { + struct fiq_debugger_output *output; + unsigned int depth; +}; + +static int report_trace(struct stackframe *frame, void *d) +{ + struct stacktrace_state *sts = d; + + if (sts->depth) { + sts->output->printf(sts->output, "%pF:\n", frame->pc); + sts->output->printf(sts->output, + " pc %016lx sp %016lx fp %016lx\n", + frame->pc, frame->sp, frame->fp); + sts->depth--; + return 0; + } + sts->output->printf(sts->output, " ...\n"); + + return sts->depth == 0; +} + +void fiq_debugger_dump_stacktrace(struct fiq_debugger_output *output, + const struct pt_regs *regs, unsigned int depth, void *ssp) +{ + struct thread_info *real_thread_info = THREAD_INFO(ssp); + struct stacktrace_state sts; + + sts.depth = depth; + sts.output = output; + *current_thread_info() = *real_thread_info; + + if (!current) + output->printf(output, "current NULL\n"); + else + output->printf(output, "pid: %d comm: %s\n", + current->pid, current->comm); + fiq_debugger_dump_regs(output, regs); + + if (!user_mode(regs)) { + struct stackframe frame; + frame.fp = regs->regs[29]; + frame.sp = regs->sp; + frame.pc = regs->pc; + output->printf(output, "\n"); + walk_stackframe(current, &frame, report_trace, &sts); + } +} diff --git a/drivers/staging/android/fiq_debugger/fiq_debugger_priv.h b/drivers/staging/android/fiq_debugger/fiq_debugger_priv.h new file mode 100644 index 00000000000000..d5d051f727a8be --- /dev/null +++ b/drivers/staging/android/fiq_debugger/fiq_debugger_priv.h @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2014 Google, Inc. + * Author: Colin Cross + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#ifndef _FIQ_DEBUGGER_PRIV_H_ +#define _FIQ_DEBUGGER_PRIV_H_ + +#define THREAD_INFO(sp) ((struct thread_info *) \ + ((unsigned long)(sp) & ~(THREAD_SIZE - 1))) + +struct fiq_debugger_output { + void (*printf)(struct fiq_debugger_output *output, const char *fmt, ...); +}; + +struct pt_regs; + +void fiq_debugger_dump_pc(struct fiq_debugger_output *output, + const struct pt_regs *regs); +void fiq_debugger_dump_regs(struct fiq_debugger_output *output, + const struct pt_regs *regs); +void fiq_debugger_dump_allregs(struct fiq_debugger_output *output, + const struct pt_regs *regs); +void fiq_debugger_dump_stacktrace(struct fiq_debugger_output *output, + const struct pt_regs *regs, unsigned int depth, void *ssp); + +#endif diff --git a/drivers/staging/android/fiq_debugger/fiq_debugger_ringbuf.h b/drivers/staging/android/fiq_debugger/fiq_debugger_ringbuf.h new file mode 100644 index 00000000000000..10c3c5d09098e1 --- /dev/null +++ b/drivers/staging/android/fiq_debugger/fiq_debugger_ringbuf.h @@ -0,0 +1,94 @@ +/* + * drivers/staging/android/fiq_debugger/fiq_debugger_ringbuf.h + * + * simple lockless ringbuffer + * + * Copyright (C) 2010 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include + +struct fiq_debugger_ringbuf { + int len; + int head; + int tail; + u8 buf[]; +}; + + +static inline struct fiq_debugger_ringbuf *fiq_debugger_ringbuf_alloc(int len) +{ + struct fiq_debugger_ringbuf *rbuf; + + rbuf = kzalloc(sizeof(*rbuf) + len, GFP_KERNEL); + if (rbuf == NULL) + return NULL; + + rbuf->len = len; + rbuf->head = 0; + rbuf->tail = 0; + smp_mb(); + + return rbuf; +} + +static inline void fiq_debugger_ringbuf_free(struct fiq_debugger_ringbuf *rbuf) +{ + kfree(rbuf); +} + +static inline int fiq_debugger_ringbuf_level(struct fiq_debugger_ringbuf *rbuf) +{ + int level = rbuf->head - rbuf->tail; + + if (level < 0) + level = rbuf->len + level; + + return level; +} + +static inline int fiq_debugger_ringbuf_room(struct fiq_debugger_ringbuf *rbuf) +{ + return rbuf->len - fiq_debugger_ringbuf_level(rbuf) - 1; +} + +static inline u8 +fiq_debugger_ringbuf_peek(struct fiq_debugger_ringbuf *rbuf, int i) +{ + return rbuf->buf[(rbuf->tail + i) % rbuf->len]; +} + +static inline int +fiq_debugger_ringbuf_consume(struct fiq_debugger_ringbuf *rbuf, int count) +{ + count = min(count, fiq_debugger_ringbuf_level(rbuf)); + + rbuf->tail = (rbuf->tail + count) % rbuf->len; + smp_mb(); + + return count; +} + +static inline int +fiq_debugger_ringbuf_push(struct fiq_debugger_ringbuf *rbuf, u8 datum) +{ + if (fiq_debugger_ringbuf_room(rbuf) == 0) + return 0; + + rbuf->buf[rbuf->head] = datum; + smp_mb(); + rbuf->head = (rbuf->head + 1) % rbuf->len; + smp_mb(); + + return 1; +} diff --git a/drivers/staging/android/fiq_debugger/fiq_watchdog.c b/drivers/staging/android/fiq_debugger/fiq_watchdog.c new file mode 100644 index 00000000000000..194b5413841772 --- /dev/null +++ b/drivers/staging/android/fiq_debugger/fiq_watchdog.c @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2014 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include +#include + +#include "fiq_watchdog.h" +#include "fiq_debugger_priv.h" + +static DEFINE_RAW_SPINLOCK(fiq_watchdog_lock); + +static void fiq_watchdog_printf(struct fiq_debugger_output *output, + const char *fmt, ...) +{ + char buf[256]; + va_list ap; + int len; + + va_start(ap, fmt); + len = vscnprintf(buf, sizeof(buf), fmt, ap); + va_end(ap); + + ramoops_console_write_buf(buf, len); +} + +struct fiq_debugger_output fiq_watchdog_output = { + .printf = fiq_watchdog_printf, +}; + +void fiq_watchdog_triggered(const struct pt_regs *regs, void *svc_sp) +{ + char msg[24]; + int len; + + raw_spin_lock(&fiq_watchdog_lock); + + len = scnprintf(msg, sizeof(msg), "watchdog fiq cpu %d\n", + THREAD_INFO(svc_sp)->cpu); + ramoops_console_write_buf(msg, len); + + fiq_debugger_dump_stacktrace(&fiq_watchdog_output, regs, 100, svc_sp); + + raw_spin_unlock(&fiq_watchdog_lock); +} diff --git a/drivers/staging/android/fiq_debugger/fiq_watchdog.h b/drivers/staging/android/fiq_debugger/fiq_watchdog.h new file mode 100644 index 00000000000000..c6b507f8d97680 --- /dev/null +++ b/drivers/staging/android/fiq_debugger/fiq_watchdog.h @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2014 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#ifndef _FIQ_WATCHDOG_H_ +#define _FIQ_WATCHDOG_H_ + +void fiq_watchdog_triggered(const struct pt_regs *regs, void *svc_sp); + +#endif diff --git a/drivers/staging/android/lowmemorykiller.c b/drivers/staging/android/lowmemorykiller.c index ec3b6656141242..6dcde85f05dc64 100644 --- a/drivers/staging/android/lowmemorykiller.c +++ b/drivers/staging/android/lowmemorykiller.c @@ -43,6 +43,9 @@ #include #include +#define CREATE_TRACE_POINTS +#include "trace/lowmemorykiller.h" + static u32 lowmem_debug_level = 1; static short lowmem_adj[6] = { 0, @@ -160,11 +163,16 @@ static unsigned long lowmem_scan(struct shrinker *s, struct shrink_control *sc) p->comm, p->pid, oom_score_adj, tasksize); } if (selected) { + long cache_size = other_file * (long)(PAGE_SIZE / 1024); + long cache_limit = minfree * (long)(PAGE_SIZE / 1024); + long free = other_free * (long)(PAGE_SIZE / 1024); + task_lock(selected); send_sig(SIGKILL, selected, 0); if (selected->mm) task_set_lmk_waiting(selected); task_unlock(selected); + trace_lowmemory_kill(selected, cache_size, cache_limit, free); lowmem_print(1, "Killing '%s' (%d), adj %hd,\n" " to free %ldkB on behalf of '%s' (%d) because\n" " cache %ldkB is below limit %ldkB for oom_score_adj %hd\n" @@ -173,10 +181,9 @@ static unsigned long lowmem_scan(struct shrinker *s, struct shrink_control *sc) selected_oom_score_adj, selected_tasksize * (long)(PAGE_SIZE / 1024), current->comm, current->pid, - other_file * (long)(PAGE_SIZE / 1024), - minfree * (long)(PAGE_SIZE / 1024), + cache_size, cache_limit, min_score_adj, - other_free * (long)(PAGE_SIZE / 1024)); + free); lowmem_deathpending_timeout = jiffies + HZ; rem += selected_tasksize; } @@ -200,12 +207,96 @@ static int __init lowmem_init(void) } device_initcall(lowmem_init); +#ifdef CONFIG_ANDROID_LOW_MEMORY_KILLER_AUTODETECT_OOM_ADJ_VALUES +static short lowmem_oom_adj_to_oom_score_adj(short oom_adj) +{ + if (oom_adj == OOM_ADJUST_MAX) + return OOM_SCORE_ADJ_MAX; + else + return (oom_adj * OOM_SCORE_ADJ_MAX) / -OOM_DISABLE; +} + +static void lowmem_autodetect_oom_adj_values(void) +{ + int i; + short oom_adj; + short oom_score_adj; + int array_size = ARRAY_SIZE(lowmem_adj); + + if (lowmem_adj_size < array_size) + array_size = lowmem_adj_size; + + if (array_size <= 0) + return; + + oom_adj = lowmem_adj[array_size - 1]; + if (oom_adj > OOM_ADJUST_MAX) + return; + + oom_score_adj = lowmem_oom_adj_to_oom_score_adj(oom_adj); + if (oom_score_adj <= OOM_ADJUST_MAX) + return; + + lowmem_print(1, "lowmem_shrink: convert oom_adj to oom_score_adj:\n"); + for (i = 0; i < array_size; i++) { + oom_adj = lowmem_adj[i]; + oom_score_adj = lowmem_oom_adj_to_oom_score_adj(oom_adj); + lowmem_adj[i] = oom_score_adj; + lowmem_print(1, "oom_adj %d => oom_score_adj %d\n", + oom_adj, oom_score_adj); + } +} + +static int lowmem_adj_array_set(const char *val, const struct kernel_param *kp) +{ + int ret; + + ret = param_array_ops.set(val, kp); + + /* HACK: Autodetect oom_adj values in lowmem_adj array */ + lowmem_autodetect_oom_adj_values(); + + return ret; +} + +static int lowmem_adj_array_get(char *buffer, const struct kernel_param *kp) +{ + return param_array_ops.get(buffer, kp); +} + +static void lowmem_adj_array_free(void *arg) +{ + param_array_ops.free(arg); +} + +static struct kernel_param_ops lowmem_adj_array_ops = { + .set = lowmem_adj_array_set, + .get = lowmem_adj_array_get, + .free = lowmem_adj_array_free, +}; + +static const struct kparam_array __param_arr_adj = { + .max = ARRAY_SIZE(lowmem_adj), + .num = &lowmem_adj_size, + .ops = ¶m_ops_short, + .elemsize = sizeof(lowmem_adj[0]), + .elem = lowmem_adj, +}; +#endif + /* * not really modular, but the easiest way to keep compat with existing * bootargs behaviour is to continue using module_param here. */ module_param_named(cost, lowmem_shrinker.seeks, int, 0644); +#ifdef CONFIG_ANDROID_LOW_MEMORY_KILLER_AUTODETECT_OOM_ADJ_VALUES +module_param_cb(adj, &lowmem_adj_array_ops, + .arr = &__param_arr_adj, + 0644); +__MODULE_PARM_TYPE(adj, "array of short"); +#else module_param_array_named(adj, lowmem_adj, short, &lowmem_adj_size, 0644); +#endif module_param_array_named(minfree, lowmem_minfree, uint, &lowmem_minfree_size, 0644); module_param_named(debug_level, lowmem_debug_level, uint, 0644); diff --git a/drivers/staging/android/sw_sync.c b/drivers/staging/android/sw_sync.c new file mode 100644 index 00000000000000..c4ff1679ebbce2 --- /dev/null +++ b/drivers/staging/android/sw_sync.c @@ -0,0 +1,260 @@ +/* + * drivers/base/sw_sync.c + * + * Copyright (C) 2012 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sw_sync.h" + +static int sw_sync_cmp(u32 a, u32 b) +{ + if (a == b) + return 0; + + return ((s32)a - (s32)b) < 0 ? -1 : 1; +} + +struct sync_pt *sw_sync_pt_create(struct sw_sync_timeline *obj, u32 value) +{ + struct sw_sync_pt *pt; + + pt = (struct sw_sync_pt *) + sync_pt_create(&obj->obj, sizeof(struct sw_sync_pt)); + + pt->value = value; + + return (struct sync_pt *)pt; +} +EXPORT_SYMBOL(sw_sync_pt_create); + +static struct sync_pt *sw_sync_pt_dup(struct sync_pt *sync_pt) +{ + struct sw_sync_pt *pt = (struct sw_sync_pt *)sync_pt; + struct sw_sync_timeline *obj = + (struct sw_sync_timeline *)sync_pt_parent(sync_pt); + + return (struct sync_pt *)sw_sync_pt_create(obj, pt->value); +} + +static int sw_sync_pt_has_signaled(struct sync_pt *sync_pt) +{ + struct sw_sync_pt *pt = (struct sw_sync_pt *)sync_pt; + struct sw_sync_timeline *obj = + (struct sw_sync_timeline *)sync_pt_parent(sync_pt); + + return sw_sync_cmp(obj->value, pt->value) >= 0; +} + +static int sw_sync_pt_compare(struct sync_pt *a, struct sync_pt *b) +{ + struct sw_sync_pt *pt_a = (struct sw_sync_pt *)a; + struct sw_sync_pt *pt_b = (struct sw_sync_pt *)b; + + return sw_sync_cmp(pt_a->value, pt_b->value); +} + +static int sw_sync_fill_driver_data(struct sync_pt *sync_pt, + void *data, int size) +{ + struct sw_sync_pt *pt = (struct sw_sync_pt *)sync_pt; + + if (size < sizeof(pt->value)) + return -ENOMEM; + + memcpy(data, &pt->value, sizeof(pt->value)); + + return sizeof(pt->value); +} + +static void sw_sync_timeline_value_str(struct sync_timeline *sync_timeline, + char *str, int size) +{ + struct sw_sync_timeline *timeline = + (struct sw_sync_timeline *)sync_timeline; + snprintf(str, size, "%d", timeline->value); +} + +static void sw_sync_pt_value_str(struct sync_pt *sync_pt, + char *str, int size) +{ + struct sw_sync_pt *pt = (struct sw_sync_pt *)sync_pt; + + snprintf(str, size, "%d", pt->value); +} + +static struct sync_timeline_ops sw_sync_timeline_ops = { + .driver_name = "sw_sync", + .dup = sw_sync_pt_dup, + .has_signaled = sw_sync_pt_has_signaled, + .compare = sw_sync_pt_compare, + .fill_driver_data = sw_sync_fill_driver_data, + .timeline_value_str = sw_sync_timeline_value_str, + .pt_value_str = sw_sync_pt_value_str, +}; + +struct sw_sync_timeline *sw_sync_timeline_create(const char *name) +{ + struct sw_sync_timeline *obj = (struct sw_sync_timeline *) + sync_timeline_create(&sw_sync_timeline_ops, + sizeof(struct sw_sync_timeline), + name); + + return obj; +} +EXPORT_SYMBOL(sw_sync_timeline_create); + +void sw_sync_timeline_inc(struct sw_sync_timeline *obj, u32 inc) +{ + obj->value += inc; + + sync_timeline_signal(&obj->obj); +} +EXPORT_SYMBOL(sw_sync_timeline_inc); + +#ifdef CONFIG_SW_SYNC_USER +/* *WARNING* + * + * improper use of this can result in deadlocking kernel drivers from userspace. + */ + +/* opening sw_sync create a new sync obj */ +static int sw_sync_open(struct inode *inode, struct file *file) +{ + struct sw_sync_timeline *obj; + char task_comm[TASK_COMM_LEN]; + + get_task_comm(task_comm, current); + + obj = sw_sync_timeline_create(task_comm); + if (!obj) + return -ENOMEM; + + file->private_data = obj; + + return 0; +} + +static int sw_sync_release(struct inode *inode, struct file *file) +{ + struct sw_sync_timeline *obj = file->private_data; + + sync_timeline_destroy(&obj->obj); + return 0; +} + +static long sw_sync_ioctl_create_fence(struct sw_sync_timeline *obj, + unsigned long arg) +{ + int fd = get_unused_fd_flags(O_CLOEXEC); + int err; + struct sync_pt *pt; + struct sync_fence *fence; + struct sw_sync_create_fence_data data; + + if (fd < 0) + return fd; + + if (copy_from_user(&data, (void __user *)arg, sizeof(data))) { + err = -EFAULT; + goto err; + } + + pt = sw_sync_pt_create(obj, data.value); + if (!pt) { + err = -ENOMEM; + goto err; + } + + data.name[sizeof(data.name) - 1] = '\0'; + fence = sync_fence_create(data.name, pt); + if (!fence) { + sync_pt_free(pt); + err = -ENOMEM; + goto err; + } + + data.fence = fd; + if (copy_to_user((void __user *)arg, &data, sizeof(data))) { + sync_fence_put(fence); + err = -EFAULT; + goto err; + } + + sync_fence_install(fence, fd); + + return 0; + +err: + put_unused_fd(fd); + return err; +} + +static long sw_sync_ioctl_inc(struct sw_sync_timeline *obj, unsigned long arg) +{ + u32 value; + + if (copy_from_user(&value, (void __user *)arg, sizeof(value))) + return -EFAULT; + + sw_sync_timeline_inc(obj, value); + + return 0; +} + +static long sw_sync_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + struct sw_sync_timeline *obj = file->private_data; + + switch (cmd) { + case SW_SYNC_IOC_CREATE_FENCE: + return sw_sync_ioctl_create_fence(obj, arg); + + case SW_SYNC_IOC_INC: + return sw_sync_ioctl_inc(obj, arg); + + default: + return -ENOTTY; + } +} + +static const struct file_operations sw_sync_fops = { + .owner = THIS_MODULE, + .open = sw_sync_open, + .release = sw_sync_release, + .unlocked_ioctl = sw_sync_ioctl, + .compat_ioctl = sw_sync_ioctl, +}; + +static struct miscdevice sw_sync_dev = { + .minor = MISC_DYNAMIC_MINOR, + .name = "sw_sync", + .fops = &sw_sync_fops, +}; + +static int __init sw_sync_device_init(void) +{ + return misc_register(&sw_sync_dev); +} +device_initcall(sw_sync_device_init); + +#endif /* CONFIG_SW_SYNC_USER */ diff --git a/drivers/staging/android/sw_sync.h b/drivers/staging/android/sw_sync.h new file mode 100644 index 00000000000000..c87ae9ebf267f6 --- /dev/null +++ b/drivers/staging/android/sw_sync.h @@ -0,0 +1,59 @@ +/* + * include/linux/sw_sync.h + * + * Copyright (C) 2012 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#ifndef _LINUX_SW_SYNC_H +#define _LINUX_SW_SYNC_H + +#include +#include +#include "sync.h" +#include "uapi/sw_sync.h" + +struct sw_sync_timeline { + struct sync_timeline obj; + + u32 value; +}; + +struct sw_sync_pt { + struct sync_pt pt; + + u32 value; +}; + +#if IS_ENABLED(CONFIG_SW_SYNC) +struct sw_sync_timeline *sw_sync_timeline_create(const char *name); +void sw_sync_timeline_inc(struct sw_sync_timeline *obj, u32 inc); + +struct sync_pt *sw_sync_pt_create(struct sw_sync_timeline *obj, u32 value); +#else +static inline struct sw_sync_timeline *sw_sync_timeline_create(const char *name) +{ + return NULL; +} + +static inline void sw_sync_timeline_inc(struct sw_sync_timeline *obj, u32 inc) +{ +} + +static inline struct sync_pt *sw_sync_pt_create(struct sw_sync_timeline *obj, + u32 value) +{ + return NULL; +} +#endif /* IS_ENABLED(CONFIG_SW_SYNC) */ + +#endif /* _LINUX_SW_SYNC_H */ diff --git a/drivers/staging/android/sync.c b/drivers/staging/android/sync.c new file mode 100644 index 00000000000000..ed43796b5b582d --- /dev/null +++ b/drivers/staging/android/sync.c @@ -0,0 +1,732 @@ +/* + * drivers/base/sync.c + * + * Copyright (C) 2012 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sync.h" + +#define CREATE_TRACE_POINTS +#include "trace/sync.h" + +static const struct fence_ops android_fence_ops; +static const struct file_operations sync_fence_fops; + +struct sync_timeline *sync_timeline_create(const struct sync_timeline_ops *ops, + int size, const char *name) +{ + struct sync_timeline *obj; + + if (size < sizeof(struct sync_timeline)) + return NULL; + + obj = kzalloc(size, GFP_KERNEL); + if (!obj) + return NULL; + + kref_init(&obj->kref); + obj->ops = ops; + obj->context = fence_context_alloc(1); + strlcpy(obj->name, name, sizeof(obj->name)); + + INIT_LIST_HEAD(&obj->child_list_head); + INIT_LIST_HEAD(&obj->active_list_head); + spin_lock_init(&obj->child_list_lock); + + sync_timeline_debug_add(obj); + + return obj; +} +EXPORT_SYMBOL(sync_timeline_create); + +static void sync_timeline_free(struct kref *kref) +{ + struct sync_timeline *obj = + container_of(kref, struct sync_timeline, kref); + + sync_timeline_debug_remove(obj); + + if (obj->ops->release_obj) + obj->ops->release_obj(obj); + + kfree(obj); +} + +static void sync_timeline_get(struct sync_timeline *obj) +{ + kref_get(&obj->kref); +} + +static void sync_timeline_put(struct sync_timeline *obj) +{ + kref_put(&obj->kref, sync_timeline_free); +} + +void sync_timeline_destroy(struct sync_timeline *obj) +{ + obj->destroyed = true; + /* + * Ensure timeline is marked as destroyed before + * changing timeline's fences status. + */ + smp_wmb(); + + /* + * signal any children that their parent is going away. + */ + sync_timeline_signal(obj); + sync_timeline_put(obj); +} +EXPORT_SYMBOL(sync_timeline_destroy); + +void sync_timeline_signal(struct sync_timeline *obj) +{ + unsigned long flags; + LIST_HEAD(signaled_pts); + struct sync_pt *pt, *next; + + trace_sync_timeline(obj); + + spin_lock_irqsave(&obj->child_list_lock, flags); + + list_for_each_entry_safe(pt, next, &obj->active_list_head, + active_list) { + if (fence_is_signaled_locked(&pt->base)) + list_del_init(&pt->active_list); + } + + spin_unlock_irqrestore(&obj->child_list_lock, flags); +} +EXPORT_SYMBOL(sync_timeline_signal); + +struct sync_pt *sync_pt_create(struct sync_timeline *obj, int size) +{ + unsigned long flags; + struct sync_pt *pt; + + if (size < sizeof(struct sync_pt)) + return NULL; + + pt = kzalloc(size, GFP_KERNEL); + if (!pt) + return NULL; + + spin_lock_irqsave(&obj->child_list_lock, flags); + sync_timeline_get(obj); + fence_init(&pt->base, &android_fence_ops, &obj->child_list_lock, + obj->context, ++obj->value); + list_add_tail(&pt->child_list, &obj->child_list_head); + INIT_LIST_HEAD(&pt->active_list); + spin_unlock_irqrestore(&obj->child_list_lock, flags); + return pt; +} +EXPORT_SYMBOL(sync_pt_create); + +void sync_pt_free(struct sync_pt *pt) +{ + fence_put(&pt->base); +} +EXPORT_SYMBOL(sync_pt_free); + +static struct sync_fence *sync_fence_alloc(int size, const char *name) +{ + struct sync_fence *fence; + + fence = kzalloc(size, GFP_KERNEL); + if (!fence) + return NULL; + + fence->file = anon_inode_getfile("sync_fence", &sync_fence_fops, + fence, 0); + if (IS_ERR(fence->file)) + goto err; + + kref_init(&fence->kref); + strlcpy(fence->name, name, sizeof(fence->name)); + + init_waitqueue_head(&fence->wq); + + return fence; + +err: + kfree(fence); + return NULL; +} + +static void fence_check_cb_func(struct fence *f, struct fence_cb *cb) +{ + struct sync_fence_cb *check; + struct sync_fence *fence; + + check = container_of(cb, struct sync_fence_cb, cb); + fence = check->fence; + + if (atomic_dec_and_test(&fence->status)) + wake_up_all(&fence->wq); +} + +/* TODO: implement a create which takes more that one sync_pt */ +struct sync_fence *sync_fence_create_dma(const char *name, struct fence *pt) +{ + struct sync_fence *fence; + + fence = sync_fence_alloc(offsetof(struct sync_fence, cbs[1]), name); + if (!fence) + return NULL; + + fence->num_fences = 1; + atomic_set(&fence->status, 1); + + fence->cbs[0].sync_pt = pt; + fence->cbs[0].fence = fence; + if (fence_add_callback(pt, &fence->cbs[0].cb, fence_check_cb_func)) + atomic_dec(&fence->status); + + sync_fence_debug_add(fence); + + return fence; +} +EXPORT_SYMBOL(sync_fence_create_dma); + +struct sync_fence *sync_fence_create(const char *name, struct sync_pt *pt) +{ + return sync_fence_create_dma(name, &pt->base); +} +EXPORT_SYMBOL(sync_fence_create); + +struct sync_fence *sync_fence_fdget(int fd) +{ + struct file *file = fget(fd); + + if (!file) + return NULL; + + if (file->f_op != &sync_fence_fops) + goto err; + + return file->private_data; + +err: + fput(file); + return NULL; +} +EXPORT_SYMBOL(sync_fence_fdget); + +void sync_fence_put(struct sync_fence *fence) +{ + fput(fence->file); +} +EXPORT_SYMBOL(sync_fence_put); + +void sync_fence_install(struct sync_fence *fence, int fd) +{ + fd_install(fd, fence->file); +} +EXPORT_SYMBOL(sync_fence_install); + +static void sync_fence_add_pt(struct sync_fence *fence, + int *i, struct fence *pt) +{ + fence->cbs[*i].sync_pt = pt; + fence->cbs[*i].fence = fence; + + if (!fence_add_callback(pt, &fence->cbs[*i].cb, fence_check_cb_func)) { + fence_get(pt); + (*i)++; + } +} + +struct sync_fence *sync_fence_merge(const char *name, + struct sync_fence *a, struct sync_fence *b) +{ + int num_fences = a->num_fences + b->num_fences; + struct sync_fence *fence; + int i, i_a, i_b; + unsigned long size = offsetof(struct sync_fence, cbs[num_fences]); + + fence = sync_fence_alloc(size, name); + if (!fence) + return NULL; + + atomic_set(&fence->status, num_fences); + + /* + * Assume sync_fence a and b are both ordered and have no + * duplicates with the same context. + * + * If a sync_fence can only be created with sync_fence_merge + * and sync_fence_create, this is a reasonable assumption. + */ + for (i = i_a = i_b = 0; i_a < a->num_fences && i_b < b->num_fences; ) { + struct fence *pt_a = a->cbs[i_a].sync_pt; + struct fence *pt_b = b->cbs[i_b].sync_pt; + + if (pt_a->context < pt_b->context) { + sync_fence_add_pt(fence, &i, pt_a); + + i_a++; + } else if (pt_a->context > pt_b->context) { + sync_fence_add_pt(fence, &i, pt_b); + + i_b++; + } else { + if (pt_a->seqno - pt_b->seqno <= INT_MAX) + sync_fence_add_pt(fence, &i, pt_a); + else + sync_fence_add_pt(fence, &i, pt_b); + + i_a++; + i_b++; + } + } + + for (; i_a < a->num_fences; i_a++) + sync_fence_add_pt(fence, &i, a->cbs[i_a].sync_pt); + + for (; i_b < b->num_fences; i_b++) + sync_fence_add_pt(fence, &i, b->cbs[i_b].sync_pt); + + if (num_fences > i) + atomic_sub(num_fences - i, &fence->status); + fence->num_fences = i; + + sync_fence_debug_add(fence); + return fence; +} +EXPORT_SYMBOL(sync_fence_merge); + +int sync_fence_wake_up_wq(wait_queue_t *curr, unsigned mode, + int wake_flags, void *key) +{ + struct sync_fence_waiter *wait; + + wait = container_of(curr, struct sync_fence_waiter, work); + list_del_init(&wait->work.task_list); + + wait->callback(wait->work.private, wait); + return 1; +} + +int sync_fence_wait_async(struct sync_fence *fence, + struct sync_fence_waiter *waiter) +{ + int err = atomic_read(&fence->status); + unsigned long flags; + + if (err < 0) + return err; + + if (!err) + return 1; + + init_waitqueue_func_entry(&waiter->work, sync_fence_wake_up_wq); + waiter->work.private = fence; + + spin_lock_irqsave(&fence->wq.lock, flags); + err = atomic_read(&fence->status); + if (err > 0) + __add_wait_queue_tail(&fence->wq, &waiter->work); + spin_unlock_irqrestore(&fence->wq.lock, flags); + + if (err < 0) + return err; + + return !err; +} +EXPORT_SYMBOL(sync_fence_wait_async); + +int sync_fence_cancel_async(struct sync_fence *fence, + struct sync_fence_waiter *waiter) +{ + unsigned long flags; + int ret = 0; + + spin_lock_irqsave(&fence->wq.lock, flags); + if (!list_empty(&waiter->work.task_list)) + list_del_init(&waiter->work.task_list); + else + ret = -ENOENT; + spin_unlock_irqrestore(&fence->wq.lock, flags); + return ret; +} +EXPORT_SYMBOL(sync_fence_cancel_async); + +int sync_fence_wait(struct sync_fence *fence, long timeout) +{ + long ret; + int i; + + if (timeout < 0) + timeout = MAX_SCHEDULE_TIMEOUT; + else + timeout = msecs_to_jiffies(timeout); + + trace_sync_wait(fence, 1); + for (i = 0; i < fence->num_fences; ++i) + trace_sync_pt(fence->cbs[i].sync_pt); + ret = wait_event_interruptible_timeout(fence->wq, + atomic_read(&fence->status) <= 0, + timeout); + trace_sync_wait(fence, 0); + + if (ret < 0) { + return ret; + } else if (ret == 0) { + if (timeout) { + pr_info("fence timeout on [%p] after %dms\n", fence, + jiffies_to_msecs(timeout)); + sync_dump(); + } + return -ETIME; + } + + ret = atomic_read(&fence->status); + if (ret) { + pr_info("fence error %ld on [%p]\n", ret, fence); + sync_dump(); + } + return ret; +} +EXPORT_SYMBOL(sync_fence_wait); + +static const char *android_fence_get_driver_name(struct fence *fence) +{ + struct sync_pt *pt = container_of(fence, struct sync_pt, base); + struct sync_timeline *parent = sync_pt_parent(pt); + + return parent->ops->driver_name; +} + +static const char *android_fence_get_timeline_name(struct fence *fence) +{ + struct sync_pt *pt = container_of(fence, struct sync_pt, base); + struct sync_timeline *parent = sync_pt_parent(pt); + + return parent->name; +} + +static void android_fence_release(struct fence *fence) +{ + struct sync_pt *pt = container_of(fence, struct sync_pt, base); + struct sync_timeline *parent = sync_pt_parent(pt); + unsigned long flags; + + spin_lock_irqsave(fence->lock, flags); + list_del(&pt->child_list); + if (WARN_ON_ONCE(!list_empty(&pt->active_list))) + list_del(&pt->active_list); + spin_unlock_irqrestore(fence->lock, flags); + + if (parent->ops->free_pt) + parent->ops->free_pt(pt); + + sync_timeline_put(parent); + fence_free(&pt->base); +} + +static bool android_fence_signaled(struct fence *fence) +{ + struct sync_pt *pt = container_of(fence, struct sync_pt, base); + struct sync_timeline *parent = sync_pt_parent(pt); + int ret; + + ret = parent->ops->has_signaled(pt); + if (ret < 0) + fence->status = ret; + return ret; +} + +static bool android_fence_enable_signaling(struct fence *fence) +{ + struct sync_pt *pt = container_of(fence, struct sync_pt, base); + struct sync_timeline *parent = sync_pt_parent(pt); + + if (android_fence_signaled(fence)) + return false; + + list_add_tail(&pt->active_list, &parent->active_list_head); + return true; +} + +static int android_fence_fill_driver_data(struct fence *fence, + void *data, int size) +{ + struct sync_pt *pt = container_of(fence, struct sync_pt, base); + struct sync_timeline *parent = sync_pt_parent(pt); + + if (!parent->ops->fill_driver_data) + return 0; + return parent->ops->fill_driver_data(pt, data, size); +} + +static void android_fence_value_str(struct fence *fence, + char *str, int size) +{ + struct sync_pt *pt = container_of(fence, struct sync_pt, base); + struct sync_timeline *parent = sync_pt_parent(pt); + + if (!parent->ops->pt_value_str) { + if (size) + *str = 0; + return; + } + parent->ops->pt_value_str(pt, str, size); +} + +static void android_fence_timeline_value_str(struct fence *fence, + char *str, int size) +{ + struct sync_pt *pt = container_of(fence, struct sync_pt, base); + struct sync_timeline *parent = sync_pt_parent(pt); + + if (!parent->ops->timeline_value_str) { + if (size) + *str = 0; + return; + } + parent->ops->timeline_value_str(parent, str, size); +} + +static const struct fence_ops android_fence_ops = { + .get_driver_name = android_fence_get_driver_name, + .get_timeline_name = android_fence_get_timeline_name, + .enable_signaling = android_fence_enable_signaling, + .signaled = android_fence_signaled, + .wait = fence_default_wait, + .release = android_fence_release, + .fill_driver_data = android_fence_fill_driver_data, + .fence_value_str = android_fence_value_str, + .timeline_value_str = android_fence_timeline_value_str, +}; + +static void sync_fence_free(struct kref *kref) +{ + struct sync_fence *fence = container_of(kref, struct sync_fence, kref); + int i; + + for (i = 0; i < fence->num_fences; ++i) { + fence_remove_callback(fence->cbs[i].sync_pt, &fence->cbs[i].cb); + fence_put(fence->cbs[i].sync_pt); + } + + kfree(fence); +} + +static int sync_fence_release(struct inode *inode, struct file *file) +{ + struct sync_fence *fence = file->private_data; + + sync_fence_debug_remove(fence); + + kref_put(&fence->kref, sync_fence_free); + return 0; +} + +static unsigned int sync_fence_poll(struct file *file, poll_table *wait) +{ + struct sync_fence *fence = file->private_data; + int status; + + poll_wait(file, &fence->wq, wait); + + status = atomic_read(&fence->status); + + if (!status) + return POLLIN; + else if (status < 0) + return POLLERR; + return 0; +} + +static long sync_fence_ioctl_wait(struct sync_fence *fence, unsigned long arg) +{ + __s32 value; + + if (copy_from_user(&value, (void __user *)arg, sizeof(value))) + return -EFAULT; + + return sync_fence_wait(fence, value); +} + +static long sync_fence_ioctl_merge(struct sync_fence *fence, unsigned long arg) +{ + int fd = get_unused_fd_flags(O_CLOEXEC); + int err; + struct sync_fence *fence2, *fence3; + struct sync_merge_data data; + + if (fd < 0) + return fd; + + if (copy_from_user(&data, (void __user *)arg, sizeof(data))) { + err = -EFAULT; + goto err_put_fd; + } + + fence2 = sync_fence_fdget(data.fd2); + if (!fence2) { + err = -ENOENT; + goto err_put_fd; + } + + data.name[sizeof(data.name) - 1] = '\0'; + fence3 = sync_fence_merge(data.name, fence, fence2); + if (!fence3) { + err = -ENOMEM; + goto err_put_fence2; + } + + data.fence = fd; + if (copy_to_user((void __user *)arg, &data, sizeof(data))) { + err = -EFAULT; + goto err_put_fence3; + } + + sync_fence_install(fence3, fd); + sync_fence_put(fence2); + return 0; + +err_put_fence3: + sync_fence_put(fence3); + +err_put_fence2: + sync_fence_put(fence2); + +err_put_fd: + put_unused_fd(fd); + return err; +} + +static int sync_fill_pt_info(struct fence *fence, void *data, int size) +{ + struct sync_pt_info *info = data; + int ret; + + if (size < sizeof(struct sync_pt_info)) + return -ENOMEM; + + info->len = sizeof(struct sync_pt_info); + + if (fence->ops->fill_driver_data) { + ret = fence->ops->fill_driver_data(fence, info->driver_data, + size - sizeof(*info)); + if (ret < 0) + return ret; + + info->len += ret; + } + + strlcpy(info->obj_name, fence->ops->get_timeline_name(fence), + sizeof(info->obj_name)); + strlcpy(info->driver_name, fence->ops->get_driver_name(fence), + sizeof(info->driver_name)); + if (fence_is_signaled(fence)) + info->status = fence->status >= 0 ? 1 : fence->status; + else + info->status = 0; + info->timestamp_ns = ktime_to_ns(fence->timestamp); + + return info->len; +} + +static long sync_fence_ioctl_fence_info(struct sync_fence *fence, + unsigned long arg) +{ + struct sync_fence_info_data *data; + __u32 size; + __u32 len = 0; + int ret, i; + + if (copy_from_user(&size, (void __user *)arg, sizeof(size))) + return -EFAULT; + + if (size < sizeof(struct sync_fence_info_data)) + return -EINVAL; + + if (size > 4096) + size = 4096; + + data = kzalloc(size, GFP_KERNEL); + if (!data) + return -ENOMEM; + + strlcpy(data->name, fence->name, sizeof(data->name)); + data->status = atomic_read(&fence->status); + if (data->status >= 0) + data->status = !data->status; + + len = sizeof(struct sync_fence_info_data); + + for (i = 0; i < fence->num_fences; ++i) { + struct fence *pt = fence->cbs[i].sync_pt; + + ret = sync_fill_pt_info(pt, (u8 *)data + len, size - len); + + if (ret < 0) + goto out; + + len += ret; + } + + data->len = len; + + if (copy_to_user((void __user *)arg, data, len)) + ret = -EFAULT; + else + ret = 0; + +out: + kfree(data); + + return ret; +} + +static long sync_fence_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + struct sync_fence *fence = file->private_data; + + switch (cmd) { + case SYNC_IOC_WAIT: + return sync_fence_ioctl_wait(fence, arg); + + case SYNC_IOC_MERGE: + return sync_fence_ioctl_merge(fence, arg); + + case SYNC_IOC_FENCE_INFO: + return sync_fence_ioctl_fence_info(fence, arg); + + default: + return -ENOTTY; + } +} + +static const struct file_operations sync_fence_fops = { + .release = sync_fence_release, + .poll = sync_fence_poll, + .unlocked_ioctl = sync_fence_ioctl, + .compat_ioctl = sync_fence_ioctl, +}; + diff --git a/drivers/staging/android/sync.h b/drivers/staging/android/sync.h new file mode 100644 index 00000000000000..afa0752275a722 --- /dev/null +++ b/drivers/staging/android/sync.h @@ -0,0 +1,366 @@ +/* + * include/linux/sync.h + * + * Copyright (C) 2012 Google, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#ifndef _LINUX_SYNC_H +#define _LINUX_SYNC_H + +#include +#include +#include +#include +#include +#include +#include + +#include "uapi/sync.h" + +struct sync_timeline; +struct sync_pt; +struct sync_fence; + +/** + * struct sync_timeline_ops - sync object implementation ops + * @driver_name: name of the implementation + * @dup: duplicate a sync_pt + * @has_signaled: returns: + * 1 if pt has signaled + * 0 if pt has not signaled + * <0 on error + * @compare: returns: + * 1 if b will signal before a + * 0 if a and b will signal at the same time + * -1 if a will signal before b + * @free_pt: called before sync_pt is freed + * @release_obj: called before sync_timeline is freed + * @fill_driver_data: write implementation specific driver data to data. + * should return an error if there is not enough room + * as specified by size. This information is returned + * to userspace by SYNC_IOC_FENCE_INFO. + * @timeline_value_str: fill str with the value of the sync_timeline's counter + * @pt_value_str: fill str with the value of the sync_pt + */ +struct sync_timeline_ops { + const char *driver_name; + + /* required */ + struct sync_pt * (*dup)(struct sync_pt *pt); + + /* required */ + int (*has_signaled)(struct sync_pt *pt); + + /* required */ + int (*compare)(struct sync_pt *a, struct sync_pt *b); + + /* optional */ + void (*free_pt)(struct sync_pt *sync_pt); + + /* optional */ + void (*release_obj)(struct sync_timeline *sync_timeline); + + /* optional */ + int (*fill_driver_data)(struct sync_pt *syncpt, void *data, int size); + + /* optional */ + void (*timeline_value_str)(struct sync_timeline *timeline, char *str, + int size); + + /* optional */ + void (*pt_value_str)(struct sync_pt *pt, char *str, int size); +}; + +/** + * struct sync_timeline - sync object + * @kref: reference count on fence. + * @ops: ops that define the implementation of the sync_timeline + * @name: name of the sync_timeline. Useful for debugging + * @destroyed: set when sync_timeline is destroyed + * @child_list_head: list of children sync_pts for this sync_timeline + * @child_list_lock: lock protecting @child_list_head, destroyed, and + * sync_pt.status + * @active_list_head: list of active (unsignaled/errored) sync_pts + * @sync_timeline_list: membership in global sync_timeline_list + */ +struct sync_timeline { + struct kref kref; + const struct sync_timeline_ops *ops; + char name[32]; + + /* protected by child_list_lock */ + bool destroyed; + int context, value; + + struct list_head child_list_head; + spinlock_t child_list_lock; + + struct list_head active_list_head; + +#ifdef CONFIG_DEBUG_FS + struct list_head sync_timeline_list; +#endif +}; + +/** + * struct sync_pt - sync point + * @fence: base fence class + * @child_list: membership in sync_timeline.child_list_head + * @active_list: membership in sync_timeline.active_list_head + * @signaled_list: membership in temporary signaled_list on stack + * @fence: sync_fence to which the sync_pt belongs + * @pt_list: membership in sync_fence.pt_list_head + * @status: 1: signaled, 0:active, <0: error + * @timestamp: time which sync_pt status transitioned from active to + * signaled or error. + */ +struct sync_pt { + struct fence base; + + struct list_head child_list; + struct list_head active_list; +}; + +static inline struct sync_timeline *sync_pt_parent(struct sync_pt *pt) +{ + return container_of(pt->base.lock, struct sync_timeline, + child_list_lock); +} + +struct sync_fence_cb { + struct fence_cb cb; + struct fence *sync_pt; + struct sync_fence *fence; +}; + +/** + * struct sync_fence - sync fence + * @file: file representing this fence + * @kref: reference count on fence. + * @name: name of sync_fence. Useful for debugging + * @pt_list_head: list of sync_pts in the fence. immutable once fence + * is created + * @status: 0: signaled, >0:active, <0: error + * + * @wq: wait queue for fence signaling + * @sync_fence_list: membership in global fence list + */ +struct sync_fence { + struct file *file; + struct kref kref; + char name[32]; +#ifdef CONFIG_DEBUG_FS + struct list_head sync_fence_list; +#endif + int num_fences; + + wait_queue_head_t wq; + atomic_t status; + + struct sync_fence_cb cbs[]; +}; + +struct sync_fence_waiter; +typedef void (*sync_callback_t)(struct sync_fence *fence, + struct sync_fence_waiter *waiter); + +/** + * struct sync_fence_waiter - metadata for asynchronous waiter on a fence + * @waiter_list: membership in sync_fence.waiter_list_head + * @callback: function pointer to call when fence signals + * @callback_data: pointer to pass to @callback + */ +struct sync_fence_waiter { + wait_queue_t work; + sync_callback_t callback; +}; + +static inline void sync_fence_waiter_init(struct sync_fence_waiter *waiter, + sync_callback_t callback) +{ + INIT_LIST_HEAD(&waiter->work.task_list); + waiter->callback = callback; +} + +/* + * API for sync_timeline implementers + */ + +/** + * sync_timeline_create() - creates a sync object + * @ops: specifies the implementation ops for the object + * @size: size to allocate for this obj + * @name: sync_timeline name + * + * Creates a new sync_timeline which will use the implementation specified by + * @ops. @size bytes will be allocated allowing for implementation specific + * data to be kept after the generic sync_timeline struct. + */ +struct sync_timeline *sync_timeline_create(const struct sync_timeline_ops *ops, + int size, const char *name); + +/** + * sync_timeline_destroy() - destroys a sync object + * @obj: sync_timeline to destroy + * + * A sync implementation should call this when the @obj is going away + * (i.e. module unload.) @obj won't actually be freed until all its children + * sync_pts are freed. + */ +void sync_timeline_destroy(struct sync_timeline *obj); + +/** + * sync_timeline_signal() - signal a status change on a sync_timeline + * @obj: sync_timeline to signal + * + * A sync implementation should call this any time one of it's sync_pts + * has signaled or has an error condition. + */ +void sync_timeline_signal(struct sync_timeline *obj); + +/** + * sync_pt_create() - creates a sync pt + * @parent: sync_pt's parent sync_timeline + * @size: size to allocate for this pt + * + * Creates a new sync_pt as a child of @parent. @size bytes will be + * allocated allowing for implementation specific data to be kept after + * the generic sync_timeline struct. + */ +struct sync_pt *sync_pt_create(struct sync_timeline *parent, int size); + +/** + * sync_pt_free() - frees a sync pt + * @pt: sync_pt to free + * + * This should only be called on sync_pts which have been created but + * not added to a fence. + */ +void sync_pt_free(struct sync_pt *pt); + +/** + * sync_fence_create() - creates a sync fence + * @name: name of fence to create + * @pt: sync_pt to add to the fence + * + * Creates a fence containg @pt. Once this is called, the fence takes + * ownership of @pt. + */ +struct sync_fence *sync_fence_create(const char *name, struct sync_pt *pt); + +/** + * sync_fence_create_dma() - creates a sync fence from dma-fence + * @name: name of fence to create + * @pt: dma-fence to add to the fence + * + * Creates a fence containg @pt. Once this is called, the fence takes + * ownership of @pt. + */ +struct sync_fence *sync_fence_create_dma(const char *name, struct fence *pt); + +/* + * API for sync_fence consumers + */ + +/** + * sync_fence_merge() - merge two fences + * @name: name of new fence + * @a: fence a + * @b: fence b + * + * Creates a new fence which contains copies of all the sync_pts in both + * @a and @b. @a and @b remain valid, independent fences. + */ +struct sync_fence *sync_fence_merge(const char *name, + struct sync_fence *a, struct sync_fence *b); + +/** + * sync_fence_fdget() - get a fence from an fd + * @fd: fd referencing a fence + * + * Ensures @fd references a valid fence, increments the refcount of the backing + * file, and returns the fence. + */ +struct sync_fence *sync_fence_fdget(int fd); + +/** + * sync_fence_put() - puts a reference of a sync fence + * @fence: fence to put + * + * Puts a reference on @fence. If this is the last reference, the fence and + * all it's sync_pts will be freed + */ +void sync_fence_put(struct sync_fence *fence); + +/** + * sync_fence_install() - installs a fence into a file descriptor + * @fence: fence to install + * @fd: file descriptor in which to install the fence + * + * Installs @fence into @fd. @fd's should be acquired through + * get_unused_fd_flags(O_CLOEXEC). + */ +void sync_fence_install(struct sync_fence *fence, int fd); + +/** + * sync_fence_wait_async() - registers and async wait on the fence + * @fence: fence to wait on + * @waiter: waiter callback struck + * + * Returns 1 if @fence has already signaled. + * + * Registers a callback to be called when @fence signals or has an error. + * @waiter should be initialized with sync_fence_waiter_init(). + */ +int sync_fence_wait_async(struct sync_fence *fence, + struct sync_fence_waiter *waiter); + +/** + * sync_fence_cancel_async() - cancels an async wait + * @fence: fence to wait on + * @waiter: waiter callback struck + * + * returns 0 if waiter was removed from fence's async waiter list. + * returns -ENOENT if waiter was not found on fence's async waiter list. + * + * Cancels a previously registered async wait. Will fail gracefully if + * @waiter was never registered or if @fence has already signaled @waiter. + */ +int sync_fence_cancel_async(struct sync_fence *fence, + struct sync_fence_waiter *waiter); + +/** + * sync_fence_wait() - wait on fence + * @fence: fence to wait on + * @tiemout: timeout in ms + * + * Wait for @fence to be signaled or have an error. Waits indefinitely + * if @timeout < 0 + */ +int sync_fence_wait(struct sync_fence *fence, long timeout); + +#ifdef CONFIG_DEBUG_FS + +void sync_timeline_debug_add(struct sync_timeline *obj); +void sync_timeline_debug_remove(struct sync_timeline *obj); +void sync_fence_debug_add(struct sync_fence *fence); +void sync_fence_debug_remove(struct sync_fence *fence); +void sync_dump(void); + +#else +# define sync_timeline_debug_add(obj) +# define sync_timeline_debug_remove(obj) +# define sync_fence_debug_add(fence) +# define sync_fence_debug_remove(fence) +# define sync_dump() +#endif +int sync_fence_wake_up_wq(wait_queue_t *curr, unsigned mode, + int wake_flags, void *key); + +#endif /* _LINUX_SYNC_H */ diff --git a/drivers/staging/android/sync_debug.c b/drivers/staging/android/sync_debug.c new file mode 100644 index 00000000000000..f45d13cdd42b90 --- /dev/null +++ b/drivers/staging/android/sync_debug.c @@ -0,0 +1,256 @@ +/* + * drivers/base/sync.c + * + * Copyright (C) 2012 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "sync.h" + +#ifdef CONFIG_DEBUG_FS + +static LIST_HEAD(sync_timeline_list_head); +static DEFINE_SPINLOCK(sync_timeline_list_lock); +static LIST_HEAD(sync_fence_list_head); +static DEFINE_SPINLOCK(sync_fence_list_lock); + +void sync_timeline_debug_add(struct sync_timeline *obj) +{ + unsigned long flags; + + spin_lock_irqsave(&sync_timeline_list_lock, flags); + list_add_tail(&obj->sync_timeline_list, &sync_timeline_list_head); + spin_unlock_irqrestore(&sync_timeline_list_lock, flags); +} + +void sync_timeline_debug_remove(struct sync_timeline *obj) +{ + unsigned long flags; + + spin_lock_irqsave(&sync_timeline_list_lock, flags); + list_del(&obj->sync_timeline_list); + spin_unlock_irqrestore(&sync_timeline_list_lock, flags); +} + +void sync_fence_debug_add(struct sync_fence *fence) +{ + unsigned long flags; + + spin_lock_irqsave(&sync_fence_list_lock, flags); + list_add_tail(&fence->sync_fence_list, &sync_fence_list_head); + spin_unlock_irqrestore(&sync_fence_list_lock, flags); +} + +void sync_fence_debug_remove(struct sync_fence *fence) +{ + unsigned long flags; + + spin_lock_irqsave(&sync_fence_list_lock, flags); + list_del(&fence->sync_fence_list); + spin_unlock_irqrestore(&sync_fence_list_lock, flags); +} + +static const char *sync_status_str(int status) +{ + if (status == 0) + return "signaled"; + + if (status > 0) + return "active"; + + return "error"; +} + +static void sync_print_pt(struct seq_file *s, struct fence *pt, bool fence) +{ + int status = 1; + + if (fence_is_signaled_locked(pt)) + status = pt->status; + + seq_printf(s, " %s%spt %s", + fence && pt->ops->get_timeline_name ? + pt->ops->get_timeline_name(pt) : "", + fence ? "_" : "", + sync_status_str(status)); + + if (status <= 0) { + struct timespec64 ts64 = + ktime_to_timespec64(pt->timestamp); + + seq_printf(s, "@%lld.%09ld", (s64)ts64.tv_sec, ts64.tv_nsec); + } + + if ((!fence || pt->ops->timeline_value_str) && + pt->ops->fence_value_str) { + char value[64]; + bool success; + + pt->ops->fence_value_str(pt, value, sizeof(value)); + success = strlen(value); + + if (success) + seq_printf(s, ": %s", value); + + if (success && fence) { + pt->ops->timeline_value_str(pt, value, sizeof(value)); + + if (strlen(value)) + seq_printf(s, " / %s", value); + } + } + + seq_puts(s, "\n"); +} + +static void sync_print_obj(struct seq_file *s, struct sync_timeline *obj) +{ + struct list_head *pos; + unsigned long flags; + + seq_printf(s, "%s %s", obj->name, obj->ops->driver_name); + + if (obj->ops->timeline_value_str) { + char value[64]; + + obj->ops->timeline_value_str(obj, value, sizeof(value)); + seq_printf(s, ": %s", value); + } + + seq_puts(s, "\n"); + + spin_lock_irqsave(&obj->child_list_lock, flags); + list_for_each(pos, &obj->child_list_head) { + struct sync_pt *pt = + container_of(pos, struct sync_pt, child_list); + sync_print_pt(s, &pt->base, false); + } + spin_unlock_irqrestore(&obj->child_list_lock, flags); +} + +static void sync_print_fence(struct seq_file *s, struct sync_fence *fence) +{ + wait_queue_t *pos; + unsigned long flags; + int i; + + seq_printf(s, "[%p] %s: %s\n", fence, fence->name, + sync_status_str(atomic_read(&fence->status))); + + for (i = 0; i < fence->num_fences; ++i) { + sync_print_pt(s, fence->cbs[i].sync_pt, true); + } + + spin_lock_irqsave(&fence->wq.lock, flags); + list_for_each_entry(pos, &fence->wq.task_list, task_list) { + struct sync_fence_waiter *waiter; + + if (pos->func != &sync_fence_wake_up_wq) + continue; + + waiter = container_of(pos, struct sync_fence_waiter, work); + + seq_printf(s, "waiter %pF\n", waiter->callback); + } + spin_unlock_irqrestore(&fence->wq.lock, flags); +} + +static int sync_debugfs_show(struct seq_file *s, void *unused) +{ + unsigned long flags; + struct list_head *pos; + + seq_puts(s, "objs:\n--------------\n"); + + spin_lock_irqsave(&sync_timeline_list_lock, flags); + list_for_each(pos, &sync_timeline_list_head) { + struct sync_timeline *obj = + container_of(pos, struct sync_timeline, + sync_timeline_list); + + sync_print_obj(s, obj); + seq_puts(s, "\n"); + } + spin_unlock_irqrestore(&sync_timeline_list_lock, flags); + + seq_puts(s, "fences:\n--------------\n"); + + spin_lock_irqsave(&sync_fence_list_lock, flags); + list_for_each(pos, &sync_fence_list_head) { + struct sync_fence *fence = + container_of(pos, struct sync_fence, sync_fence_list); + + sync_print_fence(s, fence); + seq_puts(s, "\n"); + } + spin_unlock_irqrestore(&sync_fence_list_lock, flags); + return 0; +} + +static int sync_debugfs_open(struct inode *inode, struct file *file) +{ + return single_open(file, sync_debugfs_show, inode->i_private); +} + +static const struct file_operations sync_debugfs_fops = { + .open = sync_debugfs_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static __init int sync_debugfs_init(void) +{ + debugfs_create_file("sync", S_IRUGO, NULL, NULL, &sync_debugfs_fops); + return 0; +} +late_initcall(sync_debugfs_init); + +#define DUMP_CHUNK 256 +static char sync_dump_buf[64 * 1024]; +void sync_dump(void) +{ + struct seq_file s = { + .buf = sync_dump_buf, + .size = sizeof(sync_dump_buf) - 1, + }; + int i; + + sync_debugfs_show(&s, NULL); + + for (i = 0; i < s.count; i += DUMP_CHUNK) { + if ((s.count - i) > DUMP_CHUNK) { + char c = s.buf[i + DUMP_CHUNK]; + + s.buf[i + DUMP_CHUNK] = 0; + pr_cont("%s", s.buf + i); + s.buf[i + DUMP_CHUNK] = c; + } else { + s.buf[s.count] = 0; + pr_cont("%s", s.buf + i); + } + } +} + +#endif diff --git a/drivers/staging/android/trace/lowmemorykiller.h b/drivers/staging/android/trace/lowmemorykiller.h new file mode 100644 index 00000000000000..f43d3fae75eeb2 --- /dev/null +++ b/drivers/staging/android/trace/lowmemorykiller.h @@ -0,0 +1,41 @@ +#undef TRACE_SYSTEM +#define TRACE_INCLUDE_PATH ../../drivers/staging/android/trace +#define TRACE_SYSTEM lowmemorykiller + +#if !defined(_TRACE_LOWMEMORYKILLER_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_LOWMEMORYKILLER_H + +#include + +TRACE_EVENT(lowmemory_kill, + TP_PROTO(struct task_struct *killed_task, long cache_size, \ + long cache_limit, long free), + + TP_ARGS(killed_task, cache_size, cache_limit, free), + + TP_STRUCT__entry( + __array(char, comm, TASK_COMM_LEN) + __field(pid_t, pid) + __field(long, pagecache_size) + __field(long, pagecache_limit) + __field(long, free) + ), + + TP_fast_assign( + memcpy(__entry->comm, killed_task->comm, TASK_COMM_LEN); + __entry->pid = killed_task->pid; + __entry->pagecache_size = cache_size; + __entry->pagecache_limit = cache_limit; + __entry->free = free; + ), + + TP_printk("%s (%d), page cache %ldkB (limit %ldkB), free %ldKb", + __entry->comm, __entry->pid, __entry->pagecache_size, + __entry->pagecache_limit, __entry->free) +); + + +#endif /* if !defined(_TRACE_LOWMEMORYKILLER_H) || defined(TRACE_HEADER_MULTI_READ) */ + +/* This part must be outside protection */ +#include diff --git a/drivers/staging/android/trace/sync.h b/drivers/staging/android/trace/sync.h new file mode 100644 index 00000000000000..77edb977a7bfa9 --- /dev/null +++ b/drivers/staging/android/trace/sync.h @@ -0,0 +1,82 @@ +#undef TRACE_SYSTEM +#define TRACE_INCLUDE_PATH ../../drivers/staging/android/trace +#define TRACE_SYSTEM sync + +#if !defined(_TRACE_SYNC_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_SYNC_H + +#include "../sync.h" +#include + +TRACE_EVENT(sync_timeline, + TP_PROTO(struct sync_timeline *timeline), + + TP_ARGS(timeline), + + TP_STRUCT__entry( + __string(name, timeline->name) + __array(char, value, 32) + ), + + TP_fast_assign( + __assign_str(name, timeline->name); + if (timeline->ops->timeline_value_str) { + timeline->ops->timeline_value_str(timeline, + __entry->value, + sizeof(__entry->value)); + } else { + __entry->value[0] = '\0'; + } + ), + + TP_printk("name=%s value=%s", __get_str(name), __entry->value) +); + +TRACE_EVENT(sync_wait, + TP_PROTO(struct sync_fence *fence, int begin), + + TP_ARGS(fence, begin), + + TP_STRUCT__entry( + __string(name, fence->name) + __field(s32, status) + __field(u32, begin) + ), + + TP_fast_assign( + __assign_str(name, fence->name); + __entry->status = atomic_read(&fence->status); + __entry->begin = begin; + ), + + TP_printk("%s name=%s state=%d", __entry->begin ? "begin" : "end", + __get_str(name), __entry->status) +); + +TRACE_EVENT(sync_pt, + TP_PROTO(struct fence *pt), + + TP_ARGS(pt), + + TP_STRUCT__entry( + __string(timeline, pt->ops->get_timeline_name(pt)) + __array(char, value, 32) + ), + + TP_fast_assign( + __assign_str(timeline, pt->ops->get_timeline_name(pt)); + if (pt->ops->fence_value_str) { + pt->ops->fence_value_str(pt, __entry->value, + sizeof(__entry->value)); + } else { + __entry->value[0] = '\0'; + } + ), + + TP_printk("name=%s value=%s", __get_str(timeline), __entry->value) +); + +#endif /* if !defined(_TRACE_SYNC_H) || defined(TRACE_HEADER_MULTI_READ) */ + +/* This part must be outside protection */ +#include diff --git a/drivers/staging/android/uapi/ashmem.h b/drivers/staging/android/uapi/ashmem.h index 13df42d200b7cc..ba4743c71d6b45 100644 --- a/drivers/staging/android/uapi/ashmem.h +++ b/drivers/staging/android/uapi/ashmem.h @@ -13,7 +13,6 @@ #define _UAPI_LINUX_ASHMEM_H #include -#include #define ASHMEM_NAME_LEN 256 diff --git a/drivers/staging/android/uapi/sw_sync.h b/drivers/staging/android/uapi/sw_sync.h new file mode 100644 index 00000000000000..9b5d4869505cb4 --- /dev/null +++ b/drivers/staging/android/uapi/sw_sync.h @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2012 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#ifndef _UAPI_LINUX_SW_SYNC_H +#define _UAPI_LINUX_SW_SYNC_H + +#include + +struct sw_sync_create_fence_data { + __u32 value; + char name[32]; + __s32 fence; /* fd of new fence */ +}; + +#define SW_SYNC_IOC_MAGIC 'W' + +#define SW_SYNC_IOC_CREATE_FENCE _IOWR(SW_SYNC_IOC_MAGIC, 0,\ + struct sw_sync_create_fence_data) +#define SW_SYNC_IOC_INC _IOW(SW_SYNC_IOC_MAGIC, 1, __u32) + +#endif /* _UAPI_LINUX_SW_SYNC_H */ diff --git a/drivers/staging/android/uapi/sync.h b/drivers/staging/android/uapi/sync.h new file mode 100644 index 00000000000000..e964c751f6b8f6 --- /dev/null +++ b/drivers/staging/android/uapi/sync.h @@ -0,0 +1,97 @@ +/* + * Copyright (C) 2012 Google, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#ifndef _UAPI_LINUX_SYNC_H +#define _UAPI_LINUX_SYNC_H + +#include +#include + +/** + * struct sync_merge_data - data passed to merge ioctl + * @fd2: file descriptor of second fence + * @name: name of new fence + * @fence: returns the fd of the new fence to userspace + */ +struct sync_merge_data { + __s32 fd2; /* fd of second fence */ + char name[32]; /* name of new fence */ + __s32 fence; /* fd on newly created fence */ +}; + +/** + * struct sync_pt_info - detailed sync_pt information + * @len: length of sync_pt_info including any driver_data + * @obj_name: name of parent sync_timeline + * @driver_name: name of driver implementing the parent + * @status: status of the sync_pt 0:active 1:signaled <0:error + * @timestamp_ns: timestamp of status change in nanoseconds + * @driver_data: any driver dependent data + */ +struct sync_pt_info { + __u32 len; + char obj_name[32]; + char driver_name[32]; + __s32 status; + __u64 timestamp_ns; + + __u8 driver_data[0]; +}; + +/** + * struct sync_fence_info_data - data returned from fence info ioctl + * @len: ioctl caller writes the size of the buffer its passing in. + * ioctl returns length of sync_fence_data returned to userspace + * including pt_info. + * @name: name of fence + * @status: status of fence. 1: signaled 0:active <0:error + * @pt_info: a sync_pt_info struct for every sync_pt in the fence + */ +struct sync_fence_info_data { + __u32 len; + char name[32]; + __s32 status; + + __u8 pt_info[0]; +}; + +#define SYNC_IOC_MAGIC '>' + +/** + * DOC: SYNC_IOC_WAIT - wait for a fence to signal + * + * pass timeout in milliseconds. Waits indefinitely timeout < 0. + */ +#define SYNC_IOC_WAIT _IOW(SYNC_IOC_MAGIC, 0, __s32) + +/** + * DOC: SYNC_IOC_MERGE - merge two fences + * + * Takes a struct sync_merge_data. Creates a new fence containing copies of + * the sync_pts in both the calling fd and sync_merge_data.fd2. Returns the + * new fence's fd in sync_merge_data.fence + */ +#define SYNC_IOC_MERGE _IOWR(SYNC_IOC_MAGIC, 1, struct sync_merge_data) + +/** + * DOC: SYNC_IOC_FENCE_INFO - get detailed information on a fence + * + * Takes a struct sync_fence_info_data with extra space allocated for pt_info. + * Caller should write the size of the buffer into len. On return, len is + * updated to reflect the total size of the sync_fence_info_data including + * pt_info. + * + * pt_info is a buffer containing sync_pt_infos for every sync_pt in the fence. + * To iterate over the sync_pt_infos, use the sync_pt_info.len field. + */ +#define SYNC_IOC_FENCE_INFO _IOWR(SYNC_IOC_MAGIC, 2,\ + struct sync_fence_info_data) + +#endif /* _UAPI_LINUX_SYNC_H */ diff --git a/drivers/staging/comedi/drivers/ni_mio_common.c b/drivers/staging/comedi/drivers/ni_mio_common.c index 0f97d7b611d720..1c967c30e4ce98 100644 --- a/drivers/staging/comedi/drivers/ni_mio_common.c +++ b/drivers/staging/comedi/drivers/ni_mio_common.c @@ -1832,7 +1832,7 @@ static int ni_ai_insn_read(struct comedi_device *dev, unsigned int *data) { struct ni_private *devpriv = dev->private; - unsigned int mask = (s->maxdata + 1) >> 1; + unsigned int mask = s->maxdata; int i, n; unsigned int signbits; unsigned int d; @@ -1875,7 +1875,7 @@ static int ni_ai_insn_read(struct comedi_device *dev, return -ETIME; } d += signbits; - data[n] = d; + data[n] = d & 0xffff; } } else if (devpriv->is_6143) { for (n = 0; n < insn->n; n++) { @@ -1924,9 +1924,8 @@ static int ni_ai_insn_read(struct comedi_device *dev, data[n] = dl; } else { d = ni_readw(dev, NI_E_AI_FIFO_DATA_REG); - /* subtle: needs to be short addition */ d += signbits; - data[n] = d; + data[n] = d & 0xffff; } } } diff --git a/drivers/staging/goldfish/Kconfig b/drivers/staging/goldfish/Kconfig index 4e094602437c3d..d293bbc22c792a 100644 --- a/drivers/staging/goldfish/Kconfig +++ b/drivers/staging/goldfish/Kconfig @@ -4,6 +4,14 @@ config GOLDFISH_AUDIO ---help--- Emulated audio channel for the Goldfish Android Virtual Device +config GOLDFISH_SYNC + tristate "Goldfish AVD Sync Driver" + depends on GOLDFISH + depends on SW_SYNC + depends on SYNC_FILE + ---help--- + Emulated sync fences for the Goldfish Android Virtual Device + config MTD_GOLDFISH_NAND tristate "Goldfish NAND device" depends on GOLDFISH diff --git a/drivers/staging/goldfish/Makefile b/drivers/staging/goldfish/Makefile index dec34ad58162fd..3313fce4e940d7 100644 --- a/drivers/staging/goldfish/Makefile +++ b/drivers/staging/goldfish/Makefile @@ -4,3 +4,9 @@ obj-$(CONFIG_GOLDFISH_AUDIO) += goldfish_audio.o obj-$(CONFIG_MTD_GOLDFISH_NAND) += goldfish_nand.o + +# and sync + +ccflags-y := -Idrivers/staging/android +goldfish_sync-objs := goldfish_sync_timeline_fence.o goldfish_sync_timeline.o +obj-$(CONFIG_GOLDFISH_SYNC) += goldfish_sync.o diff --git a/drivers/staging/goldfish/goldfish_audio.c b/drivers/staging/goldfish/goldfish_audio.c index bd559956f19900..0bb0ee2e691f1a 100644 --- a/drivers/staging/goldfish/goldfish_audio.c +++ b/drivers/staging/goldfish/goldfish_audio.c @@ -28,6 +28,7 @@ #include #include #include +#include MODULE_AUTHOR("Google, Inc."); MODULE_DESCRIPTION("Android QEMU Audio Driver"); @@ -116,6 +117,7 @@ static ssize_t goldfish_audio_read(struct file *fp, char __user *buf, size_t count, loff_t *pos) { struct goldfish_audio *data = fp->private_data; + unsigned long irq_flags; int length; int result = 0; @@ -129,6 +131,10 @@ static ssize_t goldfish_audio_read(struct file *fp, char __user *buf, wait_event_interruptible(data->wait, data->buffer_status & AUDIO_INT_READ_BUFFER_FULL); + spin_lock_irqsave(&data->lock, irq_flags); + data->buffer_status &= ~AUDIO_INT_READ_BUFFER_FULL; + spin_unlock_irqrestore(&data->lock, irq_flags); + length = AUDIO_READ(data, AUDIO_READ_BUFFER_AVAILABLE); /* copy data to user space */ @@ -351,12 +357,19 @@ static const struct of_device_id goldfish_audio_of_match[] = { }; MODULE_DEVICE_TABLE(of, goldfish_audio_of_match); +static const struct acpi_device_id goldfish_audio_acpi_match[] = { + { "GFSH0005", 0 }, + { }, +}; +MODULE_DEVICE_TABLE(acpi, goldfish_audio_acpi_match); + static struct platform_driver goldfish_audio_driver = { .probe = goldfish_audio_probe, .remove = goldfish_audio_remove, .driver = { .name = "goldfish_audio", .of_match_table = goldfish_audio_of_match, + .acpi_match_table = ACPI_PTR(goldfish_audio_acpi_match), } }; diff --git a/drivers/staging/goldfish/goldfish_sync_timeline.c b/drivers/staging/goldfish/goldfish_sync_timeline.c new file mode 100644 index 00000000000000..5bef4c6c02838a --- /dev/null +++ b/drivers/staging/goldfish/goldfish_sync_timeline.c @@ -0,0 +1,962 @@ +/* + * Copyright (C) 2016 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +#include + +#include +#include +#include +#include + +#include "goldfish_sync_timeline_fence.h" + +#define ERR(...) printk(KERN_ERR __VA_ARGS__); + +#define INFO(...) printk(KERN_INFO __VA_ARGS__); + +#define DPRINT(...) pr_debug(__VA_ARGS__); + +#define DTRACE() DPRINT("%s: enter", __func__) + +/* The Goldfish sync driver is designed to provide a interface + * between the underlying host's sync device and the kernel's + * fence sync framework.. + * The purpose of the device/driver is to enable lightweight + * creation and signaling of timelines and fences + * in order to synchronize the guest with host-side graphics events. + * + * Each time the interrupt trips, the driver + * may perform a sync operation. + */ + +/* The operations are: */ + +/* Ready signal - used to mark when irq should lower */ +#define CMD_SYNC_READY 0 + +/* Create a new timeline. writes timeline handle */ +#define CMD_CREATE_SYNC_TIMELINE 1 + +/* Create a fence object. reads timeline handle and time argument. + * Writes fence fd to the SYNC_REG_HANDLE register. */ +#define CMD_CREATE_SYNC_FENCE 2 + +/* Increments timeline. reads timeline handle and time argument */ +#define CMD_SYNC_TIMELINE_INC 3 + +/* Destroys a timeline. reads timeline handle */ +#define CMD_DESTROY_SYNC_TIMELINE 4 + +/* Starts a wait on the host with + * the given glsync object and sync thread handle. */ +#define CMD_TRIGGER_HOST_WAIT 5 + +/* The register layout is: */ + +#define SYNC_REG_BATCH_COMMAND 0x00 /* host->guest batch commands */ +#define SYNC_REG_BATCH_GUESTCOMMAND 0x04 /* guest->host batch commands */ +#define SYNC_REG_BATCH_COMMAND_ADDR 0x08 /* communicate physical address of host->guest batch commands */ +#define SYNC_REG_BATCH_COMMAND_ADDR_HIGH 0x0c /* 64-bit part */ +#define SYNC_REG_BATCH_GUESTCOMMAND_ADDR 0x10 /* communicate physical address of guest->host commands */ +#define SYNC_REG_BATCH_GUESTCOMMAND_ADDR_HIGH 0x14 /* 64-bit part */ +#define SYNC_REG_INIT 0x18 /* signals that the device has been probed */ + +/* There is an ioctl associated with goldfish sync driver. + * Make it conflict with ioctls that are not likely to be used + * in the emulator. + * + * '@' 00-0F linux/radeonfb.h conflict! + * '@' 00-0F drivers/video/aty/aty128fb.c conflict! + */ +#define GOLDFISH_SYNC_IOC_MAGIC '@' + +#define GOLDFISH_SYNC_IOC_QUEUE_WORK _IOWR(GOLDFISH_SYNC_IOC_MAGIC, 0, struct goldfish_sync_ioctl_info) + +/* The above definitions (command codes, register layout, ioctl definitions) + * need to be in sync with the following files: + * + * Host-side (emulator): + * external/qemu/android/emulation/goldfish_sync.h + * external/qemu-android/hw/misc/goldfish_sync.c + * + * Guest-side (system image): + * device/generic/goldfish-opengl/system/egl/goldfish_sync.h + * device/generic/goldfish/ueventd.ranchu.rc + * platform/build/target/board/generic/sepolicy/file_contexts + */ +struct goldfish_sync_hostcmd { + /* sorted for alignment */ + uint64_t handle; + uint64_t hostcmd_handle; + uint32_t cmd; + uint32_t time_arg; +}; + +struct goldfish_sync_guestcmd { + uint64_t host_command; /* uint64_t for alignment */ + uint64_t glsync_handle; + uint64_t thread_handle; + uint64_t guest_timeline_handle; +}; + +#define GOLDFISH_SYNC_MAX_CMDS 32 + +struct goldfish_sync_state { + char __iomem *reg_base; + int irq; + + /* Spinlock protects |to_do| / |to_do_end|. */ + spinlock_t lock; + /* |mutex_lock| protects all concurrent access + * to timelines for both kernel and user space. */ + struct mutex mutex_lock; + + /* Buffer holding commands issued from host. */ + struct goldfish_sync_hostcmd to_do[GOLDFISH_SYNC_MAX_CMDS]; + uint32_t to_do_end; + + /* Addresses for the reading or writing + * of individual commands. The host can directly write + * to |batch_hostcmd| (and then this driver immediately + * copies contents to |to_do|). This driver either replies + * through |batch_hostcmd| or simply issues a + * guest->host command through |batch_guestcmd|. + */ + struct goldfish_sync_hostcmd *batch_hostcmd; + struct goldfish_sync_guestcmd *batch_guestcmd; + + /* Used to give this struct itself to a work queue + * function for executing actual sync commands. */ + struct work_struct work_item; +}; + +static struct goldfish_sync_state global_sync_state[1]; + +struct goldfish_sync_timeline_obj { + struct goldfish_sync_timeline *sync_tl; + uint32_t current_time; + /* We need to be careful about when we deallocate + * this |goldfish_sync_timeline_obj| struct. + * In order to ensure proper cleanup, we need to + * consider the triggered host-side wait that may + * still be in flight when the guest close()'s a + * goldfish_sync device's sync context fd (and + * destroys the |sync_tl| field above). + * The host-side wait may raise IRQ + * and tell the kernel to increment the timeline _after_ + * the |sync_tl| has already been set to null. + * + * From observations on OpenGL apps and CTS tests, this + * happens at some very low probability upon context + * destruction or process close, but it does happen + * and it needs to be handled properly. Otherwise, + * if we clean up the surrounding |goldfish_sync_timeline_obj| + * too early, any |handle| field of any host->guest command + * might not even point to a null |sync_tl| field, + * but to garbage memory or even a reclaimed |sync_tl|. + * If we do not count such "pending waits" and kfree the object + * immediately upon |goldfish_sync_timeline_destroy|, + * we might get mysterous RCU stalls after running a long + * time because the garbage memory that is being read + * happens to be interpretable as a |spinlock_t| struct + * that is currently in the locked state. + * + * To track when to free the |goldfish_sync_timeline_obj| + * itself, we maintain a kref. + * The kref essentially counts the timeline itself plus + * the number of waits in flight. kref_init/kref_put + * are issued on + * |goldfish_sync_timeline_create|/|goldfish_sync_timeline_destroy| + * and kref_get/kref_put are issued on + * |goldfish_sync_fence_create|/|goldfish_sync_timeline_inc|. + * + * The timeline is destroyed after reference count + * reaches zero, which would happen after + * |goldfish_sync_timeline_destroy| and all pending + * |goldfish_sync_timeline_inc|'s are fulfilled. + * + * NOTE (1): We assume that |fence_create| and + * |timeline_inc| calls are 1:1, otherwise the kref scheme + * will not work. This is a valid assumption as long + * as the host-side virtual device implementation + * does not insert any timeline increments + * that we did not trigger from here. + * + * NOTE (2): The use of kref by itself requires no locks, + * but this does not mean everything works without locks. + * Related timeline operations do require a lock of some sort, + * or at least are not proven to work without it. + * In particualr, we assume that all the operations + * done on the |kref| field above are done in contexts where + * |global_sync_state->mutex_lock| is held. Do not + * remove that lock until everything is proven to work + * without it!!! */ + struct kref kref; +}; + +/* We will call |delete_timeline_obj| when the last reference count + * of the kref is decremented. This deletes the sync + * timeline object along with the wrapper itself. */ +static void delete_timeline_obj(struct kref* kref) { + struct goldfish_sync_timeline_obj* obj = + container_of(kref, struct goldfish_sync_timeline_obj, kref); + + goldfish_sync_timeline_put_internal(obj->sync_tl); + obj->sync_tl = NULL; + kfree(obj); +} + +static uint64_t gensym_ctr; +static void gensym(char *dst) +{ + sprintf(dst, "goldfish_sync:gensym:%llu", gensym_ctr); + gensym_ctr++; +} + +/* |goldfish_sync_timeline_create| assumes that |global_sync_state->mutex_lock| + * is held. */ +static struct goldfish_sync_timeline_obj* +goldfish_sync_timeline_create(void) +{ + + char timeline_name[256]; + struct goldfish_sync_timeline *res_sync_tl = NULL; + struct goldfish_sync_timeline_obj *res; + + DTRACE(); + + gensym(timeline_name); + + res_sync_tl = goldfish_sync_timeline_create_internal(timeline_name); + if (!res_sync_tl) { + ERR("Failed to create goldfish_sw_sync timeline."); + return NULL; + } + + res = kzalloc(sizeof(struct goldfish_sync_timeline_obj), GFP_KERNEL); + res->sync_tl = res_sync_tl; + res->current_time = 0; + kref_init(&res->kref); + + DPRINT("new timeline_obj=0x%p", res); + return res; +} + +/* |goldfish_sync_fence_create| assumes that |global_sync_state->mutex_lock| + * is held. */ +static int +goldfish_sync_fence_create(struct goldfish_sync_timeline_obj *obj, + uint32_t val) +{ + + int fd; + char fence_name[256]; + struct sync_pt *syncpt = NULL; + struct sync_file *sync_file_obj = NULL; + struct goldfish_sync_timeline *tl; + + DTRACE(); + + if (!obj) return -1; + + tl = obj->sync_tl; + + syncpt = goldfish_sync_pt_create_internal( + tl, sizeof(struct sync_pt) + 4, val); + if (!syncpt) { + ERR("could not create sync point! " + "goldfish_sync_timeline=0x%p val=%d", + tl, val); + return -1; + } + + fd = get_unused_fd_flags(O_CLOEXEC); + if (fd < 0) { + ERR("could not get unused fd for sync fence. " + "errno=%d", fd); + goto err_cleanup_pt; + } + + gensym(fence_name); + + sync_file_obj = sync_file_create(&syncpt->base); + if (!sync_file_obj) { + ERR("could not create sync fence! " + "goldfish_sync_timeline=0x%p val=%d sync_pt=0x%p", + tl, val, syncpt); + goto err_cleanup_fd_pt; + } + + DPRINT("installing sync fence into fd %d sync_file_obj=0x%p", + fd, sync_file_obj); + fd_install(fd, sync_file_obj->file); + kref_get(&obj->kref); + + return fd; + +err_cleanup_fd_pt: + put_unused_fd(fd); +err_cleanup_pt: + fence_put(&syncpt->base); + return -1; +} + +/* |goldfish_sync_timeline_inc| assumes that |global_sync_state->mutex_lock| + * is held. */ +static void +goldfish_sync_timeline_inc(struct goldfish_sync_timeline_obj *obj, uint32_t inc) +{ + DTRACE(); + /* Just give up if someone else nuked the timeline. + * Whoever it was won't care that it doesn't get signaled. */ + if (!obj) return; + + DPRINT("timeline_obj=0x%p", obj); + goldfish_sync_timeline_signal_internal(obj->sync_tl, inc); + DPRINT("incremented timeline. increment max_time"); + obj->current_time += inc; + + /* Here, we will end up deleting the timeline object if it + * turns out that this call was a pending increment after + * |goldfish_sync_timeline_destroy| was called. */ + kref_put(&obj->kref, delete_timeline_obj); + DPRINT("done"); +} + +/* |goldfish_sync_timeline_destroy| assumes + * that |global_sync_state->mutex_lock| is held. */ +static void +goldfish_sync_timeline_destroy(struct goldfish_sync_timeline_obj *obj) +{ + DTRACE(); + /* See description of |goldfish_sync_timeline_obj| for why we + * should not immediately destroy |obj| */ + kref_put(&obj->kref, delete_timeline_obj); +} + +static inline void +goldfish_sync_cmd_queue(struct goldfish_sync_state *sync_state, + uint32_t cmd, + uint64_t handle, + uint32_t time_arg, + uint64_t hostcmd_handle) +{ + struct goldfish_sync_hostcmd *to_add; + + DTRACE(); + + BUG_ON(sync_state->to_do_end == GOLDFISH_SYNC_MAX_CMDS); + + to_add = &sync_state->to_do[sync_state->to_do_end]; + + to_add->cmd = cmd; + to_add->handle = handle; + to_add->time_arg = time_arg; + to_add->hostcmd_handle = hostcmd_handle; + + sync_state->to_do_end += 1; +} + +static inline void +goldfish_sync_hostcmd_reply(struct goldfish_sync_state *sync_state, + uint32_t cmd, + uint64_t handle, + uint32_t time_arg, + uint64_t hostcmd_handle) +{ + unsigned long irq_flags; + struct goldfish_sync_hostcmd *batch_hostcmd = + sync_state->batch_hostcmd; + + DTRACE(); + + spin_lock_irqsave(&sync_state->lock, irq_flags); + + batch_hostcmd->cmd = cmd; + batch_hostcmd->handle = handle; + batch_hostcmd->time_arg = time_arg; + batch_hostcmd->hostcmd_handle = hostcmd_handle; + writel(0, sync_state->reg_base + SYNC_REG_BATCH_COMMAND); + + spin_unlock_irqrestore(&sync_state->lock, irq_flags); +} + +static inline void +goldfish_sync_send_guestcmd(struct goldfish_sync_state *sync_state, + uint32_t cmd, + uint64_t glsync_handle, + uint64_t thread_handle, + uint64_t timeline_handle) +{ + unsigned long irq_flags; + struct goldfish_sync_guestcmd *batch_guestcmd = + sync_state->batch_guestcmd; + + DTRACE(); + + spin_lock_irqsave(&sync_state->lock, irq_flags); + + batch_guestcmd->host_command = (uint64_t)cmd; + batch_guestcmd->glsync_handle = (uint64_t)glsync_handle; + batch_guestcmd->thread_handle = (uint64_t)thread_handle; + batch_guestcmd->guest_timeline_handle = (uint64_t)timeline_handle; + writel(0, sync_state->reg_base + SYNC_REG_BATCH_GUESTCOMMAND); + + spin_unlock_irqrestore(&sync_state->lock, irq_flags); +} + +/* |goldfish_sync_interrupt| handles IRQ raises from the virtual device. + * In the context of OpenGL, this interrupt will fire whenever we need + * to signal a fence fd in the guest, with the command + * |CMD_SYNC_TIMELINE_INC|. + * However, because this function will be called in an interrupt context, + * it is necessary to do the actual work of signaling off of interrupt context. + * The shared work queue is used for this purpose. At the end when + * all pending commands are intercepted by the interrupt handler, + * we call |schedule_work|, which will later run the actual + * desired sync command in |goldfish_sync_work_item_fn|. + */ +static irqreturn_t goldfish_sync_interrupt(int irq, void *dev_id) +{ + + struct goldfish_sync_state *sync_state = dev_id; + + uint32_t nextcmd; + uint32_t command_r; + uint64_t handle_rw; + uint32_t time_r; + uint64_t hostcmd_handle_rw; + + int count = 0; + + DTRACE(); + + sync_state = dev_id; + + spin_lock(&sync_state->lock); + + for (;;) { + + readl(sync_state->reg_base + SYNC_REG_BATCH_COMMAND); + nextcmd = sync_state->batch_hostcmd->cmd; + + if (nextcmd == 0) + break; + + command_r = nextcmd; + handle_rw = sync_state->batch_hostcmd->handle; + time_r = sync_state->batch_hostcmd->time_arg; + hostcmd_handle_rw = sync_state->batch_hostcmd->hostcmd_handle; + + goldfish_sync_cmd_queue( + sync_state, + command_r, + handle_rw, + time_r, + hostcmd_handle_rw); + + count++; + } + + spin_unlock(&sync_state->lock); + + schedule_work(&sync_state->work_item); + + return (count == 0) ? IRQ_NONE : IRQ_HANDLED; +} + +/* |goldfish_sync_work_item_fn| does the actual work of servicing + * host->guest sync commands. This function is triggered whenever + * the IRQ for the goldfish sync device is raised. Once it starts + * running, it grabs the contents of the buffer containing the + * commands it needs to execute (there may be multiple, because + * our IRQ is active high and not edge triggered), and then + * runs all of them one after the other. + */ +static void goldfish_sync_work_item_fn(struct work_struct *input) +{ + + struct goldfish_sync_state *sync_state; + int sync_fence_fd; + + struct goldfish_sync_timeline_obj *timeline; + uint64_t timeline_ptr; + + uint64_t hostcmd_handle; + + uint32_t cmd; + uint64_t handle; + uint32_t time_arg; + + struct goldfish_sync_hostcmd *todo; + uint32_t todo_end; + + unsigned long irq_flags; + + struct goldfish_sync_hostcmd to_run[GOLDFISH_SYNC_MAX_CMDS]; + uint32_t i = 0; + + sync_state = container_of(input, struct goldfish_sync_state, work_item); + + mutex_lock(&sync_state->mutex_lock); + + spin_lock_irqsave(&sync_state->lock, irq_flags); { + + todo_end = sync_state->to_do_end; + + DPRINT("num sync todos: %u", sync_state->to_do_end); + + for (i = 0; i < todo_end; i++) + to_run[i] = sync_state->to_do[i]; + + /* We expect that commands will come in at a slow enough rate + * so that incoming items will not be more than + * GOLDFISH_SYNC_MAX_CMDS. + * + * This is because the way the sync device is used, + * it's only for managing buffer data transfers per frame, + * with a sequential dependency between putting things in + * to_do and taking them out. Once a set of commands is + * queued up in to_do, the user of the device waits for + * them to be processed before queuing additional commands, + * which limits the rate at which commands come in + * to the rate at which we take them out here. + * + * We also don't expect more than MAX_CMDS to be issued + * at once; there is a correspondence between + * which buffers need swapping to the (display / buffer queue) + * to particular commands, and we don't expect there to be + * enough display or buffer queues in operation at once + * to overrun GOLDFISH_SYNC_MAX_CMDS. + */ + sync_state->to_do_end = 0; + + } spin_unlock_irqrestore(&sync_state->lock, irq_flags); + + for (i = 0; i < todo_end; i++) { + DPRINT("todo index: %u", i); + + todo = &to_run[i]; + + cmd = todo->cmd; + + handle = (uint64_t)todo->handle; + time_arg = todo->time_arg; + hostcmd_handle = (uint64_t)todo->hostcmd_handle; + + DTRACE(); + + timeline = (struct goldfish_sync_timeline_obj *)(uintptr_t)handle; + + switch (cmd) { + case CMD_SYNC_READY: + break; + case CMD_CREATE_SYNC_TIMELINE: + DPRINT("exec CMD_CREATE_SYNC_TIMELINE: " + "handle=0x%llx time_arg=%d", + handle, time_arg); + timeline = goldfish_sync_timeline_create(); + timeline_ptr = (uintptr_t)timeline; + goldfish_sync_hostcmd_reply(sync_state, CMD_CREATE_SYNC_TIMELINE, + timeline_ptr, + 0, + hostcmd_handle); + DPRINT("sync timeline created: %p", timeline); + break; + case CMD_CREATE_SYNC_FENCE: + DPRINT("exec CMD_CREATE_SYNC_FENCE: " + "handle=0x%llx time_arg=%d", + handle, time_arg); + sync_fence_fd = goldfish_sync_fence_create(timeline, time_arg); + goldfish_sync_hostcmd_reply(sync_state, CMD_CREATE_SYNC_FENCE, + sync_fence_fd, + 0, + hostcmd_handle); + break; + case CMD_SYNC_TIMELINE_INC: + DPRINT("exec CMD_SYNC_TIMELINE_INC: " + "handle=0x%llx time_arg=%d", + handle, time_arg); + goldfish_sync_timeline_inc(timeline, time_arg); + break; + case CMD_DESTROY_SYNC_TIMELINE: + DPRINT("exec CMD_DESTROY_SYNC_TIMELINE: " + "handle=0x%llx time_arg=%d", + handle, time_arg); + goldfish_sync_timeline_destroy(timeline); + break; + } + DPRINT("Done executing sync command"); + } + mutex_unlock(&sync_state->mutex_lock); +} + +/* Guest-side interface: file operations */ + +/* Goldfish sync context and ioctl info. + * + * When a sync context is created by open()-ing the goldfish sync device, we + * create a sync context (|goldfish_sync_context|). + * + * Currently, the only data required to track is the sync timeline itself + * along with the current time, which are all packed up in the + * |goldfish_sync_timeline_obj| field. We use a |goldfish_sync_context| + * as the filp->private_data. + * + * Next, when a sync context user requests that work be queued and a fence + * fd provided, we use the |goldfish_sync_ioctl_info| struct, which holds + * information about which host handles to touch for this particular + * queue-work operation. We need to know about the host-side sync thread + * and the particular host-side GLsync object. We also possibly write out + * a file descriptor. + */ +struct goldfish_sync_context { + struct goldfish_sync_timeline_obj *timeline; +}; + +struct goldfish_sync_ioctl_info { + uint64_t host_glsync_handle_in; + uint64_t host_syncthread_handle_in; + int fence_fd_out; +}; + +static int goldfish_sync_open(struct inode *inode, struct file *file) +{ + + struct goldfish_sync_context *sync_context; + + DTRACE(); + + mutex_lock(&global_sync_state->mutex_lock); + + sync_context = kzalloc(sizeof(struct goldfish_sync_context), GFP_KERNEL); + + if (sync_context == NULL) { + ERR("Creation of goldfish sync context failed!"); + mutex_unlock(&global_sync_state->mutex_lock); + return -ENOMEM; + } + + sync_context->timeline = NULL; + + file->private_data = sync_context; + + DPRINT("successfully create a sync context @0x%p", sync_context); + + mutex_unlock(&global_sync_state->mutex_lock); + + return 0; +} + +static int goldfish_sync_release(struct inode *inode, struct file *file) +{ + + struct goldfish_sync_context *sync_context; + + DTRACE(); + + mutex_lock(&global_sync_state->mutex_lock); + + sync_context = file->private_data; + + if (sync_context->timeline) + goldfish_sync_timeline_destroy(sync_context->timeline); + + sync_context->timeline = NULL; + + kfree(sync_context); + + mutex_unlock(&global_sync_state->mutex_lock); + + return 0; +} + +/* |goldfish_sync_ioctl| is the guest-facing interface of goldfish sync + * and is used in conjunction with eglCreateSyncKHR to queue up the + * actual work of waiting for the EGL sync command to complete, + * possibly returning a fence fd to the guest. + */ +static long goldfish_sync_ioctl(struct file *file, + unsigned int cmd, + unsigned long arg) +{ + struct goldfish_sync_context *sync_context_data; + struct goldfish_sync_timeline_obj *timeline; + int fd_out; + struct goldfish_sync_ioctl_info ioctl_data; + + DTRACE(); + + sync_context_data = file->private_data; + fd_out = -1; + + switch (cmd) { + case GOLDFISH_SYNC_IOC_QUEUE_WORK: + + DPRINT("exec GOLDFISH_SYNC_IOC_QUEUE_WORK"); + + mutex_lock(&global_sync_state->mutex_lock); + + if (copy_from_user(&ioctl_data, + (void __user *)arg, + sizeof(ioctl_data))) { + ERR("Failed to copy memory for ioctl_data from user."); + mutex_unlock(&global_sync_state->mutex_lock); + return -EFAULT; + } + + if (ioctl_data.host_syncthread_handle_in == 0) { + DPRINT("Error: zero host syncthread handle!!!"); + mutex_unlock(&global_sync_state->mutex_lock); + return -EFAULT; + } + + if (!sync_context_data->timeline) { + DPRINT("no timeline yet, create one."); + sync_context_data->timeline = goldfish_sync_timeline_create(); + DPRINT("timeline: 0x%p", &sync_context_data->timeline); + } + + timeline = sync_context_data->timeline; + fd_out = goldfish_sync_fence_create(timeline, + timeline->current_time + 1); + DPRINT("Created fence with fd %d and current time %u (timeline: 0x%p)", + fd_out, + sync_context_data->timeline->current_time + 1, + sync_context_data->timeline); + + ioctl_data.fence_fd_out = fd_out; + + if (copy_to_user((void __user *)arg, + &ioctl_data, + sizeof(ioctl_data))) { + DPRINT("Error, could not copy to user!!!"); + + sys_close(fd_out); + /* We won't be doing an increment, kref_put immediately. */ + kref_put(&timeline->kref, delete_timeline_obj); + mutex_unlock(&global_sync_state->mutex_lock); + return -EFAULT; + } + + /* We are now about to trigger a host-side wait; + * accumulate on |pending_waits|. */ + goldfish_sync_send_guestcmd(global_sync_state, + CMD_TRIGGER_HOST_WAIT, + ioctl_data.host_glsync_handle_in, + ioctl_data.host_syncthread_handle_in, + (uint64_t)(uintptr_t)(sync_context_data->timeline)); + + mutex_unlock(&global_sync_state->mutex_lock); + return 0; + default: + return -ENOTTY; + } +} + +static const struct file_operations goldfish_sync_fops = { + .owner = THIS_MODULE, + .open = goldfish_sync_open, + .release = goldfish_sync_release, + .unlocked_ioctl = goldfish_sync_ioctl, + .compat_ioctl = goldfish_sync_ioctl, +}; + +static struct miscdevice goldfish_sync_device = { + .name = "goldfish_sync", + .fops = &goldfish_sync_fops, +}; + + +static bool setup_verify_batch_cmd_addr(struct goldfish_sync_state *sync_state, + void *batch_addr, + uint32_t addr_offset, + uint32_t addr_offset_high) +{ + uint64_t batch_addr_phys; + uint32_t batch_addr_phys_test_lo; + uint32_t batch_addr_phys_test_hi; + + if (!batch_addr) { + ERR("Could not use batch command address!"); + return false; + } + + batch_addr_phys = virt_to_phys(batch_addr); + writel((uint32_t)(batch_addr_phys), + sync_state->reg_base + addr_offset); + writel((uint32_t)(batch_addr_phys >> 32), + sync_state->reg_base + addr_offset_high); + + batch_addr_phys_test_lo = + readl(sync_state->reg_base + addr_offset); + batch_addr_phys_test_hi = + readl(sync_state->reg_base + addr_offset_high); + + if (virt_to_phys(batch_addr) != + (((uint64_t)batch_addr_phys_test_hi << 32) | + batch_addr_phys_test_lo)) { + ERR("Invalid batch command address!"); + return false; + } + + return true; +} + +int goldfish_sync_probe(struct platform_device *pdev) +{ + struct resource *ioresource; + struct goldfish_sync_state *sync_state = global_sync_state; + int status; + + DTRACE(); + + sync_state->to_do_end = 0; + + spin_lock_init(&sync_state->lock); + mutex_init(&sync_state->mutex_lock); + + platform_set_drvdata(pdev, sync_state); + + ioresource = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (ioresource == NULL) { + ERR("platform_get_resource failed"); + return -ENODEV; + } + + sync_state->reg_base = + devm_ioremap(&pdev->dev, ioresource->start, PAGE_SIZE); + if (sync_state->reg_base == NULL) { + ERR("Could not ioremap"); + return -ENOMEM; + } + + sync_state->irq = platform_get_irq(pdev, 0); + if (sync_state->irq < 0) { + ERR("Could not platform_get_irq"); + return -ENODEV; + } + + status = devm_request_irq(&pdev->dev, + sync_state->irq, + goldfish_sync_interrupt, + IRQF_SHARED, + pdev->name, + sync_state); + if (status) { + ERR("request_irq failed"); + return -ENODEV; + } + + INIT_WORK(&sync_state->work_item, + goldfish_sync_work_item_fn); + + misc_register(&goldfish_sync_device); + + /* Obtain addresses for batch send/recv of commands. */ + { + struct goldfish_sync_hostcmd *batch_addr_hostcmd; + struct goldfish_sync_guestcmd *batch_addr_guestcmd; + + batch_addr_hostcmd = + devm_kzalloc(&pdev->dev, sizeof(struct goldfish_sync_hostcmd), + GFP_KERNEL); + batch_addr_guestcmd = + devm_kzalloc(&pdev->dev, sizeof(struct goldfish_sync_guestcmd), + GFP_KERNEL); + + if (!setup_verify_batch_cmd_addr(sync_state, + batch_addr_hostcmd, + SYNC_REG_BATCH_COMMAND_ADDR, + SYNC_REG_BATCH_COMMAND_ADDR_HIGH)) { + ERR("goldfish_sync: Could not setup batch command address"); + return -ENODEV; + } + + if (!setup_verify_batch_cmd_addr(sync_state, + batch_addr_guestcmd, + SYNC_REG_BATCH_GUESTCOMMAND_ADDR, + SYNC_REG_BATCH_GUESTCOMMAND_ADDR_HIGH)) { + ERR("goldfish_sync: Could not setup batch guest command address"); + return -ENODEV; + } + + sync_state->batch_hostcmd = batch_addr_hostcmd; + sync_state->batch_guestcmd = batch_addr_guestcmd; + } + + INFO("goldfish_sync: Initialized goldfish sync device"); + + writel(0, sync_state->reg_base + SYNC_REG_INIT); + + return 0; +} + +static int goldfish_sync_remove(struct platform_device *pdev) +{ + struct goldfish_sync_state *sync_state = global_sync_state; + + DTRACE(); + + misc_deregister(&goldfish_sync_device); + memset(sync_state, 0, sizeof(struct goldfish_sync_state)); + return 0; +} + +static const struct of_device_id goldfish_sync_of_match[] = { + { .compatible = "google,goldfish-sync", }, + {}, +}; +MODULE_DEVICE_TABLE(of, goldfish_sync_of_match); + +static const struct acpi_device_id goldfish_sync_acpi_match[] = { + { "GFSH0006", 0 }, + { }, +}; + +MODULE_DEVICE_TABLE(acpi, goldfish_sync_acpi_match); + +static struct platform_driver goldfish_sync = { + .probe = goldfish_sync_probe, + .remove = goldfish_sync_remove, + .driver = { + .name = "goldfish_sync", + .of_match_table = goldfish_sync_of_match, + .acpi_match_table = ACPI_PTR(goldfish_sync_acpi_match), + } +}; + +module_platform_driver(goldfish_sync); + +MODULE_AUTHOR("Google, Inc."); +MODULE_DESCRIPTION("Android QEMU Sync Driver"); +MODULE_LICENSE("GPL"); +MODULE_VERSION("1.0"); diff --git a/drivers/staging/goldfish/goldfish_sync_timeline_fence.c b/drivers/staging/goldfish/goldfish_sync_timeline_fence.c new file mode 100644 index 00000000000000..e671618cf888d6 --- /dev/null +++ b/drivers/staging/goldfish/goldfish_sync_timeline_fence.c @@ -0,0 +1,254 @@ +#include +#include +#include +#include +#include + +#include "goldfish_sync_timeline_fence.h" + +/* + * Timeline-based sync for Goldfish Sync + * Based on "Sync File validation framework" + * (drivers/dma-buf/sw_sync.c) + * + * Copyright (C) 2017 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +/** + * struct goldfish_sync_timeline - sync object + * @kref: reference count on fence. + * @name: name of the goldfish_sync_timeline. Useful for debugging + * @child_list_head: list of children sync_pts for this goldfish_sync_timeline + * @child_list_lock: lock protecting @child_list_head and fence.status + * @active_list_head: list of active (unsignaled/errored) sync_pts + */ +struct goldfish_sync_timeline { + struct kref kref; + char name[32]; + + /* protected by child_list_lock */ + u64 context; + int value; + + struct list_head child_list_head; + spinlock_t child_list_lock; + + struct list_head active_list_head; +}; + +static inline struct goldfish_sync_timeline *fence_parent(struct fence *fence) +{ + return container_of(fence->lock, struct goldfish_sync_timeline, + child_list_lock); +} + +static const struct fence_ops goldfish_sync_timeline_fence_ops; + +static inline struct sync_pt *goldfish_sync_fence_to_sync_pt(struct fence *fence) +{ + if (fence->ops != &goldfish_sync_timeline_fence_ops) + return NULL; + return container_of(fence, struct sync_pt, base); +} + +/** + * goldfish_sync_timeline_create_internal() - creates a sync object + * @name: sync_timeline name + * + * Creates a new sync_timeline. Returns the sync_timeline object or NULL in + * case of error. + */ +struct goldfish_sync_timeline +*goldfish_sync_timeline_create_internal(const char *name) +{ + struct goldfish_sync_timeline *obj; + + obj = kzalloc(sizeof(*obj), GFP_KERNEL); + if (!obj) + return NULL; + + kref_init(&obj->kref); + obj->context = fence_context_alloc(1); + strlcpy(obj->name, name, sizeof(obj->name)); + + INIT_LIST_HEAD(&obj->child_list_head); + INIT_LIST_HEAD(&obj->active_list_head); + spin_lock_init(&obj->child_list_lock); + + return obj; +} + +static void goldfish_sync_timeline_free_internal(struct kref *kref) +{ + struct goldfish_sync_timeline *obj = + container_of(kref, struct goldfish_sync_timeline, kref); + + kfree(obj); +} + +static void goldfish_sync_timeline_get_internal( + struct goldfish_sync_timeline *obj) +{ + kref_get(&obj->kref); +} + +void goldfish_sync_timeline_put_internal(struct goldfish_sync_timeline *obj) +{ + kref_put(&obj->kref, goldfish_sync_timeline_free_internal); +} + +/** + * goldfish_sync_timeline_signal() - + * signal a status change on a goldfish_sync_timeline + * @obj: sync_timeline to signal + * @inc: num to increment on timeline->value + * + * A sync implementation should call this any time one of it's fences + * has signaled or has an error condition. + */ +void goldfish_sync_timeline_signal_internal(struct goldfish_sync_timeline *obj, + unsigned int inc) +{ + unsigned long flags; + struct sync_pt *pt, *next; + + spin_lock_irqsave(&obj->child_list_lock, flags); + + obj->value += inc; + + list_for_each_entry_safe(pt, next, &obj->active_list_head, + active_list) { + if (fence_is_signaled_locked(&pt->base)) + list_del_init(&pt->active_list); + } + + spin_unlock_irqrestore(&obj->child_list_lock, flags); +} + +/** + * goldfish_sync_pt_create_internal() - creates a sync pt + * @parent: fence's parent sync_timeline + * @size: size to allocate for this pt + * @inc: value of the fence + * + * Creates a new sync_pt as a child of @parent. @size bytes will be + * allocated allowing for implementation specific data to be kept after + * the generic sync_timeline struct. Returns the sync_pt object or + * NULL in case of error. + */ +struct sync_pt *goldfish_sync_pt_create_internal( + struct goldfish_sync_timeline *obj, int size, + unsigned int value) +{ + unsigned long flags; + struct sync_pt *pt; + + if (size < sizeof(*pt)) + return NULL; + + pt = kzalloc(size, GFP_KERNEL); + if (!pt) + return NULL; + + spin_lock_irqsave(&obj->child_list_lock, flags); + goldfish_sync_timeline_get_internal(obj); + fence_init(&pt->base, &goldfish_sync_timeline_fence_ops, &obj->child_list_lock, + obj->context, value); + list_add_tail(&pt->child_list, &obj->child_list_head); + INIT_LIST_HEAD(&pt->active_list); + spin_unlock_irqrestore(&obj->child_list_lock, flags); + return pt; +} + +static const char *goldfish_sync_timeline_fence_get_driver_name( + struct fence *fence) +{ + return "sw_sync"; +} + +static const char *goldfish_sync_timeline_fence_get_timeline_name( + struct fence *fence) +{ + struct goldfish_sync_timeline *parent = fence_parent(fence); + + return parent->name; +} + +static void goldfish_sync_timeline_fence_release(struct fence *fence) +{ + struct sync_pt *pt = goldfish_sync_fence_to_sync_pt(fence); + struct goldfish_sync_timeline *parent = fence_parent(fence); + unsigned long flags; + + spin_lock_irqsave(fence->lock, flags); + list_del(&pt->child_list); + if (!list_empty(&pt->active_list)) + list_del(&pt->active_list); + spin_unlock_irqrestore(fence->lock, flags); + + goldfish_sync_timeline_put_internal(parent); + fence_free(fence); +} + +static bool goldfish_sync_timeline_fence_signaled(struct fence *fence) +{ + struct goldfish_sync_timeline *parent = fence_parent(fence); + + return (fence->seqno > parent->value) ? false : true; +} + +static bool goldfish_sync_timeline_fence_enable_signaling(struct fence *fence) +{ + struct sync_pt *pt = goldfish_sync_fence_to_sync_pt(fence); + struct goldfish_sync_timeline *parent = fence_parent(fence); + + if (goldfish_sync_timeline_fence_signaled(fence)) + return false; + + list_add_tail(&pt->active_list, &parent->active_list_head); + return true; +} + +static void goldfish_sync_timeline_fence_disable_signaling(struct fence *fence) +{ + struct sync_pt *pt = container_of(fence, struct sync_pt, base); + + list_del_init(&pt->active_list); +} + +static void goldfish_sync_timeline_fence_value_str(struct fence *fence, + char *str, int size) +{ + snprintf(str, size, "%d", fence->seqno); +} + +static void goldfish_sync_timeline_fence_timeline_value_str( + struct fence *fence, + char *str, int size) +{ + struct goldfish_sync_timeline *parent = fence_parent(fence); + + snprintf(str, size, "%d", parent->value); +} + +static const struct fence_ops goldfish_sync_timeline_fence_ops = { + .get_driver_name = goldfish_sync_timeline_fence_get_driver_name, + .get_timeline_name = goldfish_sync_timeline_fence_get_timeline_name, + .enable_signaling = goldfish_sync_timeline_fence_enable_signaling, + .disable_signaling = goldfish_sync_timeline_fence_disable_signaling, + .signaled = goldfish_sync_timeline_fence_signaled, + .wait = fence_default_wait, + .release = goldfish_sync_timeline_fence_release, + .fence_value_str = goldfish_sync_timeline_fence_value_str, + .timeline_value_str = goldfish_sync_timeline_fence_timeline_value_str, +}; diff --git a/drivers/staging/goldfish/goldfish_sync_timeline_fence.h b/drivers/staging/goldfish/goldfish_sync_timeline_fence.h new file mode 100644 index 00000000000000..fc25924652c160 --- /dev/null +++ b/drivers/staging/goldfish/goldfish_sync_timeline_fence.h @@ -0,0 +1,58 @@ +#include +#include + +/** + * struct sync_pt - sync_pt object + * @base: base fence object + * @child_list: sync timeline child's list + * @active_list: sync timeline active child's list + */ +struct sync_pt { + struct fence base; + struct list_head child_list; + struct list_head active_list; +}; + +/** + * goldfish_sync_timeline_create_internal() - creates a sync object + * @name: goldfish_sync_timeline name + * + * Creates a new goldfish_sync_timeline. + * Returns the goldfish_sync_timeline object or NULL in case of error. + */ +struct goldfish_sync_timeline +*goldfish_sync_timeline_create_internal(const char *name); + +/** + * goldfish_sync_pt_create_internal() - creates a sync pt + * @parent: fence's parent goldfish_sync_timeline + * @size: size to allocate for this pt + * @inc: value of the fence + * + * Creates a new sync_pt as a child of @parent. @size bytes will be + * allocated allowing for implementation specific data to be kept after + * the generic sync_timeline struct. Returns the sync_pt object or + * NULL in case of error. + */ +struct sync_pt +*goldfish_sync_pt_create_internal(struct goldfish_sync_timeline *obj, + int size, unsigned int value); + +/** + * goldfish_sync_timeline_signal_internal() - + * signal a status change on a sync_timeline + * @obj: goldfish_sync_timeline to signal + * @inc: num to increment on timeline->value + * + * A sync implementation should call this any time one of it's fences + * has signaled or has an error condition. + */ +void goldfish_sync_timeline_signal_internal(struct goldfish_sync_timeline *obj, + unsigned int inc); + +/** + * goldfish_sync_timeline_put_internal() - dec refcount of a sync_timeline + * and clean up memory if it was the last ref. + * @obj: goldfish_sync_timeline to decref + */ +void goldfish_sync_timeline_put_internal(struct goldfish_sync_timeline *obj); diff --git a/drivers/staging/greybus/timesync_platform.c b/drivers/staging/greybus/timesync_platform.c index 113f3d6c4b3a6c..27f75b17679b8f 100644 --- a/drivers/staging/greybus/timesync_platform.c +++ b/drivers/staging/greybus/timesync_platform.c @@ -45,12 +45,18 @@ u32 gb_timesync_platform_get_clock_rate(void) int gb_timesync_platform_lock_bus(struct gb_timesync_svc *pdata) { + if (!arche_platform_change_state_cb) + return 0; + return arche_platform_change_state_cb(ARCHE_PLATFORM_STATE_TIME_SYNC, pdata); } void gb_timesync_platform_unlock_bus(void) { + if (!arche_platform_change_state_cb) + return; + arche_platform_change_state_cb(ARCHE_PLATFORM_STATE_ACTIVE, NULL); } diff --git a/drivers/staging/iio/adc/ad7606_core.c b/drivers/staging/iio/adc/ad7606_core.c index f79ee61851f602..cbd6bc52050f16 100644 --- a/drivers/staging/iio/adc/ad7606_core.c +++ b/drivers/staging/iio/adc/ad7606_core.c @@ -189,7 +189,7 @@ static ssize_t ad7606_store_oversampling_ratio(struct device *dev, mutex_lock(&indio_dev->mlock); gpio_set_value(st->pdata->gpio_os0, (ret >> 0) & 1); gpio_set_value(st->pdata->gpio_os1, (ret >> 1) & 1); - gpio_set_value(st->pdata->gpio_os1, (ret >> 2) & 1); + gpio_set_value(st->pdata->gpio_os2, (ret >> 2) & 1); st->oversampling = lval; mutex_unlock(&indio_dev->mlock); diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_pool.c b/drivers/staging/lustre/lustre/ldlm/ldlm_pool.c index 9a1136e32dfc5d..34efb499c00b29 100644 --- a/drivers/staging/lustre/lustre/ldlm/ldlm_pool.c +++ b/drivers/staging/lustre/lustre/ldlm/ldlm_pool.c @@ -356,10 +356,10 @@ static int ldlm_pool_recalc(struct ldlm_pool *pl) u32 recalc_interval_sec; int count; - recalc_interval_sec = ktime_get_seconds() - pl->pl_recalc_time; + recalc_interval_sec = ktime_get_real_seconds() - pl->pl_recalc_time; if (recalc_interval_sec > 0) { spin_lock(&pl->pl_lock); - recalc_interval_sec = ktime_get_seconds() - pl->pl_recalc_time; + recalc_interval_sec = ktime_get_real_seconds() - pl->pl_recalc_time; if (recalc_interval_sec > 0) { /* @@ -382,7 +382,7 @@ static int ldlm_pool_recalc(struct ldlm_pool *pl) count); } - recalc_interval_sec = pl->pl_recalc_time - ktime_get_seconds() + + recalc_interval_sec = pl->pl_recalc_time - ktime_get_real_seconds() + pl->pl_recalc_period; if (recalc_interval_sec <= 0) { /* DEBUG: should be re-removed after LU-4536 is fixed */ @@ -657,7 +657,7 @@ int ldlm_pool_init(struct ldlm_pool *pl, struct ldlm_namespace *ns, spin_lock_init(&pl->pl_lock); atomic_set(&pl->pl_granted, 0); - pl->pl_recalc_time = ktime_get_seconds(); + pl->pl_recalc_time = ktime_get_real_seconds(); atomic_set(&pl->pl_lock_volume_factor, 1); atomic_set(&pl->pl_grant_rate, 0); diff --git a/drivers/staging/lustre/lustre/osc/osc_page.c b/drivers/staging/lustre/lustre/osc/osc_page.c index 2a7a70aa9e802d..9168451d2f292c 100644 --- a/drivers/staging/lustre/lustre/osc/osc_page.c +++ b/drivers/staging/lustre/lustre/osc/osc_page.c @@ -542,7 +542,6 @@ long osc_lru_shrink(const struct lu_env *env, struct client_obd *cli, struct cl_object *clobj = NULL; struct cl_page **pvec; struct osc_page *opg; - struct osc_page *temp; int maxscan = 0; long count = 0; int index = 0; @@ -569,13 +568,15 @@ long osc_lru_shrink(const struct lu_env *env, struct client_obd *cli, spin_lock(&cli->cl_lru_list_lock); maxscan = min(target << 1, atomic_long_read(&cli->cl_lru_in_list)); - list_for_each_entry_safe(opg, temp, &cli->cl_lru_list, ops_lru) { + while (!list_empty(&cli->cl_lru_list)) { struct cl_page *page; bool will_free = false; if (--maxscan < 0) break; + opg = list_entry(cli->cl_lru_list.next, struct osc_page, + ops_lru); page = opg->ops_cl.cpl_page; if (lru_page_busy(cli, page)) { list_move_tail(&opg->ops_lru, &cli->cl_lru_list); diff --git a/drivers/staging/media/davinci_vpfe/vpfe_video.c b/drivers/staging/media/davinci_vpfe/vpfe_video.c index 8be9f854510f4d..89dd6b989254fa 100644 --- a/drivers/staging/media/davinci_vpfe/vpfe_video.c +++ b/drivers/staging/media/davinci_vpfe/vpfe_video.c @@ -1362,7 +1362,7 @@ static int vpfe_reqbufs(struct file *file, void *priv, ret = vb2_queue_init(q); if (ret) { v4l2_err(&vpfe_dev->v4l2_dev, "vb2_queue_init() failed\n"); - return ret; + goto unlock_out; } fh->io_allowed = 1; diff --git a/drivers/staging/media/s5p-cec/s5p_cec.c b/drivers/staging/media/s5p-cec/s5p_cec.c index 1780a08b73c961..58d7562311360b 100644 --- a/drivers/staging/media/s5p-cec/s5p_cec.c +++ b/drivers/staging/media/s5p-cec/s5p_cec.c @@ -231,7 +231,7 @@ static int s5p_cec_remove(struct platform_device *pdev) return 0; } -static int s5p_cec_runtime_suspend(struct device *dev) +static int __maybe_unused s5p_cec_runtime_suspend(struct device *dev) { struct s5p_cec_dev *cec = dev_get_drvdata(dev); @@ -239,7 +239,7 @@ static int s5p_cec_runtime_suspend(struct device *dev) return 0; } -static int s5p_cec_runtime_resume(struct device *dev) +static int __maybe_unused s5p_cec_runtime_resume(struct device *dev) { struct s5p_cec_dev *cec = dev_get_drvdata(dev); int ret; diff --git a/drivers/staging/octeon/ethernet.c b/drivers/staging/octeon/ethernet.c index d02e3e31ed293d..12354440a334a3 100644 --- a/drivers/staging/octeon/ethernet.c +++ b/drivers/staging/octeon/ethernet.c @@ -776,6 +776,7 @@ static int cvm_oct_probe(struct platform_device *pdev) /* Initialize the device private structure. */ struct octeon_ethernet *priv = netdev_priv(dev); + SET_NETDEV_DEV(dev, &pdev->dev); dev->netdev_ops = &cvm_oct_pow_netdev_ops; priv->imode = CVMX_HELPER_INTERFACE_MODE_DISABLED; priv->port = CVMX_PIP_NUM_INPUT_PORTS; @@ -820,6 +821,7 @@ static int cvm_oct_probe(struct platform_device *pdev) } /* Initialize the device private structure. */ + SET_NETDEV_DEV(dev, &pdev->dev); priv = netdev_priv(dev); priv->netdev = dev; priv->of_node = cvm_oct_node_for_port(pip, interface, diff --git a/drivers/target/iscsi/iscsi_target_configfs.c b/drivers/target/iscsi/iscsi_target_configfs.c index 923c032f0b95f1..e980e2d0c2db55 100644 --- a/drivers/target/iscsi/iscsi_target_configfs.c +++ b/drivers/target/iscsi/iscsi_target_configfs.c @@ -100,8 +100,10 @@ static ssize_t lio_target_np_driver_store(struct config_item *item, tpg_np_new = iscsit_tpg_add_network_portal(tpg, &np->np_sockaddr, tpg_np, type); - if (IS_ERR(tpg_np_new)) + if (IS_ERR(tpg_np_new)) { + rc = PTR_ERR(tpg_np_new); goto out; + } } else { tpg_np_new = iscsit_tpg_locate_child_np(tpg_np, type); if (tpg_np_new) { diff --git a/drivers/target/iscsi/iscsi_target_tpg.c b/drivers/target/iscsi/iscsi_target_tpg.c index 0814e5894a9616..205a509b0dfb28 100644 --- a/drivers/target/iscsi/iscsi_target_tpg.c +++ b/drivers/target/iscsi/iscsi_target_tpg.c @@ -260,7 +260,6 @@ int iscsit_tpg_add_portal_group(struct iscsi_tiqn *tiqn, struct iscsi_portal_gro iscsi_release_param_list(tpg->param_list); tpg->param_list = NULL; } - kfree(tpg); return -ENOMEM; } diff --git a/drivers/target/sbp/sbp_target.c b/drivers/target/sbp/sbp_target.c index 58bb6ed181853b..6ca388eca33bda 100644 --- a/drivers/target/sbp/sbp_target.c +++ b/drivers/target/sbp/sbp_target.c @@ -928,7 +928,7 @@ static struct sbp_target_request *sbp_mgt_get_req(struct sbp_session *sess, struct sbp_target_request *req; int tag; - tag = percpu_ida_alloc(&se_sess->sess_tag_pool, GFP_ATOMIC); + tag = percpu_ida_alloc(&se_sess->sess_tag_pool, TASK_RUNNING); if (tag < 0) return ERR_PTR(-ENOMEM); diff --git a/drivers/target/target_core_device.c b/drivers/target/target_core_device.c index 6b423485c5d6b4..ea9617c7b403b6 100644 --- a/drivers/target/target_core_device.c +++ b/drivers/target/target_core_device.c @@ -351,7 +351,15 @@ int core_enable_device_list_for_node( kfree(new); return -EINVAL; } - BUG_ON(orig->se_lun_acl != NULL); + if (orig->se_lun_acl != NULL) { + pr_warn_ratelimited("Detected existing explicit" + " se_lun_acl->se_lun_group reference for %s" + " mapped_lun: %llu, failing\n", + nacl->initiatorname, mapped_lun); + mutex_unlock(&nacl->lun_entry_mutex); + kfree(new); + return -EINVAL; + } rcu_assign_pointer(new->se_lun, lun); rcu_assign_pointer(new->se_lun_acl, lun_acl); diff --git a/drivers/target/target_core_sbc.c b/drivers/target/target_core_sbc.c index 04f616b3ba0a84..aabd6602da6ccc 100644 --- a/drivers/target/target_core_sbc.c +++ b/drivers/target/target_core_sbc.c @@ -450,6 +450,7 @@ static sense_reason_t compare_and_write_post(struct se_cmd *cmd, bool success, int *post_ret) { struct se_device *dev = cmd->se_dev; + sense_reason_t ret = TCM_NO_SENSE; /* * Only set SCF_COMPARE_AND_WRITE_POST to force a response fall-through @@ -457,9 +458,12 @@ static sense_reason_t compare_and_write_post(struct se_cmd *cmd, bool success, * sent to the backend driver. */ spin_lock_irq(&cmd->t_state_lock); - if ((cmd->transport_state & CMD_T_SENT) && !cmd->scsi_status) { + if (cmd->transport_state & CMD_T_SENT) { cmd->se_cmd_flags |= SCF_COMPARE_AND_WRITE_POST; *post_ret = 1; + + if (cmd->scsi_status == SAM_STAT_CHECK_CONDITION) + ret = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; } spin_unlock_irq(&cmd->t_state_lock); @@ -469,7 +473,7 @@ static sense_reason_t compare_and_write_post(struct se_cmd *cmd, bool success, */ up(&dev->caw_sem); - return TCM_NO_SENSE; + return ret; } static sense_reason_t compare_and_write_callback(struct se_cmd *cmd, bool success, diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c index 7dfefd66df9387..767d1eb6e0356b 100644 --- a/drivers/target/target_core_transport.c +++ b/drivers/target/target_core_transport.c @@ -457,8 +457,20 @@ static void target_complete_nacl(struct kref *kref) { struct se_node_acl *nacl = container_of(kref, struct se_node_acl, acl_kref); + struct se_portal_group *se_tpg = nacl->se_tpg; - complete(&nacl->acl_free_comp); + if (!nacl->dynamic_stop) { + complete(&nacl->acl_free_comp); + return; + } + + mutex_lock(&se_tpg->acl_node_mutex); + list_del(&nacl->acl_list); + mutex_unlock(&se_tpg->acl_node_mutex); + + core_tpg_wait_for_nacl_pr_ref(nacl); + core_free_device_list_for_node(nacl, se_tpg); + kfree(nacl); } void target_put_nacl(struct se_node_acl *nacl) @@ -499,12 +511,39 @@ EXPORT_SYMBOL(transport_deregister_session_configfs); void transport_free_session(struct se_session *se_sess) { struct se_node_acl *se_nacl = se_sess->se_node_acl; + /* * Drop the se_node_acl->nacl_kref obtained from within * core_tpg_get_initiator_node_acl(). */ if (se_nacl) { + struct se_portal_group *se_tpg = se_nacl->se_tpg; + const struct target_core_fabric_ops *se_tfo = se_tpg->se_tpg_tfo; + unsigned long flags; + se_sess->se_node_acl = NULL; + + /* + * Also determine if we need to drop the extra ->cmd_kref if + * it had been previously dynamically generated, and + * the endpoint is not caching dynamic ACLs. + */ + mutex_lock(&se_tpg->acl_node_mutex); + if (se_nacl->dynamic_node_acl && + !se_tfo->tpg_check_demo_mode_cache(se_tpg)) { + spin_lock_irqsave(&se_nacl->nacl_sess_lock, flags); + if (list_empty(&se_nacl->acl_sess_list)) + se_nacl->dynamic_stop = true; + spin_unlock_irqrestore(&se_nacl->nacl_sess_lock, flags); + + if (se_nacl->dynamic_stop) + list_del(&se_nacl->acl_list); + } + mutex_unlock(&se_tpg->acl_node_mutex); + + if (se_nacl->dynamic_stop) + target_put_nacl(se_nacl); + target_put_nacl(se_nacl); } if (se_sess->sess_cmd_map) { @@ -518,16 +557,12 @@ EXPORT_SYMBOL(transport_free_session); void transport_deregister_session(struct se_session *se_sess) { struct se_portal_group *se_tpg = se_sess->se_tpg; - const struct target_core_fabric_ops *se_tfo; - struct se_node_acl *se_nacl; unsigned long flags; - bool drop_nacl = false; if (!se_tpg) { transport_free_session(se_sess); return; } - se_tfo = se_tpg->se_tpg_tfo; spin_lock_irqsave(&se_tpg->session_lock, flags); list_del(&se_sess->sess_list); @@ -535,33 +570,15 @@ void transport_deregister_session(struct se_session *se_sess) se_sess->fabric_sess_ptr = NULL; spin_unlock_irqrestore(&se_tpg->session_lock, flags); - /* - * Determine if we need to do extra work for this initiator node's - * struct se_node_acl if it had been previously dynamically generated. - */ - se_nacl = se_sess->se_node_acl; - - mutex_lock(&se_tpg->acl_node_mutex); - if (se_nacl && se_nacl->dynamic_node_acl) { - if (!se_tfo->tpg_check_demo_mode_cache(se_tpg)) { - list_del(&se_nacl->acl_list); - drop_nacl = true; - } - } - mutex_unlock(&se_tpg->acl_node_mutex); - - if (drop_nacl) { - core_tpg_wait_for_nacl_pr_ref(se_nacl); - core_free_device_list_for_node(se_nacl, se_tpg); - se_sess->se_node_acl = NULL; - kfree(se_nacl); - } pr_debug("TARGET_CORE[%s]: Deregistered fabric_sess\n", se_tpg->se_tpg_tfo->get_fabric_name()); /* * If last kref is dropping now for an explicit NodeACL, awake sleeping * ->acl_free_comp caller to wakeup configfs se_node_acl->acl_group * removal context from within transport_free_session() code. + * + * For dynamic ACL, target_put_nacl() uses target_complete_nacl() + * to release all remaining generate_node_acl=1 created ACL resources. */ transport_free_session(se_sess); @@ -3086,7 +3103,6 @@ static void target_tmr_work(struct work_struct *work) spin_unlock_irqrestore(&cmd->t_state_lock, flags); goto check_stop; } - cmd->t_state = TRANSPORT_ISTATE_PROCESSING; spin_unlock_irqrestore(&cmd->t_state_lock, flags); cmd->se_tfo->queue_tm_rsp(cmd); @@ -3099,11 +3115,25 @@ int transport_generic_handle_tmr( struct se_cmd *cmd) { unsigned long flags; + bool aborted = false; spin_lock_irqsave(&cmd->t_state_lock, flags); - cmd->transport_state |= CMD_T_ACTIVE; + if (cmd->transport_state & CMD_T_ABORTED) { + aborted = true; + } else { + cmd->t_state = TRANSPORT_ISTATE_PROCESSING; + cmd->transport_state |= CMD_T_ACTIVE; + } spin_unlock_irqrestore(&cmd->t_state_lock, flags); + if (aborted) { + pr_warn_ratelimited("handle_tmr caught CMD_T_ABORTED TMR %d" + "ref_tag: %llu tag: %llu\n", cmd->se_tmr_req->function, + cmd->se_tmr_req->ref_task_tag, cmd->tag); + transport_cmd_check_stop_to_fabric(cmd); + return 0; + } + INIT_WORK(&cmd->work, target_tmr_work); queue_work(cmd->se_dev->tmr_wq, &cmd->work); return 0; diff --git a/drivers/target/target_core_user.c b/drivers/target/target_core_user.c index 47562509b4892b..70c143a5c38c3f 100644 --- a/drivers/target/target_core_user.c +++ b/drivers/target/target_core_user.c @@ -685,8 +685,6 @@ static int tcmu_check_expired_cmd(int id, void *p, void *data) target_complete_cmd(cmd->se_cmd, SAM_STAT_CHECK_CONDITION); cmd->se_cmd = NULL; - kmem_cache_free(tcmu_cmd_cache, cmd); - return 0; } diff --git a/drivers/target/target_core_xcopy.c b/drivers/target/target_core_xcopy.c index 094a1440eacb3d..18848ba8d2ba0a 100644 --- a/drivers/target/target_core_xcopy.c +++ b/drivers/target/target_core_xcopy.c @@ -836,7 +836,7 @@ static void target_xcopy_do_work(struct work_struct *work) " CHECK_CONDITION -> sending response\n", rc); ec_cmd->scsi_status = SAM_STAT_CHECK_CONDITION; } - target_complete_cmd(ec_cmd, SAM_STAT_CHECK_CONDITION); + target_complete_cmd(ec_cmd, ec_cmd->scsi_status); } sense_reason_t target_do_xcopy(struct se_cmd *se_cmd) diff --git a/drivers/tee/Kconfig b/drivers/tee/Kconfig new file mode 100644 index 00000000000000..2330a4eb4e8b71 --- /dev/null +++ b/drivers/tee/Kconfig @@ -0,0 +1,18 @@ +# Generic Trusted Execution Environment Configuration +config TEE + tristate "Trusted Execution Environment support" + select DMA_SHARED_BUFFER + select GENERIC_ALLOCATOR + help + This implements a generic interface towards a Trusted Execution + Environment (TEE). + +if TEE + +menu "TEE drivers" + +source "drivers/tee/optee/Kconfig" + +endmenu + +endif diff --git a/drivers/tee/Makefile b/drivers/tee/Makefile new file mode 100644 index 00000000000000..7a4e4a1ac39c0f --- /dev/null +++ b/drivers/tee/Makefile @@ -0,0 +1,5 @@ +obj-$(CONFIG_TEE) += tee.o +tee-objs += tee_core.o +tee-objs += tee_shm.o +tee-objs += tee_shm_pool.o +obj-$(CONFIG_OPTEE) += optee/ diff --git a/drivers/tee/optee/Kconfig b/drivers/tee/optee/Kconfig new file mode 100644 index 00000000000000..0126de898036cd --- /dev/null +++ b/drivers/tee/optee/Kconfig @@ -0,0 +1,7 @@ +# OP-TEE Trusted Execution Environment Configuration +config OPTEE + tristate "OP-TEE" + depends on HAVE_ARM_SMCCC + help + This implements the OP-TEE Trusted Execution Environment (TEE) + driver. diff --git a/drivers/tee/optee/Makefile b/drivers/tee/optee/Makefile new file mode 100644 index 00000000000000..92fe5789bcce29 --- /dev/null +++ b/drivers/tee/optee/Makefile @@ -0,0 +1,5 @@ +obj-$(CONFIG_OPTEE) += optee.o +optee-objs += core.o +optee-objs += call.o +optee-objs += rpc.o +optee-objs += supp.o diff --git a/drivers/tee/optee/call.c b/drivers/tee/optee/call.c new file mode 100644 index 00000000000000..97fad0e2f90a07 --- /dev/null +++ b/drivers/tee/optee/call.c @@ -0,0 +1,435 @@ +/* + * Copyright (c) 2015, Linaro Limited + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include "optee_private.h" +#include "optee_smc.h" + +struct optee_call_waiter { + struct list_head list_node; + struct completion c; + bool completed; +}; + +static void optee_cq_wait_init(struct optee_call_queue *cq, + struct optee_call_waiter *w) +{ + mutex_lock(&cq->mutex); + + /* + * We add ourselves to the queue, but we don't wait. This + * guarantees that we don't lose a completion if secure world + * returns busy and another thread just exited and try to complete + * someone. + */ + w->completed = false; + init_completion(&w->c); + list_add_tail(&w->list_node, &cq->waiters); + + mutex_unlock(&cq->mutex); +} + +static void optee_cq_wait_for_completion(struct optee_call_queue *cq, + struct optee_call_waiter *w) +{ + wait_for_completion(&w->c); + + mutex_lock(&cq->mutex); + + /* Move to end of list to get out of the way for other waiters */ + list_del(&w->list_node); + w->completed = false; + reinit_completion(&w->c); + list_add_tail(&w->list_node, &cq->waiters); + + mutex_unlock(&cq->mutex); +} + +static void optee_cq_complete_one(struct optee_call_queue *cq) +{ + struct optee_call_waiter *w; + + list_for_each_entry(w, &cq->waiters, list_node) { + if (!w->completed) { + complete(&w->c); + w->completed = true; + break; + } + } +} + +static void optee_cq_wait_final(struct optee_call_queue *cq, + struct optee_call_waiter *w) +{ + mutex_lock(&cq->mutex); + + /* Get out of the list */ + list_del(&w->list_node); + + optee_cq_complete_one(cq); + /* + * If we're completed we've got a completion that some other task + * could have used instead. + */ + if (w->completed) + optee_cq_complete_one(cq); + + mutex_unlock(&cq->mutex); +} + +/* Requires the filpstate mutex to be held */ +static struct optee_session *find_session(struct optee_context_data *ctxdata, + u32 session_id) +{ + struct optee_session *sess; + + list_for_each_entry(sess, &ctxdata->sess_list, list_node) + if (sess->session_id == session_id) + return sess; + + return NULL; +} + +/** + * optee_do_call_with_arg() - Do an SMC to OP-TEE in secure world + * @ctx: calling context + * @parg: physical address of message to pass to secure world + * + * Does and SMC to OP-TEE in secure world and handles eventual resulting + * Remote Procedure Calls (RPC) from OP-TEE. + * + * Returns return code from secure world, 0 is OK + */ +u32 optee_do_call_with_arg(struct tee_context *ctx, phys_addr_t parg) +{ + struct optee *optee = tee_get_drvdata(ctx->teedev); + struct optee_call_waiter w; + struct optee_rpc_param param = { }; + u32 ret; + + param.a0 = OPTEE_SMC_CALL_WITH_ARG; + reg_pair_from_64(¶m.a1, ¶m.a2, parg); + /* Initialize waiter */ + optee_cq_wait_init(&optee->call_queue, &w); + while (true) { + struct arm_smccc_res res; + + optee->invoke_fn(param.a0, param.a1, param.a2, param.a3, + param.a4, param.a5, param.a6, param.a7, + &res); + + if (res.a0 == OPTEE_SMC_RETURN_ETHREAD_LIMIT) { + /* + * Out of threads in secure world, wait for a thread + * become available. + */ + optee_cq_wait_for_completion(&optee->call_queue, &w); + } else if (OPTEE_SMC_RETURN_IS_RPC(res.a0)) { + param.a0 = res.a0; + param.a1 = res.a1; + param.a2 = res.a2; + param.a3 = res.a3; + optee_handle_rpc(ctx, ¶m); + } else { + ret = res.a0; + break; + } + } + + /* + * We're done with our thread in secure world, if there's any + * thread waiters wake up one. + */ + optee_cq_wait_final(&optee->call_queue, &w); + + return ret; +} + +static struct tee_shm *get_msg_arg(struct tee_context *ctx, size_t num_params, + struct optee_msg_arg **msg_arg, + phys_addr_t *msg_parg) +{ + int rc; + struct tee_shm *shm; + struct optee_msg_arg *ma; + + shm = tee_shm_alloc(ctx, OPTEE_MSG_GET_ARG_SIZE(num_params), + TEE_SHM_MAPPED); + if (IS_ERR(shm)) + return shm; + + ma = tee_shm_get_va(shm, 0); + if (IS_ERR(ma)) { + rc = PTR_ERR(ma); + goto out; + } + + rc = tee_shm_get_pa(shm, 0, msg_parg); + if (rc) + goto out; + + memset(ma, 0, OPTEE_MSG_GET_ARG_SIZE(num_params)); + ma->num_params = num_params; + *msg_arg = ma; +out: + if (rc) { + tee_shm_free(shm); + return ERR_PTR(rc); + } + + return shm; +} + +int optee_open_session(struct tee_context *ctx, + struct tee_ioctl_open_session_arg *arg, + struct tee_param *param) +{ + struct optee_context_data *ctxdata = ctx->data; + int rc; + struct tee_shm *shm; + struct optee_msg_arg *msg_arg; + phys_addr_t msg_parg; + struct optee_msg_param *msg_param; + struct optee_session *sess = NULL; + + /* +2 for the meta parameters added below */ + shm = get_msg_arg(ctx, arg->num_params + 2, &msg_arg, &msg_parg); + if (IS_ERR(shm)) + return PTR_ERR(shm); + + msg_arg->cmd = OPTEE_MSG_CMD_OPEN_SESSION; + msg_arg->cancel_id = arg->cancel_id; + msg_param = OPTEE_MSG_GET_PARAMS(msg_arg); + + /* + * Initialize and add the meta parameters needed when opening a + * session. + */ + msg_param[0].attr = OPTEE_MSG_ATTR_TYPE_VALUE_INPUT | + OPTEE_MSG_ATTR_META; + msg_param[1].attr = OPTEE_MSG_ATTR_TYPE_VALUE_INPUT | + OPTEE_MSG_ATTR_META; + memcpy(&msg_param[0].u.value, arg->uuid, sizeof(arg->uuid)); + memcpy(&msg_param[1].u.value, arg->uuid, sizeof(arg->clnt_uuid)); + msg_param[1].u.value.c = arg->clnt_login; + + rc = optee_to_msg_param(msg_param + 2, arg->num_params, param); + if (rc) + goto out; + + sess = kzalloc(sizeof(*sess), GFP_KERNEL); + if (!sess) { + rc = -ENOMEM; + goto out; + } + + if (optee_do_call_with_arg(ctx, msg_parg)) { + msg_arg->ret = TEEC_ERROR_COMMUNICATION; + msg_arg->ret_origin = TEEC_ORIGIN_COMMS; + } + + if (msg_arg->ret == TEEC_SUCCESS) { + /* A new session has been created, add it to the list. */ + sess->session_id = msg_arg->session; + mutex_lock(&ctxdata->mutex); + list_add(&sess->list_node, &ctxdata->sess_list); + mutex_unlock(&ctxdata->mutex); + } else { + kfree(sess); + } + + if (optee_from_msg_param(param, arg->num_params, msg_param + 2)) { + arg->ret = TEEC_ERROR_COMMUNICATION; + arg->ret_origin = TEEC_ORIGIN_COMMS; + /* Close session again to avoid leakage */ + optee_close_session(ctx, msg_arg->session); + } else { + arg->session = msg_arg->session; + arg->ret = msg_arg->ret; + arg->ret_origin = msg_arg->ret_origin; + } +out: + tee_shm_free(shm); + + return rc; +} + +int optee_close_session(struct tee_context *ctx, u32 session) +{ + struct optee_context_data *ctxdata = ctx->data; + struct tee_shm *shm; + struct optee_msg_arg *msg_arg; + phys_addr_t msg_parg; + struct optee_session *sess; + + /* Check that the session is valid and remove it from the list */ + mutex_lock(&ctxdata->mutex); + sess = find_session(ctxdata, session); + if (sess) + list_del(&sess->list_node); + mutex_unlock(&ctxdata->mutex); + if (!sess) + return -EINVAL; + kfree(sess); + + shm = get_msg_arg(ctx, 0, &msg_arg, &msg_parg); + if (IS_ERR(shm)) + return PTR_ERR(shm); + + msg_arg->cmd = OPTEE_MSG_CMD_CLOSE_SESSION; + msg_arg->session = session; + optee_do_call_with_arg(ctx, msg_parg); + + tee_shm_free(shm); + return 0; +} + +int optee_invoke_func(struct tee_context *ctx, struct tee_ioctl_invoke_arg *arg, + struct tee_param *param) +{ + struct optee_context_data *ctxdata = ctx->data; + struct tee_shm *shm; + struct optee_msg_arg *msg_arg; + phys_addr_t msg_parg; + struct optee_msg_param *msg_param; + struct optee_session *sess; + int rc; + + /* Check that the session is valid */ + mutex_lock(&ctxdata->mutex); + sess = find_session(ctxdata, arg->session); + mutex_unlock(&ctxdata->mutex); + if (!sess) + return -EINVAL; + + shm = get_msg_arg(ctx, arg->num_params, &msg_arg, &msg_parg); + if (IS_ERR(shm)) + return PTR_ERR(shm); + msg_arg->cmd = OPTEE_MSG_CMD_INVOKE_COMMAND; + msg_arg->func = arg->func; + msg_arg->session = arg->session; + msg_arg->cancel_id = arg->cancel_id; + msg_param = OPTEE_MSG_GET_PARAMS(msg_arg); + + rc = optee_to_msg_param(msg_param, arg->num_params, param); + if (rc) + goto out; + + if (optee_do_call_with_arg(ctx, msg_parg)) { + msg_arg->ret = TEEC_ERROR_COMMUNICATION; + msg_arg->ret_origin = TEEC_ORIGIN_COMMS; + } + + if (optee_from_msg_param(param, arg->num_params, msg_param)) { + msg_arg->ret = TEEC_ERROR_COMMUNICATION; + msg_arg->ret_origin = TEEC_ORIGIN_COMMS; + } + + arg->ret = msg_arg->ret; + arg->ret_origin = msg_arg->ret_origin; +out: + tee_shm_free(shm); + return rc; +} + +int optee_cancel_req(struct tee_context *ctx, u32 cancel_id, u32 session) +{ + struct optee_context_data *ctxdata = ctx->data; + struct tee_shm *shm; + struct optee_msg_arg *msg_arg; + phys_addr_t msg_parg; + struct optee_session *sess; + + /* Check that the session is valid */ + mutex_lock(&ctxdata->mutex); + sess = find_session(ctxdata, session); + mutex_unlock(&ctxdata->mutex); + if (!sess) + return -EINVAL; + + shm = get_msg_arg(ctx, 0, &msg_arg, &msg_parg); + if (IS_ERR(shm)) + return PTR_ERR(shm); + + msg_arg->cmd = OPTEE_MSG_CMD_CANCEL; + msg_arg->session = session; + msg_arg->cancel_id = cancel_id; + optee_do_call_with_arg(ctx, msg_parg); + + tee_shm_free(shm); + return 0; +} + +/** + * optee_enable_shm_cache() - Enables caching of some shared memory allocation + * in OP-TEE + * @optee: main service struct + */ +void optee_enable_shm_cache(struct optee *optee) +{ + struct optee_call_waiter w; + + /* We need to retry until secure world isn't busy. */ + optee_cq_wait_init(&optee->call_queue, &w); + while (true) { + struct arm_smccc_res res; + + optee->invoke_fn(OPTEE_SMC_ENABLE_SHM_CACHE, 0, 0, 0, 0, 0, 0, + 0, &res); + if (res.a0 == OPTEE_SMC_RETURN_OK) + break; + optee_cq_wait_for_completion(&optee->call_queue, &w); + } + optee_cq_wait_final(&optee->call_queue, &w); +} + +/** + * optee_enable_shm_cache() - Disables caching of some shared memory allocation + * in OP-TEE + * @optee: main service struct + */ +void optee_disable_shm_cache(struct optee *optee) +{ + struct optee_call_waiter w; + + /* We need to retry until secure world isn't busy. */ + optee_cq_wait_init(&optee->call_queue, &w); + while (true) { + union { + struct arm_smccc_res smccc; + struct optee_smc_disable_shm_cache_result result; + } res; + + optee->invoke_fn(OPTEE_SMC_DISABLE_SHM_CACHE, 0, 0, 0, 0, 0, 0, + 0, &res.smccc); + if (res.result.status == OPTEE_SMC_RETURN_ENOTAVAIL) + break; /* All shm's freed */ + if (res.result.status == OPTEE_SMC_RETURN_OK) { + struct tee_shm *shm; + + shm = reg_pair_to_ptr(res.result.shm_upper32, + res.result.shm_lower32); + tee_shm_free(shm); + } else { + optee_cq_wait_for_completion(&optee->call_queue, &w); + } + } + optee_cq_wait_final(&optee->call_queue, &w); +} diff --git a/drivers/tee/optee/core.c b/drivers/tee/optee/core.c new file mode 100644 index 00000000000000..1a9853c82e73cc --- /dev/null +++ b/drivers/tee/optee/core.c @@ -0,0 +1,595 @@ +/* + * Copyright (c) 2015, Linaro Limited + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "optee_private.h" +#include "optee_smc.h" + +#define DRIVER_NAME "optee" + +#define OPTEE_SHM_NUM_PRIV_PAGES 1 + +/** + * optee_from_msg_param() - convert from OPTEE_MSG parameters to + * struct tee_param + * @params: subsystem internal parameter representation + * @num_params: number of elements in the parameter arrays + * @msg_params: OPTEE_MSG parameters + * Returns 0 on success or <0 on failure + */ +int optee_from_msg_param(struct tee_param *params, size_t num_params, + const struct optee_msg_param *msg_params) +{ + int rc; + size_t n; + struct tee_shm *shm; + phys_addr_t pa; + + for (n = 0; n < num_params; n++) { + struct tee_param *p = params + n; + const struct optee_msg_param *mp = msg_params + n; + u32 attr = mp->attr & OPTEE_MSG_ATTR_TYPE_MASK; + + switch (attr) { + case OPTEE_MSG_ATTR_TYPE_NONE: + p->attr = TEE_IOCTL_PARAM_ATTR_TYPE_NONE; + memset(&p->u, 0, sizeof(p->u)); + break; + case OPTEE_MSG_ATTR_TYPE_VALUE_INPUT: + case OPTEE_MSG_ATTR_TYPE_VALUE_OUTPUT: + case OPTEE_MSG_ATTR_TYPE_VALUE_INOUT: + p->attr = TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT + + attr - OPTEE_MSG_ATTR_TYPE_VALUE_INPUT; + p->u.value.a = mp->u.value.a; + p->u.value.b = mp->u.value.b; + p->u.value.c = mp->u.value.c; + break; + case OPTEE_MSG_ATTR_TYPE_TMEM_INPUT: + case OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT: + case OPTEE_MSG_ATTR_TYPE_TMEM_INOUT: + p->attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT + + attr - OPTEE_MSG_ATTR_TYPE_TMEM_INPUT; + p->u.memref.size = mp->u.tmem.size; + shm = (struct tee_shm *)(unsigned long) + mp->u.tmem.shm_ref; + if (!shm) { + p->u.memref.shm_offs = 0; + p->u.memref.shm = NULL; + break; + } + rc = tee_shm_get_pa(shm, 0, &pa); + if (rc) + return rc; + p->u.memref.shm_offs = mp->u.tmem.buf_ptr - pa; + p->u.memref.shm = shm; + + /* Check that the memref is covered by the shm object */ + if (p->u.memref.size) { + size_t o = p->u.memref.shm_offs + + p->u.memref.size - 1; + + rc = tee_shm_get_pa(shm, o, NULL); + if (rc) + return rc; + } + break; + default: + return -EINVAL; + } + } + return 0; +} + +/** + * optee_to_msg_param() - convert from struct tee_params to OPTEE_MSG parameters + * @msg_params: OPTEE_MSG parameters + * @num_params: number of elements in the parameter arrays + * @params: subsystem itnernal parameter representation + * Returns 0 on success or <0 on failure + */ +int optee_to_msg_param(struct optee_msg_param *msg_params, size_t num_params, + const struct tee_param *params) +{ + int rc; + size_t n; + phys_addr_t pa; + + for (n = 0; n < num_params; n++) { + const struct tee_param *p = params + n; + struct optee_msg_param *mp = msg_params + n; + + switch (p->attr) { + case TEE_IOCTL_PARAM_ATTR_TYPE_NONE: + mp->attr = TEE_IOCTL_PARAM_ATTR_TYPE_NONE; + memset(&mp->u, 0, sizeof(mp->u)); + break; + case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT: + case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_OUTPUT: + case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INOUT: + mp->attr = OPTEE_MSG_ATTR_TYPE_VALUE_INPUT + p->attr - + TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT; + mp->u.value.a = p->u.value.a; + mp->u.value.b = p->u.value.b; + mp->u.value.c = p->u.value.c; + break; + case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT: + case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_OUTPUT: + case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INOUT: + mp->attr = OPTEE_MSG_ATTR_TYPE_TMEM_INPUT + + p->attr - + TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT; + mp->u.tmem.shm_ref = (unsigned long)p->u.memref.shm; + mp->u.tmem.size = p->u.memref.size; + if (!p->u.memref.shm) { + mp->u.tmem.buf_ptr = 0; + break; + } + rc = tee_shm_get_pa(p->u.memref.shm, + p->u.memref.shm_offs, &pa); + if (rc) + return rc; + mp->u.tmem.buf_ptr = pa; + mp->attr |= OPTEE_MSG_ATTR_CACHE_PREDEFINED << + OPTEE_MSG_ATTR_CACHE_SHIFT; + break; + default: + return -EINVAL; + } + } + return 0; +} + +static void optee_get_version(struct tee_device *teedev, + struct tee_ioctl_version_data *vers) +{ + struct tee_ioctl_version_data v = { + .impl_id = TEE_IMPL_ID_OPTEE, + .impl_caps = TEE_OPTEE_CAP_TZ, + .gen_caps = TEE_GEN_CAP_GP, + }; + *vers = v; +} + +static int optee_open(struct tee_context *ctx) +{ + struct optee_context_data *ctxdata; + struct tee_device *teedev = ctx->teedev; + struct optee *optee = tee_get_drvdata(teedev); + + ctxdata = kzalloc(sizeof(*ctxdata), GFP_KERNEL); + if (!ctxdata) + return -ENOMEM; + + if (teedev == optee->supp_teedev) { + bool busy = true; + + mutex_lock(&optee->supp.mutex); + if (!optee->supp.ctx) { + busy = false; + optee->supp.ctx = ctx; + } + mutex_unlock(&optee->supp.mutex); + if (busy) { + kfree(ctxdata); + return -EBUSY; + } + } + + mutex_init(&ctxdata->mutex); + INIT_LIST_HEAD(&ctxdata->sess_list); + + ctx->data = ctxdata; + return 0; +} + +static void optee_release(struct tee_context *ctx) +{ + struct optee_context_data *ctxdata = ctx->data; + struct tee_device *teedev = ctx->teedev; + struct optee *optee = tee_get_drvdata(teedev); + struct tee_shm *shm; + struct optee_msg_arg *arg = NULL; + phys_addr_t parg; + struct optee_session *sess; + struct optee_session *sess_tmp; + + if (!ctxdata) + return; + + shm = tee_shm_alloc(ctx, sizeof(struct optee_msg_arg), TEE_SHM_MAPPED); + if (!IS_ERR(shm)) { + arg = tee_shm_get_va(shm, 0); + /* + * If va2pa fails for some reason, we can't call + * optee_close_session(), only free the memory. Secure OS + * will leak sessions and finally refuse more sessions, but + * we will at least let normal world reclaim its memory. + */ + if (!IS_ERR(arg)) + tee_shm_va2pa(shm, arg, &parg); + } + + list_for_each_entry_safe(sess, sess_tmp, &ctxdata->sess_list, + list_node) { + list_del(&sess->list_node); + if (!IS_ERR_OR_NULL(arg)) { + memset(arg, 0, sizeof(*arg)); + arg->cmd = OPTEE_MSG_CMD_CLOSE_SESSION; + arg->session = sess->session_id; + optee_do_call_with_arg(ctx, parg); + } + kfree(sess); + } + kfree(ctxdata); + + if (!IS_ERR(shm)) + tee_shm_free(shm); + + ctx->data = NULL; + + if (teedev == optee->supp_teedev) + optee_supp_release(&optee->supp); +} + +static struct tee_driver_ops optee_ops = { + .get_version = optee_get_version, + .open = optee_open, + .release = optee_release, + .open_session = optee_open_session, + .close_session = optee_close_session, + .invoke_func = optee_invoke_func, + .cancel_req = optee_cancel_req, +}; + +static struct tee_desc optee_desc = { + .name = DRIVER_NAME "-clnt", + .ops = &optee_ops, + .owner = THIS_MODULE, +}; + +static struct tee_driver_ops optee_supp_ops = { + .get_version = optee_get_version, + .open = optee_open, + .release = optee_release, + .supp_recv = optee_supp_recv, + .supp_send = optee_supp_send, +}; + +static struct tee_desc optee_supp_desc = { + .name = DRIVER_NAME "-supp", + .ops = &optee_supp_ops, + .owner = THIS_MODULE, + .flags = TEE_DESC_PRIVILEGED, +}; + +static bool optee_msg_api_uid_is_optee_api(optee_invoke_fn *invoke_fn) +{ + struct arm_smccc_res res; + + invoke_fn(OPTEE_SMC_CALLS_UID, 0, 0, 0, 0, 0, 0, 0, &res); + + if (res.a0 == OPTEE_MSG_UID_0 && res.a1 == OPTEE_MSG_UID_1 && + res.a2 == OPTEE_MSG_UID_2 && res.a3 == OPTEE_MSG_UID_3) + return true; + return false; +} + +static bool optee_msg_api_revision_is_compatible(optee_invoke_fn *invoke_fn) +{ + union { + struct arm_smccc_res smccc; + struct optee_smc_calls_revision_result result; + } res; + + invoke_fn(OPTEE_SMC_CALLS_REVISION, 0, 0, 0, 0, 0, 0, 0, &res.smccc); + + if (res.result.major == OPTEE_MSG_REVISION_MAJOR && + (int)res.result.minor >= OPTEE_MSG_REVISION_MINOR) + return true; + return false; +} + +static bool optee_msg_exchange_capabilities(optee_invoke_fn *invoke_fn, + u32 *sec_caps) +{ + union { + struct arm_smccc_res smccc; + struct optee_smc_exchange_capabilities_result result; + } res; + u32 a1 = 0; + + /* + * TODO This isn't enough to tell if it's UP system (from kernel + * point of view) or not, is_smp() returns the the information + * needed, but can't be called directly from here. + */ +#ifndef CONFIG_SMP + a1 |= OPTEE_SMC_NSEC_CAP_UNIPROCESSOR; +#endif + + invoke_fn(OPTEE_SMC_EXCHANGE_CAPABILITIES, a1, 0, 0, 0, 0, 0, 0, + &res.smccc); + + if (res.result.status != OPTEE_SMC_RETURN_OK) + return false; + + *sec_caps = res.result.capabilities; + return true; +} + +static struct tee_shm_pool * +optee_config_shm_ioremap(struct device *dev, optee_invoke_fn *invoke_fn, + void __iomem **ioremaped_shm) +{ + union { + struct arm_smccc_res smccc; + struct optee_smc_get_shm_config_result result; + } res; + struct tee_shm_pool *pool; + unsigned long vaddr; + phys_addr_t paddr; + size_t size; + phys_addr_t begin; + phys_addr_t end; + void __iomem *va; + struct tee_shm_pool_mem_info priv_info; + struct tee_shm_pool_mem_info dmabuf_info; + + invoke_fn(OPTEE_SMC_GET_SHM_CONFIG, 0, 0, 0, 0, 0, 0, 0, &res.smccc); + if (res.result.status != OPTEE_SMC_RETURN_OK) { + dev_info(dev, "shm service not available\n"); + return ERR_PTR(-ENOENT); + } + + if (res.result.settings != OPTEE_SMC_SHM_CACHED) { + dev_err(dev, "only normal cached shared memory supported\n"); + return ERR_PTR(-EINVAL); + } + + begin = roundup(res.result.start, PAGE_SIZE); + end = rounddown(res.result.start + res.result.size, PAGE_SIZE); + paddr = begin; + size = end - begin; + + if (size < 2 * OPTEE_SHM_NUM_PRIV_PAGES * PAGE_SIZE) { + dev_err(dev, "too small shared memory area\n"); + return ERR_PTR(-EINVAL); + } + + va = ioremap_cache(paddr, size); + if (!va) { + dev_err(dev, "shared memory ioremap failed\n"); + return ERR_PTR(-EINVAL); + } + vaddr = (unsigned long)va; + + priv_info.vaddr = vaddr; + priv_info.paddr = paddr; + priv_info.size = OPTEE_SHM_NUM_PRIV_PAGES * PAGE_SIZE; + dmabuf_info.vaddr = vaddr + OPTEE_SHM_NUM_PRIV_PAGES * PAGE_SIZE; + dmabuf_info.paddr = paddr + OPTEE_SHM_NUM_PRIV_PAGES * PAGE_SIZE; + dmabuf_info.size = size - OPTEE_SHM_NUM_PRIV_PAGES * PAGE_SIZE; + + pool = tee_shm_pool_alloc_res_mem(dev, &priv_info, &dmabuf_info); + if (IS_ERR(pool)) { + iounmap(va); + goto out; + } + + *ioremaped_shm = va; +out: + return pool; +} + +static int get_invoke_func(struct device *dev, optee_invoke_fn **invoke_fn) +{ + struct device_node *np = dev->of_node; + const char *method; + + dev_info(dev, "probing for conduit method from DT.\n"); + + if (of_property_read_string(np, "method", &method)) { + dev_warn(dev, "missing \"method\" property\n"); + return -ENXIO; + } + + if (!strcmp("hvc", method)) { + *invoke_fn = arm_smccc_hvc; + } else if (!strcmp("smc", method)) { + *invoke_fn = arm_smccc_smc; + } else { + dev_warn(dev, "invalid \"method\" property: %s\n", method); + return -EINVAL; + } + return 0; +} + +static int optee_probe(struct platform_device *pdev) +{ + optee_invoke_fn *invoke_fn; + struct tee_shm_pool *pool; + struct optee *optee = NULL; + void __iomem *ioremaped_shm = NULL; + struct tee_device *teedev; + u32 sec_caps; + int rc; + + rc = get_invoke_func(&pdev->dev, &invoke_fn); + if (rc) + return rc; + + if (!optee_msg_api_uid_is_optee_api(invoke_fn)) { + dev_warn(&pdev->dev, "api uid mismatch\n"); + return -EINVAL; + } + + if (!optee_msg_api_revision_is_compatible(invoke_fn)) { + dev_warn(&pdev->dev, "api revision mismatch\n"); + return -EINVAL; + } + + if (!optee_msg_exchange_capabilities(invoke_fn, &sec_caps)) { + dev_warn(&pdev->dev, "capabilities mismatch\n"); + return -EINVAL; + } + + /* + * We have no other option for shared memory, if secure world + * doesn't have any reserved memory we can use we can't continue. + */ + if (!(sec_caps & OPTEE_SMC_SEC_CAP_HAVE_RESERVERED_SHM)) + return -EINVAL; + + pool = optee_config_shm_ioremap(&pdev->dev, invoke_fn, &ioremaped_shm); + if (IS_ERR(pool)) + return PTR_ERR(pool); + + optee = devm_kzalloc(&pdev->dev, sizeof(*optee), GFP_KERNEL); + if (!optee) { + rc = -ENOMEM; + goto err; + } + + optee->dev = &pdev->dev; + optee->invoke_fn = invoke_fn; + + teedev = tee_device_alloc(&optee_desc, &pdev->dev, pool, optee); + if (IS_ERR(teedev)) { + rc = PTR_ERR(teedev); + goto err; + } + optee->teedev = teedev; + + teedev = tee_device_alloc(&optee_supp_desc, &pdev->dev, pool, optee); + if (IS_ERR(teedev)) { + rc = PTR_ERR(teedev); + goto err; + } + optee->supp_teedev = teedev; + + rc = tee_device_register(optee->teedev); + if (rc) + goto err; + + rc = tee_device_register(optee->supp_teedev); + if (rc) + goto err; + + mutex_init(&optee->call_queue.mutex); + INIT_LIST_HEAD(&optee->call_queue.waiters); + optee_wait_queue_init(&optee->wait_queue); + optee_supp_init(&optee->supp); + optee->ioremaped_shm = ioremaped_shm; + optee->pool = pool; + + platform_set_drvdata(pdev, optee); + + optee_enable_shm_cache(optee); + + dev_info(&pdev->dev, "initialized driver\n"); + return 0; +err: + if (optee) { + /* + * tee_device_unregister() is safe to call even if the + * devices hasn't been registered with + * tee_device_register() yet. + */ + tee_device_unregister(optee->supp_teedev); + tee_device_unregister(optee->teedev); + } + if (pool) + tee_shm_pool_free(pool); + if (ioremaped_shm) + iounmap(ioremaped_shm); + return rc; +} + +static int optee_remove(struct platform_device *pdev) +{ + struct optee *optee = platform_get_drvdata(pdev); + + /* + * Ask OP-TEE to free all cached shared memory objects to decrease + * reference counters and also avoid wild pointers in secure world + * into the old shared memory range. + */ + optee_disable_shm_cache(optee); + + /* + * The two devices has to be unregistered before we can free the + * other resources. + */ + tee_device_unregister(optee->supp_teedev); + tee_device_unregister(optee->teedev); + + tee_shm_pool_free(optee->pool); + if (optee->ioremaped_shm) + iounmap(optee->ioremaped_shm); + optee_wait_queue_exit(&optee->wait_queue); + optee_supp_uninit(&optee->supp); + mutex_destroy(&optee->call_queue.mutex); + + return 0; +} + +static const struct of_device_id optee_match[] = { + { .compatible = "linaro,optee-tz" }, + {}, +}; + +static struct platform_driver optee_driver = { + .driver = { + .name = DRIVER_NAME, + .of_match_table = optee_match, + }, + .probe = optee_probe, + .remove = optee_remove, +}; + +static int __init optee_driver_init(void) +{ + struct device_node *node; + + /* + * Preferred path is /firmware/optee, but it's the matching that + * matters. + */ + for_each_matching_node(node, optee_match) + of_platform_device_create(node, NULL, NULL); + + return platform_driver_register(&optee_driver); +} +module_init(optee_driver_init); + +static void __exit optee_driver_exit(void) +{ + platform_driver_unregister(&optee_driver); +} +module_exit(optee_driver_exit); + +MODULE_AUTHOR("Linaro"); +MODULE_DESCRIPTION("OP-TEE driver"); +MODULE_SUPPORTED_DEVICE(""); +MODULE_VERSION("1.0"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/tee/optee/optee_msg.h b/drivers/tee/optee/optee_msg.h new file mode 100644 index 00000000000000..3c0a912136107d --- /dev/null +++ b/drivers/tee/optee/optee_msg.h @@ -0,0 +1,435 @@ +/* + * Copyright (c) 2015-2016, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef _OPTEE_MSG_H +#define _OPTEE_MSG_H + +#include +#include + +/* + * This file defines the OP-TEE message protocol used to communicate + * with an instance of OP-TEE running in secure world. + * + * This file is divided into three sections. + * 1. Formatting of messages. + * 2. Requests from normal world + * 3. Requests from secure world, Remote Procedure Call (RPC), handled by + * tee-supplicant. + */ + +/***************************************************************************** + * Part 1 - formatting of messages + *****************************************************************************/ + +#define OPTEE_MSG_ATTR_TYPE_NONE 0x0 +#define OPTEE_MSG_ATTR_TYPE_VALUE_INPUT 0x1 +#define OPTEE_MSG_ATTR_TYPE_VALUE_OUTPUT 0x2 +#define OPTEE_MSG_ATTR_TYPE_VALUE_INOUT 0x3 +#define OPTEE_MSG_ATTR_TYPE_RMEM_INPUT 0x5 +#define OPTEE_MSG_ATTR_TYPE_RMEM_OUTPUT 0x6 +#define OPTEE_MSG_ATTR_TYPE_RMEM_INOUT 0x7 +#define OPTEE_MSG_ATTR_TYPE_TMEM_INPUT 0x9 +#define OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT 0xa +#define OPTEE_MSG_ATTR_TYPE_TMEM_INOUT 0xb + +#define OPTEE_MSG_ATTR_TYPE_MASK GENMASK(7, 0) + +/* + * Meta parameter to be absorbed by the Secure OS and not passed + * to the Trusted Application. + * + * Currently only used with OPTEE_MSG_CMD_OPEN_SESSION. + */ +#define OPTEE_MSG_ATTR_META BIT(8) + +/* + * The temporary shared memory object is not physically contigous and this + * temp memref is followed by another fragment until the last temp memref + * that doesn't have this bit set. + */ +#define OPTEE_MSG_ATTR_FRAGMENT BIT(9) + +/* + * Memory attributes for caching passed with temp memrefs. The actual value + * used is defined outside the message protocol with the exception of + * OPTEE_MSG_ATTR_CACHE_PREDEFINED which means the attributes already + * defined for the memory range should be used. If optee_smc.h is used as + * bearer of this protocol OPTEE_SMC_SHM_* is used for values. + */ +#define OPTEE_MSG_ATTR_CACHE_SHIFT 16 +#define OPTEE_MSG_ATTR_CACHE_MASK GENMASK(2, 0) +#define OPTEE_MSG_ATTR_CACHE_PREDEFINED 0 + +/* + * Same values as TEE_LOGIN_* from TEE Internal API + */ +#define OPTEE_MSG_LOGIN_PUBLIC 0x00000000 +#define OPTEE_MSG_LOGIN_USER 0x00000001 +#define OPTEE_MSG_LOGIN_GROUP 0x00000002 +#define OPTEE_MSG_LOGIN_APPLICATION 0x00000004 +#define OPTEE_MSG_LOGIN_APPLICATION_USER 0x00000005 +#define OPTEE_MSG_LOGIN_APPLICATION_GROUP 0x00000006 + +/** + * struct optee_msg_param_tmem - temporary memory reference parameter + * @buf_ptr: Address of the buffer + * @size: Size of the buffer + * @shm_ref: Temporary shared memory reference, pointer to a struct tee_shm + * + * Secure and normal world communicates pointers as physical address + * instead of the virtual address. This is because secure and normal world + * have completely independent memory mapping. Normal world can even have a + * hypervisor which need to translate the guest physical address (AKA IPA + * in ARM documentation) to a real physical address before passing the + * structure to secure world. + */ +struct optee_msg_param_tmem { + u64 buf_ptr; + u64 size; + u64 shm_ref; +}; + +/** + * struct optee_msg_param_rmem - registered memory reference parameter + * @offs: Offset into shared memory reference + * @size: Size of the buffer + * @shm_ref: Shared memory reference, pointer to a struct tee_shm + */ +struct optee_msg_param_rmem { + u64 offs; + u64 size; + u64 shm_ref; +}; + +/** + * struct optee_msg_param_value - opaque value parameter + * + * Value parameters are passed unchecked between normal and secure world. + */ +struct optee_msg_param_value { + u64 a; + u64 b; + u64 c; +}; + +/** + * struct optee_msg_param - parameter used together with struct optee_msg_arg + * @attr: attributes + * @tmem: parameter by temporary memory reference + * @rmem: parameter by registered memory reference + * @value: parameter by opaque value + * + * @attr & OPTEE_MSG_ATTR_TYPE_MASK indicates if tmem, rmem or value is used in + * the union. OPTEE_MSG_ATTR_TYPE_VALUE_* indicates value, + * OPTEE_MSG_ATTR_TYPE_TMEM_* indicates tmem and + * OPTEE_MSG_ATTR_TYPE_RMEM_* indicates rmem. + * OPTEE_MSG_ATTR_TYPE_NONE indicates that none of the members are used. + */ +struct optee_msg_param { + u64 attr; + union { + struct optee_msg_param_tmem tmem; + struct optee_msg_param_rmem rmem; + struct optee_msg_param_value value; + } u; +}; + +/** + * struct optee_msg_arg - call argument + * @cmd: Command, one of OPTEE_MSG_CMD_* or OPTEE_MSG_RPC_CMD_* + * @func: Trusted Application function, specific to the Trusted Application, + * used if cmd == OPTEE_MSG_CMD_INVOKE_COMMAND + * @session: In parameter for all OPTEE_MSG_CMD_* except + * OPTEE_MSG_CMD_OPEN_SESSION where it's an output parameter instead + * @cancel_id: Cancellation id, a unique value to identify this request + * @ret: return value + * @ret_origin: origin of the return value + * @num_params: number of parameters supplied to the OS Command + * @params: the parameters supplied to the OS Command + * + * All normal calls to Trusted OS uses this struct. If cmd requires further + * information than what these field holds it can be passed as a parameter + * tagged as meta (setting the OPTEE_MSG_ATTR_META bit in corresponding + * attrs field). All parameters tagged as meta has to come first. + * + * Temp memref parameters can be fragmented if supported by the Trusted OS + * (when optee_smc.h is bearer of this protocol this is indicated with + * OPTEE_SMC_SEC_CAP_UNREGISTERED_SHM). If a logical memref parameter is + * fragmented then has all but the last fragment the + * OPTEE_MSG_ATTR_FRAGMENT bit set in attrs. Even if a memref is fragmented + * it will still be presented as a single logical memref to the Trusted + * Application. + */ +struct optee_msg_arg { + u32 cmd; + u32 func; + u32 session; + u32 cancel_id; + u32 pad; + u32 ret; + u32 ret_origin; + u32 num_params; + + /* + * this struct is 8 byte aligned since the 'struct optee_msg_param' + * which follows requires 8 byte alignment. + * + * Commented out element used to visualize the layout dynamic part + * of the struct. This field is not available at all if + * num_params == 0. + * + * params is accessed through the macro OPTEE_MSG_GET_PARAMS + * + * struct optee_msg_param params[num_params]; + */ +} __aligned(8); + +/** + * OPTEE_MSG_GET_PARAMS - return pointer to struct optee_msg_param * + * + * @x: Pointer to a struct optee_msg_arg + * + * Returns a pointer to the params[] inside a struct optee_msg_arg. + */ +#define OPTEE_MSG_GET_PARAMS(x) \ + (struct optee_msg_param *)(((struct optee_msg_arg *)(x)) + 1) + +/** + * OPTEE_MSG_GET_ARG_SIZE - return size of struct optee_msg_arg + * + * @num_params: Number of parameters embedded in the struct optee_msg_arg + * + * Returns the size of the struct optee_msg_arg together with the number + * of embedded parameters. + */ +#define OPTEE_MSG_GET_ARG_SIZE(num_params) \ + (sizeof(struct optee_msg_arg) + \ + sizeof(struct optee_msg_param) * (num_params)) + +/***************************************************************************** + * Part 2 - requests from normal world + *****************************************************************************/ + +/* + * Return the following UID if using API specified in this file without + * further extensions: + * 384fb3e0-e7f8-11e3-af63-0002a5d5c51b. + * Represented in 4 32-bit words in OPTEE_MSG_UID_0, OPTEE_MSG_UID_1, + * OPTEE_MSG_UID_2, OPTEE_MSG_UID_3. + */ +#define OPTEE_MSG_UID_0 0x384fb3e0 +#define OPTEE_MSG_UID_1 0xe7f811e3 +#define OPTEE_MSG_UID_2 0xaf630002 +#define OPTEE_MSG_UID_3 0xa5d5c51b +#define OPTEE_MSG_FUNCID_CALLS_UID 0xFF01 + +/* + * Returns 2.0 if using API specified in this file without further + * extensions. Represented in 2 32-bit words in OPTEE_MSG_REVISION_MAJOR + * and OPTEE_MSG_REVISION_MINOR + */ +#define OPTEE_MSG_REVISION_MAJOR 2 +#define OPTEE_MSG_REVISION_MINOR 0 +#define OPTEE_MSG_FUNCID_CALLS_REVISION 0xFF03 + +/* + * Get UUID of Trusted OS. + * + * Used by non-secure world to figure out which Trusted OS is installed. + * Note that returned UUID is the UUID of the Trusted OS, not of the API. + * + * Returns UUID in 4 32-bit words in the same way as + * OPTEE_MSG_FUNCID_CALLS_UID described above. + */ +#define OPTEE_MSG_OS_OPTEE_UUID_0 0x486178e0 +#define OPTEE_MSG_OS_OPTEE_UUID_1 0xe7f811e3 +#define OPTEE_MSG_OS_OPTEE_UUID_2 0xbc5e0002 +#define OPTEE_MSG_OS_OPTEE_UUID_3 0xa5d5c51b +#define OPTEE_MSG_FUNCID_GET_OS_UUID 0x0000 + +/* + * Get revision of Trusted OS. + * + * Used by non-secure world to figure out which version of the Trusted OS + * is installed. Note that the returned revision is the revision of the + * Trusted OS, not of the API. + * + * Returns revision in 2 32-bit words in the same way as + * OPTEE_MSG_CALLS_REVISION described above. + */ +#define OPTEE_MSG_OS_OPTEE_REVISION_MAJOR 1 +#define OPTEE_MSG_OS_OPTEE_REVISION_MINOR 0 +#define OPTEE_MSG_FUNCID_GET_OS_REVISION 0x0001 + +/* + * Do a secure call with struct optee_msg_arg as argument + * The OPTEE_MSG_CMD_* below defines what goes in struct optee_msg_arg::cmd + * + * OPTEE_MSG_CMD_OPEN_SESSION opens a session to a Trusted Application. + * The first two parameters are tagged as meta, holding two value + * parameters to pass the following information: + * param[0].u.value.a-b uuid of Trusted Application + * param[1].u.value.a-b uuid of Client + * param[1].u.value.c Login class of client OPTEE_MSG_LOGIN_* + * + * OPTEE_MSG_CMD_INVOKE_COMMAND invokes a command a previously opened + * session to a Trusted Application. struct optee_msg_arg::func is Trusted + * Application function, specific to the Trusted Application. + * + * OPTEE_MSG_CMD_CLOSE_SESSION closes a previously opened session to + * Trusted Application. + * + * OPTEE_MSG_CMD_CANCEL cancels a currently invoked command. + * + * OPTEE_MSG_CMD_REGISTER_SHM registers a shared memory reference. The + * information is passed as: + * [in] param[0].attr OPTEE_MSG_ATTR_TYPE_TMEM_INPUT + * [| OPTEE_MSG_ATTR_FRAGMENT] + * [in] param[0].u.tmem.buf_ptr physical address (of first fragment) + * [in] param[0].u.tmem.size size (of first fragment) + * [in] param[0].u.tmem.shm_ref holds shared memory reference + * ... + * The shared memory can optionally be fragmented, temp memrefs can follow + * each other with all but the last with the OPTEE_MSG_ATTR_FRAGMENT bit set. + * + * OPTEE_MSG_CMD_UNREGISTER_SHM unregisteres a previously registered shared + * memory reference. The information is passed as: + * [in] param[0].attr OPTEE_MSG_ATTR_TYPE_RMEM_INPUT + * [in] param[0].u.rmem.shm_ref holds shared memory reference + * [in] param[0].u.rmem.offs 0 + * [in] param[0].u.rmem.size 0 + */ +#define OPTEE_MSG_CMD_OPEN_SESSION 0 +#define OPTEE_MSG_CMD_INVOKE_COMMAND 1 +#define OPTEE_MSG_CMD_CLOSE_SESSION 2 +#define OPTEE_MSG_CMD_CANCEL 3 +#define OPTEE_MSG_CMD_REGISTER_SHM 4 +#define OPTEE_MSG_CMD_UNREGISTER_SHM 5 +#define OPTEE_MSG_FUNCID_CALL_WITH_ARG 0x0004 + +/***************************************************************************** + * Part 3 - Requests from secure world, RPC + *****************************************************************************/ + +/* + * All RPC is done with a struct optee_msg_arg as bearer of information, + * struct optee_msg_arg::arg holds values defined by OPTEE_MSG_RPC_CMD_* below + * + * RPC communication with tee-supplicant is reversed compared to normal + * client communication desribed above. The supplicant receives requests + * and sends responses. + */ + +/* + * Load a TA into memory, defined in tee-supplicant + */ +#define OPTEE_MSG_RPC_CMD_LOAD_TA 0 + +/* + * Reserved + */ +#define OPTEE_MSG_RPC_CMD_RPMB 1 + +/* + * File system access, defined in tee-supplicant + */ +#define OPTEE_MSG_RPC_CMD_FS 2 + +/* + * Get time + * + * Returns number of seconds and nano seconds since the Epoch, + * 1970-01-01 00:00:00 +0000 (UTC). + * + * [out] param[0].u.value.a Number of seconds + * [out] param[0].u.value.b Number of nano seconds. + */ +#define OPTEE_MSG_RPC_CMD_GET_TIME 3 + +/* + * Wait queue primitive, helper for secure world to implement a wait queue + * + * Waiting on a key + * [in] param[0].u.value.a OPTEE_MSG_RPC_WAIT_QUEUE_SLEEP + * [in] param[0].u.value.b wait key + * + * Waking up a key + * [in] param[0].u.value.a OPTEE_MSG_RPC_WAIT_QUEUE_WAKEUP + * [in] param[0].u.value.b wakeup key + */ +#define OPTEE_MSG_RPC_CMD_WAIT_QUEUE 4 +#define OPTEE_MSG_RPC_WAIT_QUEUE_SLEEP 0 +#define OPTEE_MSG_RPC_WAIT_QUEUE_WAKEUP 1 + +/* + * Suspend execution + * + * [in] param[0].value .a number of milliseconds to suspend + */ +#define OPTEE_MSG_RPC_CMD_SUSPEND 5 + +/* + * Allocate a piece of shared memory + * + * Shared memory can optionally be fragmented, to support that additional + * spare param entries are allocated to make room for eventual fragments. + * The spare param entries has .attr = OPTEE_MSG_ATTR_TYPE_NONE when + * unused. All returned temp memrefs except the last should have the + * OPTEE_MSG_ATTR_FRAGMENT bit set in the attr field. + * + * [in] param[0].u.value.a type of memory one of + * OPTEE_MSG_RPC_SHM_TYPE_* below + * [in] param[0].u.value.b requested size + * [in] param[0].u.value.c required alignment + * + * [out] param[0].u.tmem.buf_ptr physical address (of first fragment) + * [out] param[0].u.tmem.size size (of first fragment) + * [out] param[0].u.tmem.shm_ref shared memory reference + * ... + * [out] param[n].u.tmem.buf_ptr physical address + * [out] param[n].u.tmem.size size + * [out] param[n].u.tmem.shm_ref shared memory reference (same value + * as in param[n-1].u.tmem.shm_ref) + */ +#define OPTEE_MSG_RPC_CMD_SHM_ALLOC 6 +/* Memory that can be shared with a non-secure user space application */ +#define OPTEE_MSG_RPC_SHM_TYPE_APPL 0 +/* Memory only shared with non-secure kernel */ +#define OPTEE_MSG_RPC_SHM_TYPE_KERNEL 1 + +/* + * Free shared memory previously allocated with OPTEE_MSG_RPC_CMD_SHM_ALLOC + * + * [in] param[0].u.value.a type of memory one of + * OPTEE_MSG_RPC_SHM_TYPE_* above + * [in] param[0].u.value.b value of shared memory reference + * returned in param[0].u.tmem.shm_ref + * above + */ +#define OPTEE_MSG_RPC_CMD_SHM_FREE 7 + +#endif /* _OPTEE_MSG_H */ diff --git a/drivers/tee/optee/optee_private.h b/drivers/tee/optee/optee_private.h new file mode 100644 index 00000000000000..58c8ef33359a82 --- /dev/null +++ b/drivers/tee/optee/optee_private.h @@ -0,0 +1,174 @@ +/* + * Copyright (c) 2015, Linaro Limited + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#ifndef OPTEE_PRIVATE_H +#define OPTEE_PRIVATE_H + +#include +#include +#include +#include +#include "optee_msg.h" + +#define OPTEE_MAX_ARG_SIZE 1024 + +/* Some Global Platform error codes used in this driver */ +#define TEEC_SUCCESS 0x00000000 +#define TEEC_ERROR_BAD_PARAMETERS 0xFFFF0006 +#define TEEC_ERROR_COMMUNICATION 0xFFFF000E +#define TEEC_ERROR_OUT_OF_MEMORY 0xFFFF000C + +#define TEEC_ORIGIN_COMMS 0x00000002 + +typedef void (optee_invoke_fn)(unsigned long, unsigned long, unsigned long, + unsigned long, unsigned long, unsigned long, + unsigned long, unsigned long, + struct arm_smccc_res *); + +struct optee_call_queue { + /* Serializes access to this struct */ + struct mutex mutex; + struct list_head waiters; +}; + +struct optee_wait_queue { + /* Serializes access to this struct */ + struct mutex mu; + struct list_head db; +}; + +/** + * struct optee_supp - supplicant synchronization struct + * @ctx the context of current connected supplicant. + * if !NULL the supplicant device is available for use, + * else busy + * @mutex: held while accessing content of this struct + * @req_id: current request id if supplicant is doing synchronous + * communication, else -1 + * @reqs: queued request not yet retrieved by supplicant + * @idr: IDR holding all requests currently being processed + * by supplicant + * @reqs_c: completion used by supplicant when waiting for a + * request to be queued. + */ +struct optee_supp { + /* Serializes access to this struct */ + struct mutex mutex; + struct tee_context *ctx; + + int req_id; + struct list_head reqs; + struct idr idr; + struct completion reqs_c; +}; + +/** + * struct optee - main service struct + * @supp_teedev: supplicant device + * @teedev: client device + * @dev: probed device + * @invoke_fn: function to issue smc or hvc + * @call_queue: queue of threads waiting to call @invoke_fn + * @wait_queue: queue of threads from secure world waiting for a + * secure world sync object + * @supp: supplicant synchronization struct for RPC to supplicant + * @pool: shared memory pool + * @ioremaped_shm virtual address of memory in shared memory pool + */ +struct optee { + struct tee_device *supp_teedev; + struct tee_device *teedev; + struct device *dev; + optee_invoke_fn *invoke_fn; + struct optee_call_queue call_queue; + struct optee_wait_queue wait_queue; + struct optee_supp supp; + struct tee_shm_pool *pool; + void __iomem *ioremaped_shm; +}; + +struct optee_session { + struct list_head list_node; + u32 session_id; +}; + +struct optee_context_data { + /* Serializes access to this struct */ + struct mutex mutex; + struct list_head sess_list; +}; + +struct optee_rpc_param { + u32 a0; + u32 a1; + u32 a2; + u32 a3; + u32 a4; + u32 a5; + u32 a6; + u32 a7; +}; + +void optee_handle_rpc(struct tee_context *ctx, struct optee_rpc_param *param); + +void optee_wait_queue_init(struct optee_wait_queue *wq); +void optee_wait_queue_exit(struct optee_wait_queue *wq); + +u32 optee_supp_thrd_req(struct tee_context *ctx, u32 func, size_t num_params, + struct tee_param *param); + +int optee_supp_read(struct tee_context *ctx, void __user *buf, size_t len); +int optee_supp_write(struct tee_context *ctx, void __user *buf, size_t len); +void optee_supp_init(struct optee_supp *supp); +void optee_supp_uninit(struct optee_supp *supp); +void optee_supp_release(struct optee_supp *supp); + +int optee_supp_recv(struct tee_context *ctx, u32 *func, u32 *num_params, + struct tee_param *param); +int optee_supp_send(struct tee_context *ctx, u32 ret, u32 num_params, + struct tee_param *param); + +u32 optee_do_call_with_arg(struct tee_context *ctx, phys_addr_t parg); +int optee_open_session(struct tee_context *ctx, + struct tee_ioctl_open_session_arg *arg, + struct tee_param *param); +int optee_close_session(struct tee_context *ctx, u32 session); +int optee_invoke_func(struct tee_context *ctx, struct tee_ioctl_invoke_arg *arg, + struct tee_param *param); +int optee_cancel_req(struct tee_context *ctx, u32 cancel_id, u32 session); + +void optee_enable_shm_cache(struct optee *optee); +void optee_disable_shm_cache(struct optee *optee); + +int optee_from_msg_param(struct tee_param *params, size_t num_params, + const struct optee_msg_param *msg_params); +int optee_to_msg_param(struct optee_msg_param *msg_params, size_t num_params, + const struct tee_param *params); + +/* + * Small helpers + */ + +static inline void *reg_pair_to_ptr(u32 reg0, u32 reg1) +{ + return (void *)(unsigned long)(((u64)reg0 << 32) | reg1); +} + +static inline void reg_pair_from_64(u32 *reg0, u32 *reg1, u64 val) +{ + *reg0 = val >> 32; + *reg1 = val; +} + +#endif /*OPTEE_PRIVATE_H*/ diff --git a/drivers/tee/optee/optee_smc.h b/drivers/tee/optee/optee_smc.h new file mode 100644 index 00000000000000..d470d450b1822f --- /dev/null +++ b/drivers/tee/optee/optee_smc.h @@ -0,0 +1,450 @@ +/* + * Copyright (c) 2015-2016, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef OPTEE_SMC_H +#define OPTEE_SMC_H + +#include +#include + +#define OPTEE_SMC_STD_CALL_VAL(func_num) \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_STD_CALL, ARM_SMCCC_SMC_32, \ + ARM_SMCCC_OWNER_TRUSTED_OS, (func_num)) +#define OPTEE_SMC_FAST_CALL_VAL(func_num) \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, ARM_SMCCC_SMC_32, \ + ARM_SMCCC_OWNER_TRUSTED_OS, (func_num)) + +/* + * Function specified by SMC Calling convention. + */ +#define OPTEE_SMC_FUNCID_CALLS_COUNT 0xFF00 +#define OPTEE_SMC_CALLS_COUNT \ + ARM_SMCCC_CALL_VAL(OPTEE_SMC_FAST_CALL, SMCCC_SMC_32, \ + SMCCC_OWNER_TRUSTED_OS_END, \ + OPTEE_SMC_FUNCID_CALLS_COUNT) + +/* + * Normal cached memory (write-back), shareable for SMP systems and not + * shareable for UP systems. + */ +#define OPTEE_SMC_SHM_CACHED 1 + +/* + * a0..a7 is used as register names in the descriptions below, on arm32 + * that translates to r0..r7 and on arm64 to w0..w7. In both cases it's + * 32-bit registers. + */ + +/* + * Function specified by SMC Calling convention + * + * Return one of the following UIDs if using API specified in this file + * without further extentions: + * 65cb6b93-af0c-4617-8ed6-644a8d1140f8 + * see also OPTEE_SMC_UID_* in optee_msg.h + */ +#define OPTEE_SMC_FUNCID_CALLS_UID OPTEE_MSG_FUNCID_CALLS_UID +#define OPTEE_SMC_CALLS_UID \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, ARM_SMCCC_SMC_32, \ + ARM_SMCCC_OWNER_TRUSTED_OS_END, \ + OPTEE_SMC_FUNCID_CALLS_UID) + +/* + * Function specified by SMC Calling convention + * + * Returns 2.0 if using API specified in this file without further extentions. + * see also OPTEE_MSG_REVISION_* in optee_msg.h + */ +#define OPTEE_SMC_FUNCID_CALLS_REVISION OPTEE_MSG_FUNCID_CALLS_REVISION +#define OPTEE_SMC_CALLS_REVISION \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, ARM_SMCCC_SMC_32, \ + ARM_SMCCC_OWNER_TRUSTED_OS_END, \ + OPTEE_SMC_FUNCID_CALLS_REVISION) + +struct optee_smc_calls_revision_result { + unsigned long major; + unsigned long minor; + unsigned long reserved0; + unsigned long reserved1; +}; + +/* + * Get UUID of Trusted OS. + * + * Used by non-secure world to figure out which Trusted OS is installed. + * Note that returned UUID is the UUID of the Trusted OS, not of the API. + * + * Returns UUID in a0-4 in the same way as OPTEE_SMC_CALLS_UID + * described above. + */ +#define OPTEE_SMC_FUNCID_GET_OS_UUID OPTEE_MSG_FUNCID_GET_OS_UUID +#define OPTEE_SMC_CALL_GET_OS_UUID \ + OPTEE_SMC_FAST_CALL_VAL(OPTEE_SMC_FUNCID_GET_OS_UUID) + +/* + * Get revision of Trusted OS. + * + * Used by non-secure world to figure out which version of the Trusted OS + * is installed. Note that the returned revision is the revision of the + * Trusted OS, not of the API. + * + * Returns revision in a0-1 in the same way as OPTEE_SMC_CALLS_REVISION + * described above. + */ +#define OPTEE_SMC_FUNCID_GET_OS_REVISION OPTEE_MSG_FUNCID_GET_OS_REVISION +#define OPTEE_SMC_CALL_GET_OS_REVISION \ + OPTEE_SMC_FAST_CALL_VAL(OPTEE_SMC_FUNCID_GET_OS_REVISION) + +/* + * Call with struct optee_msg_arg as argument + * + * Call register usage: + * a0 SMC Function ID, OPTEE_SMC*CALL_WITH_ARG + * a1 Upper 32bit of a 64bit physical pointer to a struct optee_msg_arg + * a2 Lower 32bit of a 64bit physical pointer to a struct optee_msg_arg + * a3 Cache settings, not used if physical pointer is in a predefined shared + * memory area else per OPTEE_SMC_SHM_* + * a4-6 Not used + * a7 Hypervisor Client ID register + * + * Normal return register usage: + * a0 Return value, OPTEE_SMC_RETURN_* + * a1-3 Not used + * a4-7 Preserved + * + * OPTEE_SMC_RETURN_ETHREAD_LIMIT return register usage: + * a0 Return value, OPTEE_SMC_RETURN_ETHREAD_LIMIT + * a1-3 Preserved + * a4-7 Preserved + * + * RPC return register usage: + * a0 Return value, OPTEE_SMC_RETURN_IS_RPC(val) + * a1-2 RPC parameters + * a3-7 Resume information, must be preserved + * + * Possible return values: + * OPTEE_SMC_RETURN_UNKNOWN_FUNCTION Trusted OS does not recognize this + * function. + * OPTEE_SMC_RETURN_OK Call completed, result updated in + * the previously supplied struct + * optee_msg_arg. + * OPTEE_SMC_RETURN_ETHREAD_LIMIT Number of Trusted OS threads exceeded, + * try again later. + * OPTEE_SMC_RETURN_EBADADDR Bad physcial pointer to struct + * optee_msg_arg. + * OPTEE_SMC_RETURN_EBADCMD Bad/unknown cmd in struct optee_msg_arg + * OPTEE_SMC_RETURN_IS_RPC() Call suspended by RPC call to normal + * world. + */ +#define OPTEE_SMC_FUNCID_CALL_WITH_ARG OPTEE_MSG_FUNCID_CALL_WITH_ARG +#define OPTEE_SMC_CALL_WITH_ARG \ + OPTEE_SMC_STD_CALL_VAL(OPTEE_SMC_FUNCID_CALL_WITH_ARG) + +/* + * Get Shared Memory Config + * + * Returns the Secure/Non-secure shared memory config. + * + * Call register usage: + * a0 SMC Function ID, OPTEE_SMC_GET_SHM_CONFIG + * a1-6 Not used + * a7 Hypervisor Client ID register + * + * Have config return register usage: + * a0 OPTEE_SMC_RETURN_OK + * a1 Physical address of start of SHM + * a2 Size of of SHM + * a3 Cache settings of memory, as defined by the + * OPTEE_SMC_SHM_* values above + * a4-7 Preserved + * + * Not available register usage: + * a0 OPTEE_SMC_RETURN_ENOTAVAIL + * a1-3 Not used + * a4-7 Preserved + */ +#define OPTEE_SMC_FUNCID_GET_SHM_CONFIG 7 +#define OPTEE_SMC_GET_SHM_CONFIG \ + OPTEE_SMC_FAST_CALL_VAL(OPTEE_SMC_FUNCID_GET_SHM_CONFIG) + +struct optee_smc_get_shm_config_result { + unsigned long status; + unsigned long start; + unsigned long size; + unsigned long settings; +}; + +/* + * Exchanges capabilities between normal world and secure world + * + * Call register usage: + * a0 SMC Function ID, OPTEE_SMC_EXCHANGE_CAPABILITIES + * a1 bitfield of normal world capabilities OPTEE_SMC_NSEC_CAP_* + * a2-6 Not used + * a7 Hypervisor Client ID register + * + * Normal return register usage: + * a0 OPTEE_SMC_RETURN_OK + * a1 bitfield of secure world capabilities OPTEE_SMC_SEC_CAP_* + * a2-7 Preserved + * + * Error return register usage: + * a0 OPTEE_SMC_RETURN_ENOTAVAIL, can't use the capabilities from normal world + * a1 bitfield of secure world capabilities OPTEE_SMC_SEC_CAP_* + * a2-7 Preserved + */ +/* Normal world works as a uniprocessor system */ +#define OPTEE_SMC_NSEC_CAP_UNIPROCESSOR BIT(0) +/* Secure world has reserved shared memory for normal world to use */ +#define OPTEE_SMC_SEC_CAP_HAVE_RESERVERED_SHM BIT(0) +/* Secure world can communicate via previously unregistered shared memory */ +#define OPTEE_SMC_SEC_CAP_UNREGISTERED_SHM BIT(1) +#define OPTEE_SMC_FUNCID_EXCHANGE_CAPABILITIES 9 +#define OPTEE_SMC_EXCHANGE_CAPABILITIES \ + OPTEE_SMC_FAST_CALL_VAL(OPTEE_SMC_FUNCID_EXCHANGE_CAPABILITIES) + +struct optee_smc_exchange_capabilities_result { + unsigned long status; + unsigned long capabilities; + unsigned long reserved0; + unsigned long reserved1; +}; + +/* + * Disable and empties cache of shared memory objects + * + * Secure world can cache frequently used shared memory objects, for + * example objects used as RPC arguments. When secure world is idle this + * function returns one shared memory reference to free. To disable the + * cache and free all cached objects this function has to be called until + * it returns OPTEE_SMC_RETURN_ENOTAVAIL. + * + * Call register usage: + * a0 SMC Function ID, OPTEE_SMC_DISABLE_SHM_CACHE + * a1-6 Not used + * a7 Hypervisor Client ID register + * + * Normal return register usage: + * a0 OPTEE_SMC_RETURN_OK + * a1 Upper 32bit of a 64bit Shared memory cookie + * a2 Lower 32bit of a 64bit Shared memory cookie + * a3-7 Preserved + * + * Cache empty return register usage: + * a0 OPTEE_SMC_RETURN_ENOTAVAIL + * a1-7 Preserved + * + * Not idle return register usage: + * a0 OPTEE_SMC_RETURN_EBUSY + * a1-7 Preserved + */ +#define OPTEE_SMC_FUNCID_DISABLE_SHM_CACHE 10 +#define OPTEE_SMC_DISABLE_SHM_CACHE \ + OPTEE_SMC_FAST_CALL_VAL(OPTEE_SMC_FUNCID_DISABLE_SHM_CACHE) + +struct optee_smc_disable_shm_cache_result { + unsigned long status; + unsigned long shm_upper32; + unsigned long shm_lower32; + unsigned long reserved0; +}; + +/* + * Enable cache of shared memory objects + * + * Secure world can cache frequently used shared memory objects, for + * example objects used as RPC arguments. When secure world is idle this + * function returns OPTEE_SMC_RETURN_OK and the cache is enabled. If + * secure world isn't idle OPTEE_SMC_RETURN_EBUSY is returned. + * + * Call register usage: + * a0 SMC Function ID, OPTEE_SMC_ENABLE_SHM_CACHE + * a1-6 Not used + * a7 Hypervisor Client ID register + * + * Normal return register usage: + * a0 OPTEE_SMC_RETURN_OK + * a1-7 Preserved + * + * Not idle return register usage: + * a0 OPTEE_SMC_RETURN_EBUSY + * a1-7 Preserved + */ +#define OPTEE_SMC_FUNCID_ENABLE_SHM_CACHE 11 +#define OPTEE_SMC_ENABLE_SHM_CACHE \ + OPTEE_SMC_FAST_CALL_VAL(OPTEE_SMC_FUNCID_ENABLE_SHM_CACHE) + +/* + * Resume from RPC (for example after processing an IRQ) + * + * Call register usage: + * a0 SMC Function ID, OPTEE_SMC_CALL_RETURN_FROM_RPC + * a1-3 Value of a1-3 when OPTEE_SMC_CALL_WITH_ARG returned + * OPTEE_SMC_RETURN_RPC in a0 + * + * Return register usage is the same as for OPTEE_SMC_*CALL_WITH_ARG above. + * + * Possible return values + * OPTEE_SMC_RETURN_UNKNOWN_FUNCTION Trusted OS does not recognize this + * function. + * OPTEE_SMC_RETURN_OK Original call completed, result + * updated in the previously supplied. + * struct optee_msg_arg + * OPTEE_SMC_RETURN_RPC Call suspended by RPC call to normal + * world. + * OPTEE_SMC_RETURN_ERESUME Resume failed, the opaque resume + * information was corrupt. + */ +#define OPTEE_SMC_FUNCID_RETURN_FROM_RPC 3 +#define OPTEE_SMC_CALL_RETURN_FROM_RPC \ + OPTEE_SMC_STD_CALL_VAL(OPTEE_SMC_FUNCID_RETURN_FROM_RPC) + +#define OPTEE_SMC_RETURN_RPC_PREFIX_MASK 0xFFFF0000 +#define OPTEE_SMC_RETURN_RPC_PREFIX 0xFFFF0000 +#define OPTEE_SMC_RETURN_RPC_FUNC_MASK 0x0000FFFF + +#define OPTEE_SMC_RETURN_GET_RPC_FUNC(ret) \ + ((ret) & OPTEE_SMC_RETURN_RPC_FUNC_MASK) + +#define OPTEE_SMC_RPC_VAL(func) ((func) | OPTEE_SMC_RETURN_RPC_PREFIX) + +/* + * Allocate memory for RPC parameter passing. The memory is used to hold a + * struct optee_msg_arg. + * + * "Call" register usage: + * a0 This value, OPTEE_SMC_RETURN_RPC_ALLOC + * a1 Size in bytes of required argument memory + * a2 Not used + * a3 Resume information, must be preserved + * a4-5 Not used + * a6-7 Resume information, must be preserved + * + * "Return" register usage: + * a0 SMC Function ID, OPTEE_SMC_CALL_RETURN_FROM_RPC. + * a1 Upper 32bits of 64bit physical pointer to allocated + * memory, (a1 == 0 && a2 == 0) if size was 0 or if memory can't + * be allocated. + * a2 Lower 32bits of 64bit physical pointer to allocated + * memory, (a1 == 0 && a2 == 0) if size was 0 or if memory can't + * be allocated + * a3 Preserved + * a4 Upper 32bits of 64bit Shared memory cookie used when freeing + * the memory or doing an RPC + * a5 Lower 32bits of 64bit Shared memory cookie used when freeing + * the memory or doing an RPC + * a6-7 Preserved + */ +#define OPTEE_SMC_RPC_FUNC_ALLOC 0 +#define OPTEE_SMC_RETURN_RPC_ALLOC \ + OPTEE_SMC_RPC_VAL(OPTEE_SMC_RPC_FUNC_ALLOC) + +/* + * Free memory previously allocated by OPTEE_SMC_RETURN_RPC_ALLOC + * + * "Call" register usage: + * a0 This value, OPTEE_SMC_RETURN_RPC_FREE + * a1 Upper 32bits of 64bit shared memory cookie belonging to this + * argument memory + * a2 Lower 32bits of 64bit shared memory cookie belonging to this + * argument memory + * a3-7 Resume information, must be preserved + * + * "Return" register usage: + * a0 SMC Function ID, OPTEE_SMC_CALL_RETURN_FROM_RPC. + * a1-2 Not used + * a3-7 Preserved + */ +#define OPTEE_SMC_RPC_FUNC_FREE 2 +#define OPTEE_SMC_RETURN_RPC_FREE \ + OPTEE_SMC_RPC_VAL(OPTEE_SMC_RPC_FUNC_FREE) + +/* + * Deliver an IRQ in normal world. + * + * "Call" register usage: + * a0 OPTEE_SMC_RETURN_RPC_IRQ + * a1-7 Resume information, must be preserved + * + * "Return" register usage: + * a0 SMC Function ID, OPTEE_SMC_CALL_RETURN_FROM_RPC. + * a1-7 Preserved + */ +#define OPTEE_SMC_RPC_FUNC_IRQ 4 +#define OPTEE_SMC_RETURN_RPC_IRQ \ + OPTEE_SMC_RPC_VAL(OPTEE_SMC_RPC_FUNC_IRQ) + +/* + * Do an RPC request. The supplied struct optee_msg_arg tells which + * request to do and the parameters for the request. The following fields + * are used (the rest are unused): + * - cmd the Request ID + * - ret return value of the request, filled in by normal world + * - num_params number of parameters for the request + * - params the parameters + * - param_attrs attributes of the parameters + * + * "Call" register usage: + * a0 OPTEE_SMC_RETURN_RPC_CMD + * a1 Upper 32bit of a 64bit Shared memory cookie holding a + * struct optee_msg_arg, must be preserved, only the data should + * be updated + * a2 Lower 32bit of a 64bit Shared memory cookie holding a + * struct optee_msg_arg, must be preserved, only the data should + * be updated + * a3-7 Resume information, must be preserved + * + * "Return" register usage: + * a0 SMC Function ID, OPTEE_SMC_CALL_RETURN_FROM_RPC. + * a1-2 Not used + * a3-7 Preserved + */ +#define OPTEE_SMC_RPC_FUNC_CMD 5 +#define OPTEE_SMC_RETURN_RPC_CMD \ + OPTEE_SMC_RPC_VAL(OPTEE_SMC_RPC_FUNC_CMD) + +/* Returned in a0 */ +#define OPTEE_SMC_RETURN_UNKNOWN_FUNCTION 0xFFFFFFFF + +/* Returned in a0 only from Trusted OS functions */ +#define OPTEE_SMC_RETURN_OK 0x0 +#define OPTEE_SMC_RETURN_ETHREAD_LIMIT 0x1 +#define OPTEE_SMC_RETURN_EBUSY 0x2 +#define OPTEE_SMC_RETURN_ERESUME 0x3 +#define OPTEE_SMC_RETURN_EBADADDR 0x4 +#define OPTEE_SMC_RETURN_EBADCMD 0x5 +#define OPTEE_SMC_RETURN_ENOMEM 0x6 +#define OPTEE_SMC_RETURN_ENOTAVAIL 0x7 +#define OPTEE_SMC_RETURN_IS_RPC(ret) __optee_smc_return_is_rpc((ret)) + +static inline bool __optee_smc_return_is_rpc(u32 ret) +{ + return ret != OPTEE_SMC_RETURN_UNKNOWN_FUNCTION && + (ret & OPTEE_SMC_RETURN_RPC_PREFIX_MASK) == + OPTEE_SMC_RETURN_RPC_PREFIX; +} + +#endif /* OPTEE_SMC_H */ diff --git a/drivers/tee/optee/rpc.c b/drivers/tee/optee/rpc.c new file mode 100644 index 00000000000000..5b58091ab5ab45 --- /dev/null +++ b/drivers/tee/optee/rpc.c @@ -0,0 +1,404 @@ +/* + * Copyright (c) 2015-2016, Linaro Limited + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ +#include +#include +#include +#include +#include "optee_private.h" +#include "optee_smc.h" + +struct wq_entry { + struct list_head link; + struct completion c; + u32 key; +}; + +void optee_wait_queue_init(struct optee_wait_queue *priv) +{ + mutex_init(&priv->mu); + INIT_LIST_HEAD(&priv->db); +} + +void optee_wait_queue_exit(struct optee_wait_queue *priv) +{ + mutex_destroy(&priv->mu); +} + +static void handle_rpc_func_cmd_get_time(struct optee_msg_arg *arg) +{ + struct optee_msg_param *params; + struct timespec64 ts; + + if (arg->num_params != 1) + goto bad; + params = OPTEE_MSG_GET_PARAMS(arg); + if ((params->attr & OPTEE_MSG_ATTR_TYPE_MASK) != + OPTEE_MSG_ATTR_TYPE_VALUE_OUTPUT) + goto bad; + + getnstimeofday64(&ts); + params->u.value.a = ts.tv_sec; + params->u.value.b = ts.tv_nsec; + + arg->ret = TEEC_SUCCESS; + return; +bad: + arg->ret = TEEC_ERROR_BAD_PARAMETERS; +} + +static struct wq_entry *wq_entry_get(struct optee_wait_queue *wq, u32 key) +{ + struct wq_entry *w; + + mutex_lock(&wq->mu); + + list_for_each_entry(w, &wq->db, link) + if (w->key == key) + goto out; + + w = kmalloc(sizeof(*w), GFP_KERNEL); + if (w) { + init_completion(&w->c); + w->key = key; + list_add_tail(&w->link, &wq->db); + } +out: + mutex_unlock(&wq->mu); + return w; +} + +static void wq_sleep(struct optee_wait_queue *wq, u32 key) +{ + struct wq_entry *w = wq_entry_get(wq, key); + + if (w) { + wait_for_completion(&w->c); + mutex_lock(&wq->mu); + list_del(&w->link); + mutex_unlock(&wq->mu); + kfree(w); + } +} + +static void wq_wakeup(struct optee_wait_queue *wq, u32 key) +{ + struct wq_entry *w = wq_entry_get(wq, key); + + if (w) + complete(&w->c); +} + +static void handle_rpc_func_cmd_wq(struct optee *optee, + struct optee_msg_arg *arg) +{ + struct optee_msg_param *params; + + if (arg->num_params != 1) + goto bad; + + params = OPTEE_MSG_GET_PARAMS(arg); + if ((params->attr & OPTEE_MSG_ATTR_TYPE_MASK) != + OPTEE_MSG_ATTR_TYPE_VALUE_INPUT) + goto bad; + + switch (params->u.value.a) { + case OPTEE_MSG_RPC_WAIT_QUEUE_SLEEP: + wq_sleep(&optee->wait_queue, params->u.value.b); + break; + case OPTEE_MSG_RPC_WAIT_QUEUE_WAKEUP: + wq_wakeup(&optee->wait_queue, params->u.value.b); + break; + default: + goto bad; + } + + arg->ret = TEEC_SUCCESS; + return; +bad: + arg->ret = TEEC_ERROR_BAD_PARAMETERS; +} + +static void handle_rpc_func_cmd_wait(struct optee_msg_arg *arg) +{ + struct optee_msg_param *params; + u32 msec_to_wait; + + if (arg->num_params != 1) + goto bad; + + params = OPTEE_MSG_GET_PARAMS(arg); + if ((params->attr & OPTEE_MSG_ATTR_TYPE_MASK) != + OPTEE_MSG_ATTR_TYPE_VALUE_INPUT) + goto bad; + + msec_to_wait = params->u.value.a; + + /* set task's state to interruptible sleep */ + set_current_state(TASK_INTERRUPTIBLE); + + /* take a nap */ + schedule_timeout(msecs_to_jiffies(msec_to_wait)); + + arg->ret = TEEC_SUCCESS; + return; +bad: + arg->ret = TEEC_ERROR_BAD_PARAMETERS; +} + +static void handle_rpc_supp_cmd(struct tee_context *ctx, + struct optee_msg_arg *arg) +{ + struct tee_param *params; + struct optee_msg_param *msg_params = OPTEE_MSG_GET_PARAMS(arg); + + arg->ret_origin = TEEC_ORIGIN_COMMS; + + params = kmalloc_array(arg->num_params, sizeof(struct tee_param), + GFP_KERNEL); + if (!params) { + arg->ret = TEEC_ERROR_OUT_OF_MEMORY; + return; + } + + if (optee_from_msg_param(params, arg->num_params, msg_params)) { + arg->ret = TEEC_ERROR_BAD_PARAMETERS; + goto out; + } + + arg->ret = optee_supp_thrd_req(ctx, arg->cmd, arg->num_params, params); + + if (optee_to_msg_param(msg_params, arg->num_params, params)) + arg->ret = TEEC_ERROR_BAD_PARAMETERS; +out: + kfree(params); +} + +static struct tee_shm *cmd_alloc_suppl(struct tee_context *ctx, size_t sz) +{ + u32 ret; + struct tee_param param; + struct optee *optee = tee_get_drvdata(ctx->teedev); + struct tee_shm *shm; + + param.attr = TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INOUT; + param.u.value.a = OPTEE_MSG_RPC_SHM_TYPE_APPL; + param.u.value.b = sz; + param.u.value.c = 0; + + ret = optee_supp_thrd_req(ctx, OPTEE_MSG_RPC_CMD_SHM_ALLOC, 1, ¶m); + if (ret) + return ERR_PTR(-ENOMEM); + + mutex_lock(&optee->supp.mutex); + /* Increases count as secure world doesn't have a reference */ + shm = tee_shm_get_from_id(optee->supp.ctx, param.u.value.c); + mutex_unlock(&optee->supp.mutex); + return shm; +} + +static void handle_rpc_func_cmd_shm_alloc(struct tee_context *ctx, + struct optee_msg_arg *arg) +{ + struct optee_msg_param *params = OPTEE_MSG_GET_PARAMS(arg); + phys_addr_t pa; + struct tee_shm *shm; + size_t sz; + size_t n; + + arg->ret_origin = TEEC_ORIGIN_COMMS; + + if (!arg->num_params || + params->attr != OPTEE_MSG_ATTR_TYPE_VALUE_INPUT) { + arg->ret = TEEC_ERROR_BAD_PARAMETERS; + return; + } + + for (n = 1; n < arg->num_params; n++) { + if (params[n].attr != OPTEE_MSG_ATTR_TYPE_NONE) { + arg->ret = TEEC_ERROR_BAD_PARAMETERS; + return; + } + } + + sz = params->u.value.b; + switch (params->u.value.a) { + case OPTEE_MSG_RPC_SHM_TYPE_APPL: + shm = cmd_alloc_suppl(ctx, sz); + break; + case OPTEE_MSG_RPC_SHM_TYPE_KERNEL: + shm = tee_shm_alloc(ctx, sz, TEE_SHM_MAPPED); + break; + default: + arg->ret = TEEC_ERROR_BAD_PARAMETERS; + return; + } + + if (IS_ERR(shm)) { + arg->ret = TEEC_ERROR_OUT_OF_MEMORY; + return; + } + + if (tee_shm_get_pa(shm, 0, &pa)) { + arg->ret = TEEC_ERROR_BAD_PARAMETERS; + goto bad; + } + + params[0].attr = OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT; + params[0].u.tmem.buf_ptr = pa; + params[0].u.tmem.size = sz; + params[0].u.tmem.shm_ref = (unsigned long)shm; + arg->ret = TEEC_SUCCESS; + return; +bad: + tee_shm_free(shm); +} + +static void cmd_free_suppl(struct tee_context *ctx, struct tee_shm *shm) +{ + struct tee_param param; + + param.attr = TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INOUT; + param.u.value.a = OPTEE_MSG_RPC_SHM_TYPE_APPL; + param.u.value.b = tee_shm_get_id(shm); + param.u.value.c = 0; + + /* + * Match the tee_shm_get_from_id() in cmd_alloc_suppl() as secure + * world has released its reference. + * + * It's better to do this before sending the request to supplicant + * as we'd like to let the process doing the initial allocation to + * do release the last reference too in order to avoid stacking + * many pending fput() on the client process. This could otherwise + * happen if secure world does many allocate and free in a single + * invoke. + */ + tee_shm_put(shm); + + optee_supp_thrd_req(ctx, OPTEE_MSG_RPC_CMD_SHM_FREE, 1, ¶m); +} + +static void handle_rpc_func_cmd_shm_free(struct tee_context *ctx, + struct optee_msg_arg *arg) +{ + struct optee_msg_param *params = OPTEE_MSG_GET_PARAMS(arg); + struct tee_shm *shm; + + arg->ret_origin = TEEC_ORIGIN_COMMS; + + if (arg->num_params != 1 || + params->attr != OPTEE_MSG_ATTR_TYPE_VALUE_INPUT) { + arg->ret = TEEC_ERROR_BAD_PARAMETERS; + return; + } + + shm = (struct tee_shm *)(unsigned long)params->u.value.b; + switch (params->u.value.a) { + case OPTEE_MSG_RPC_SHM_TYPE_APPL: + cmd_free_suppl(ctx, shm); + break; + case OPTEE_MSG_RPC_SHM_TYPE_KERNEL: + tee_shm_free(shm); + break; + default: + arg->ret = TEEC_ERROR_BAD_PARAMETERS; + } + arg->ret = TEEC_SUCCESS; +} + +static void handle_rpc_func_cmd(struct tee_context *ctx, struct optee *optee, + struct tee_shm *shm) +{ + struct optee_msg_arg *arg; + + arg = tee_shm_get_va(shm, 0); + if (IS_ERR(arg)) { + dev_err(optee->dev, "%s: tee_shm_get_va %p failed\n", + __func__, shm); + return; + } + + switch (arg->cmd) { + case OPTEE_MSG_RPC_CMD_GET_TIME: + handle_rpc_func_cmd_get_time(arg); + break; + case OPTEE_MSG_RPC_CMD_WAIT_QUEUE: + handle_rpc_func_cmd_wq(optee, arg); + break; + case OPTEE_MSG_RPC_CMD_SUSPEND: + handle_rpc_func_cmd_wait(arg); + break; + case OPTEE_MSG_RPC_CMD_SHM_ALLOC: + handle_rpc_func_cmd_shm_alloc(ctx, arg); + break; + case OPTEE_MSG_RPC_CMD_SHM_FREE: + handle_rpc_func_cmd_shm_free(ctx, arg); + break; + default: + handle_rpc_supp_cmd(ctx, arg); + } +} + +/** + * optee_handle_rpc() - handle RPC from secure world + * @ctx: context doing the RPC + * @param: value of registers for the RPC + * + * Result of RPC is written back into @param. + */ +void optee_handle_rpc(struct tee_context *ctx, struct optee_rpc_param *param) +{ + struct tee_device *teedev = ctx->teedev; + struct optee *optee = tee_get_drvdata(teedev); + struct tee_shm *shm; + phys_addr_t pa; + + switch (OPTEE_SMC_RETURN_GET_RPC_FUNC(param->a0)) { + case OPTEE_SMC_RPC_FUNC_ALLOC: + shm = tee_shm_alloc(ctx, param->a1, TEE_SHM_MAPPED); + if (!IS_ERR(shm) && !tee_shm_get_pa(shm, 0, &pa)) { + reg_pair_from_64(¶m->a1, ¶m->a2, pa); + reg_pair_from_64(¶m->a4, ¶m->a5, + (unsigned long)shm); + } else { + param->a1 = 0; + param->a2 = 0; + param->a4 = 0; + param->a5 = 0; + } + break; + case OPTEE_SMC_RPC_FUNC_FREE: + shm = reg_pair_to_ptr(param->a1, param->a2); + tee_shm_free(shm); + break; + case OPTEE_SMC_RPC_FUNC_IRQ: + /* + * An IRQ was raised while secure world was executing, + * since all IRQs are handled in Linux a dummy RPC is + * performed to let Linux take the IRQ through the normal + * vector. + */ + break; + case OPTEE_SMC_RPC_FUNC_CMD: + shm = reg_pair_to_ptr(param->a1, param->a2); + handle_rpc_func_cmd(ctx, optee, shm); + break; + default: + dev_warn(optee->dev, "Unknown RPC func 0x%x\n", + (u32)OPTEE_SMC_RETURN_GET_RPC_FUNC(param->a0)); + break; + } + + param->a0 = OPTEE_SMC_CALL_RETURN_FROM_RPC; +} diff --git a/drivers/tee/optee/supp.c b/drivers/tee/optee/supp.c new file mode 100644 index 00000000000000..df35fc01fd3e5e --- /dev/null +++ b/drivers/tee/optee/supp.c @@ -0,0 +1,382 @@ +/* + * Copyright (c) 2015, Linaro Limited + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ +#include +#include +#include +#include "optee_private.h" + +struct optee_supp_req { + struct list_head link; + + bool busy; + u32 func; + u32 ret; + size_t num_params; + struct tee_param *param; + + struct completion c; +}; + +void optee_supp_init(struct optee_supp *supp) +{ + memset(supp, 0, sizeof(*supp)); + mutex_init(&supp->mutex); + init_completion(&supp->reqs_c); + idr_init(&supp->idr); + INIT_LIST_HEAD(&supp->reqs); + supp->req_id = -1; +} + +void optee_supp_uninit(struct optee_supp *supp) +{ + mutex_destroy(&supp->mutex); + idr_destroy(&supp->idr); +} + +void optee_supp_release(struct optee_supp *supp) +{ + int id; + struct optee_supp_req *req; + struct optee_supp_req *req_tmp; + + mutex_lock(&supp->mutex); + + /* Abort all request retrieved by supplicant */ + idr_for_each_entry(&supp->idr, req, id) { + req->busy = false; + idr_remove(&supp->idr, id); + req->ret = TEEC_ERROR_COMMUNICATION; + complete(&req->c); + } + + /* Abort all queued requests */ + list_for_each_entry_safe(req, req_tmp, &supp->reqs, link) { + list_del(&req->link); + req->ret = TEEC_ERROR_COMMUNICATION; + complete(&req->c); + } + + supp->ctx = NULL; + supp->req_id = -1; + + mutex_unlock(&supp->mutex); +} + +/** + * optee_supp_thrd_req() - request service from supplicant + * @ctx: context doing the request + * @func: function requested + * @num_params: number of elements in @param array + * @param: parameters for function + * + * Returns result of operation to be passed to secure world + */ +u32 optee_supp_thrd_req(struct tee_context *ctx, u32 func, size_t num_params, + struct tee_param *param) + +{ + struct optee *optee = tee_get_drvdata(ctx->teedev); + struct optee_supp *supp = &optee->supp; + struct optee_supp_req *req = kzalloc(sizeof(*req), GFP_KERNEL); + bool interruptable; + u32 ret; + + if (!req) + return TEEC_ERROR_OUT_OF_MEMORY; + + init_completion(&req->c); + req->func = func; + req->num_params = num_params; + req->param = param; + + /* Insert the request in the request list */ + mutex_lock(&supp->mutex); + list_add_tail(&req->link, &supp->reqs); + mutex_unlock(&supp->mutex); + + /* Tell an eventual waiter there's a new request */ + complete(&supp->reqs_c); + + /* + * Wait for supplicant to process and return result, once we've + * returned from wait_for_completion(&req->c) successfully we have + * exclusive access again. + */ + while (wait_for_completion_interruptible(&req->c)) { + mutex_lock(&supp->mutex); + interruptable = !supp->ctx; + if (interruptable) { + /* + * There's no supplicant available and since the + * supp->mutex currently is held none can + * become available until the mutex released + * again. + * + * Interrupting an RPC to supplicant is only + * allowed as a way of slightly improving the user + * experience in case the supplicant hasn't been + * started yet. During normal operation the supplicant + * will serve all requests in a timely manner and + * interrupting then wouldn't make sense. + */ + interruptable = !req->busy; + if (!req->busy) + list_del(&req->link); + } + mutex_unlock(&supp->mutex); + + if (interruptable) { + req->ret = TEEC_ERROR_COMMUNICATION; + break; + } + } + + ret = req->ret; + kfree(req); + + return ret; +} + +static struct optee_supp_req *supp_pop_entry(struct optee_supp *supp, + int num_params, int *id) +{ + struct optee_supp_req *req; + + if (supp->req_id != -1) { + /* + * Supplicant should not mix synchronous and asnynchronous + * requests. + */ + return ERR_PTR(-EINVAL); + } + + if (list_empty(&supp->reqs)) + return NULL; + + req = list_first_entry(&supp->reqs, struct optee_supp_req, link); + + if (num_params < req->num_params) { + /* Not enough room for parameters */ + return ERR_PTR(-EINVAL); + } + + *id = idr_alloc(&supp->idr, req, 1, 0, GFP_KERNEL); + if (*id < 0) + return ERR_PTR(-ENOMEM); + + list_del(&req->link); + req->busy = true; + + return req; +} + +static int supp_check_recv_params(size_t num_params, struct tee_param *params, + size_t *num_meta) +{ + size_t n; + + if (!num_params) + return -EINVAL; + + /* + * If there's memrefs we need to decrease those as they where + * increased earlier and we'll even refuse to accept any below. + */ + for (n = 0; n < num_params; n++) + if (tee_param_is_memref(params + n) && params[n].u.memref.shm) + tee_shm_put(params[n].u.memref.shm); + + /* + * We only expect parameters as TEE_IOCTL_PARAM_ATTR_TYPE_NONE with + * or without the TEE_IOCTL_PARAM_ATTR_META bit set. + */ + for (n = 0; n < num_params; n++) + if (params[n].attr && + params[n].attr != TEE_IOCTL_PARAM_ATTR_META) + return -EINVAL; + + /* At most we'll need one meta parameter so no need to check for more */ + if (params->attr == TEE_IOCTL_PARAM_ATTR_META) + *num_meta = 1; + else + *num_meta = 0; + + return 0; +} + +/** + * optee_supp_recv() - receive request for supplicant + * @ctx: context receiving the request + * @func: requested function in supplicant + * @num_params: number of elements allocated in @param, updated with number + * used elements + * @param: space for parameters for @func + * + * Returns 0 on success or <0 on failure + */ +int optee_supp_recv(struct tee_context *ctx, u32 *func, u32 *num_params, + struct tee_param *param) +{ + struct tee_device *teedev = ctx->teedev; + struct optee *optee = tee_get_drvdata(teedev); + struct optee_supp *supp = &optee->supp; + struct optee_supp_req *req = NULL; + int id; + size_t num_meta; + int rc; + + rc = supp_check_recv_params(*num_params, param, &num_meta); + if (rc) + return rc; + + while (true) { + mutex_lock(&supp->mutex); + req = supp_pop_entry(supp, *num_params - num_meta, &id); + mutex_unlock(&supp->mutex); + + if (req) { + if (IS_ERR(req)) + return PTR_ERR(req); + break; + } + + /* + * If we didn't get a request we'll block in + * wait_for_completion() to avoid needless spinning. + * + * This is where supplicant will be hanging most of + * the time, let's make this interruptable so we + * can easily restart supplicant if needed. + */ + if (wait_for_completion_interruptible(&supp->reqs_c)) + return -ERESTARTSYS; + } + + if (num_meta) { + /* + * tee-supplicant support meta parameters -> requsts can be + * processed asynchronously. + */ + param->attr = TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INOUT | + TEE_IOCTL_PARAM_ATTR_META; + param->u.value.a = id; + param->u.value.b = 0; + param->u.value.c = 0; + } else { + mutex_lock(&supp->mutex); + supp->req_id = id; + mutex_unlock(&supp->mutex); + } + + *func = req->func; + *num_params = req->num_params + num_meta; + memcpy(param + num_meta, req->param, + sizeof(struct tee_param) * req->num_params); + + return 0; +} + +static struct optee_supp_req *supp_pop_req(struct optee_supp *supp, + size_t num_params, + struct tee_param *param, + size_t *num_meta) +{ + struct optee_supp_req *req; + int id; + size_t nm; + const u32 attr = TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INOUT | + TEE_IOCTL_PARAM_ATTR_META; + + if (!num_params) + return ERR_PTR(-EINVAL); + + if (supp->req_id == -1) { + if (param->attr != attr) + return ERR_PTR(-EINVAL); + id = param->u.value.a; + nm = 1; + } else { + id = supp->req_id; + nm = 0; + } + + req = idr_find(&supp->idr, id); + if (!req) + return ERR_PTR(-ENOENT); + + if ((num_params - nm) != req->num_params) + return ERR_PTR(-EINVAL); + + req->busy = false; + idr_remove(&supp->idr, id); + supp->req_id = -1; + *num_meta = nm; + + return req; +} + +/** + * optee_supp_send() - send result of request from supplicant + * @ctx: context sending result + * @ret: return value of request + * @num_params: number of parameters returned + * @param: returned parameters + * + * Returns 0 on success or <0 on failure. + */ +int optee_supp_send(struct tee_context *ctx, u32 ret, u32 num_params, + struct tee_param *param) +{ + struct tee_device *teedev = ctx->teedev; + struct optee *optee = tee_get_drvdata(teedev); + struct optee_supp *supp = &optee->supp; + struct optee_supp_req *req; + size_t n; + size_t num_meta; + + mutex_lock(&supp->mutex); + req = supp_pop_req(supp, num_params, param, &num_meta); + mutex_unlock(&supp->mutex); + + if (IS_ERR(req)) { + /* Something is wrong, let supplicant restart. */ + return PTR_ERR(req); + } + + /* Update out and in/out parameters */ + for (n = 0; n < req->num_params; n++) { + struct tee_param *p = req->param + n; + + switch (p->attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) { + case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_OUTPUT: + case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INOUT: + p->u.value.a = param[n + num_meta].u.value.a; + p->u.value.b = param[n + num_meta].u.value.b; + p->u.value.c = param[n + num_meta].u.value.c; + break; + case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_OUTPUT: + case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INOUT: + p->u.memref.size = param[n + num_meta].u.memref.size; + break; + default: + break; + } + } + req->ret = ret; + + /* Let the requesting thread continue */ + complete(&req->c); + + return 0; +} diff --git a/drivers/tee/tee_core.c b/drivers/tee/tee_core.c new file mode 100644 index 00000000000000..b040e17c743f4d --- /dev/null +++ b/drivers/tee/tee_core.c @@ -0,0 +1,1039 @@ +/* + * Copyright (c) 2015-2016, Linaro Limited + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#define pr_fmt(fmt) "%s: " fmt, __func__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "tee_private.h" + +#define TEE_NUM_DEVICES 32 + +#define TEE_IOCTL_PARAM_SIZE(x) (sizeof(struct tee_param) * (x)) + +/* + * Unprivileged devices in the lower half range and privileged devices in + * the upper half range. + */ +static DECLARE_BITMAP(dev_mask, TEE_NUM_DEVICES); +static DEFINE_SPINLOCK(driver_lock); + +static struct class *tee_class; +static dev_t tee_devt; + +static struct tee_context *teedev_open(struct tee_device *teedev) +{ + int rc; + struct tee_context *ctx; + + if (!tee_device_get(teedev)) + return ERR_PTR(-EINVAL); + + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) { + rc = -ENOMEM; + goto err; + } + + ctx->teedev = teedev; + INIT_LIST_HEAD(&ctx->list_shm); + rc = teedev->desc->ops->open(ctx); + if (rc) + goto err; + + return ctx; +err: + kfree(ctx); + tee_device_put(teedev); + return ERR_PTR(rc); + +} + +static void teedev_close_context(struct tee_context *ctx) +{ + struct tee_shm *shm; + + ctx->teedev->desc->ops->release(ctx); + mutex_lock(&ctx->teedev->mutex); + list_for_each_entry(shm, &ctx->list_shm, link) + shm->ctx = NULL; + mutex_unlock(&ctx->teedev->mutex); + tee_device_put(ctx->teedev); + kfree(ctx); +} + +static int tee_open(struct inode *inode, struct file *filp) +{ + struct tee_context *ctx; + + ctx = teedev_open(container_of(inode->i_cdev, struct tee_device, cdev)); + if (IS_ERR(ctx)) + return PTR_ERR(ctx); + + filp->private_data = ctx; + return 0; +} + +static int tee_release(struct inode *inode, struct file *filp) +{ + teedev_close_context(filp->private_data); + return 0; +} + +static int tee_ioctl_version(struct tee_context *ctx, + struct tee_ioctl_version_data __user *uvers) +{ + struct tee_ioctl_version_data vers; + + ctx->teedev->desc->ops->get_version(ctx->teedev, &vers); + + if (ctx->teedev->desc->flags & TEE_DESC_PRIVILEGED) + vers.gen_caps |= TEE_GEN_CAP_PRIVILEGED; + + if (copy_to_user(uvers, &vers, sizeof(vers))) + return -EFAULT; + + return 0; +} + +static int tee_ioctl_shm_alloc(struct tee_context *ctx, + struct tee_ioctl_shm_alloc_data __user *udata) +{ + long ret; + struct tee_ioctl_shm_alloc_data data; + struct tee_shm *shm; + + if (copy_from_user(&data, udata, sizeof(data))) + return -EFAULT; + + /* Currently no input flags are supported */ + if (data.flags) + return -EINVAL; + + data.id = -1; + + shm = tee_shm_alloc(ctx, data.size, TEE_SHM_MAPPED | TEE_SHM_DMA_BUF); + if (IS_ERR(shm)) + return PTR_ERR(shm); + + data.id = shm->id; + data.flags = shm->flags; + data.size = shm->size; + + if (copy_to_user(udata, &data, sizeof(data))) + ret = -EFAULT; + else + ret = tee_shm_get_fd(shm); + + /* + * When user space closes the file descriptor the shared memory + * should be freed or if tee_shm_get_fd() failed then it will + * be freed immediately. + */ + tee_shm_put(shm); + return ret; +} + +static int tee_ioctl_shm_register_fd(struct tee_context *ctx, + struct tee_ioctl_shm_register_fd_data __user *udata) +{ + struct tee_ioctl_shm_register_fd_data data; + struct tee_shm *shm; + long ret; + + if (copy_from_user(&data, udata, sizeof(data))) + return -EFAULT; + + /* Currently no input flags are supported */ + if (data.flags) + return -EINVAL; + + shm = tee_shm_register_fd(ctx, data.fd); + if (IS_ERR_OR_NULL(shm)) + return -EINVAL; + + data.id = shm->id; + data.flags = shm->flags; + data.size = shm->size; + + if (copy_to_user(udata, &data, sizeof(data))) + ret = -EFAULT; + else + ret = tee_shm_get_fd(shm); + + /* + * When user space closes the file descriptor the shared memory + * should be freed or if tee_shm_get_fd() failed then it will + * be freed immediately. + */ + tee_shm_put(shm); + return ret; +} + +static int params_from_user(struct tee_context *ctx, struct tee_param *params, + size_t num_params, + struct tee_ioctl_param __user *uparams) +{ + size_t n; + + for (n = 0; n < num_params; n++) { + struct tee_shm *shm; + struct tee_ioctl_param ip; + + if (copy_from_user(&ip, uparams + n, sizeof(ip))) + return -EFAULT; + + /* All unused attribute bits has to be zero */ + if (ip.attr & ~TEE_IOCTL_PARAM_ATTR_MASK) + return -EINVAL; + + params[n].attr = ip.attr; + switch (ip.attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) { + case TEE_IOCTL_PARAM_ATTR_TYPE_NONE: + case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_OUTPUT: + break; + case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT: + case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INOUT: + params[n].u.value.a = ip.u.value.a; + params[n].u.value.b = ip.u.value.b; + params[n].u.value.c = ip.u.value.c; + break; + case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT: + case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_OUTPUT: + case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INOUT: + /* + * If we fail to get a pointer to a shared memory + * object (and increase the ref count) from an + * identifier we return an error. All pointers that + * has been added in params have an increased ref + * count. It's the callers responibility to do + * tee_shm_put() on all resolved pointers. + */ + shm = tee_shm_get_from_id(ctx, ip.u.memref.shm_id); + if (IS_ERR(shm)) + return PTR_ERR(shm); + + params[n].u.memref.shm_offs = ip.u.memref.shm_offs; + params[n].u.memref.size = ip.u.memref.size; + params[n].u.memref.shm = shm; + break; + default: + /* Unknown attribute */ + return -EINVAL; + } + } + return 0; +} + +static int params_to_user(struct tee_ioctl_param __user *uparams, + size_t num_params, struct tee_param *params) +{ + size_t n; + + for (n = 0; n < num_params; n++) { + struct tee_ioctl_param __user *up = uparams + n; + struct tee_param *p = params + n; + + switch (p->attr) { + case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_OUTPUT: + case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INOUT: + if (put_user(p->u.value.a, &up->u.value.a) || + put_user(p->u.value.b, &up->u.value.b) || + put_user(p->u.value.c, &up->u.value.c)) + return -EFAULT; + break; + case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_OUTPUT: + case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INOUT: + if (put_user((u64)p->u.memref.size, &up->u.memref.size)) + return -EFAULT; + default: + break; + } + } + return 0; +} + +static int tee_ioctl_open_session(struct tee_context *ctx, + struct tee_ioctl_buf_data __user *ubuf) +{ + int rc; + size_t n; + struct tee_ioctl_buf_data buf; + struct tee_ioctl_open_session_arg __user *uarg; + struct tee_ioctl_open_session_arg arg; + struct tee_ioctl_param __user *uparams = NULL; + struct tee_param *params = NULL; + bool have_session = false; + + if (!ctx->teedev->desc->ops->open_session) + return -EINVAL; + + if (copy_from_user(&buf, ubuf, sizeof(buf))) + return -EFAULT; + + if (buf.buf_len > TEE_MAX_ARG_SIZE || + buf.buf_len < sizeof(struct tee_ioctl_open_session_arg)) + return -EINVAL; + + uarg = (struct tee_ioctl_open_session_arg __user *)(unsigned long) + buf.buf_ptr; + if (copy_from_user(&arg, uarg, sizeof(arg))) + return -EFAULT; + + if (sizeof(arg) + TEE_IOCTL_PARAM_SIZE(arg.num_params) != buf.buf_len) + return -EINVAL; + + if (arg.num_params) { + params = kcalloc(arg.num_params, sizeof(struct tee_param), + GFP_KERNEL); + if (!params) + return -ENOMEM; + uparams = (struct tee_ioctl_param __user *)(uarg + 1); + rc = params_from_user(ctx, params, arg.num_params, uparams); + if (rc) + goto out; + } + + rc = ctx->teedev->desc->ops->open_session(ctx, &arg, params); + if (rc) + goto out; + have_session = true; + + if (put_user(arg.session, &uarg->session) || + put_user(arg.ret, &uarg->ret) || + put_user(arg.ret_origin, &uarg->ret_origin)) { + rc = -EFAULT; + goto out; + } + rc = params_to_user(uparams, arg.num_params, params); +out: + /* + * If we've succeeded to open the session but failed to communicate + * it back to user space, close the session again to avoid leakage. + */ + if (rc && have_session && ctx->teedev->desc->ops->close_session) + ctx->teedev->desc->ops->close_session(ctx, arg.session); + + if (params) { + /* Decrease ref count for all valid shared memory pointers */ + for (n = 0; n < arg.num_params; n++) + if (tee_param_is_memref(params + n) && + params[n].u.memref.shm) + tee_shm_put(params[n].u.memref.shm); + kfree(params); + } + + return rc; +} + +static int tee_ioctl_invoke(struct tee_context *ctx, + struct tee_ioctl_buf_data __user *ubuf) +{ + int rc; + size_t n; + struct tee_ioctl_buf_data buf; + struct tee_ioctl_invoke_arg __user *uarg; + struct tee_ioctl_invoke_arg arg; + struct tee_ioctl_param __user *uparams = NULL; + struct tee_param *params = NULL; + + if (!ctx->teedev->desc->ops->invoke_func) + return -EINVAL; + + if (copy_from_user(&buf, ubuf, sizeof(buf))) + return -EFAULT; + + if (buf.buf_len > TEE_MAX_ARG_SIZE || + buf.buf_len < sizeof(struct tee_ioctl_invoke_arg)) + return -EINVAL; + + uarg = (struct tee_ioctl_invoke_arg __user *)(unsigned long)buf.buf_ptr; + if (copy_from_user(&arg, uarg, sizeof(arg))) + return -EFAULT; + + if (sizeof(arg) + TEE_IOCTL_PARAM_SIZE(arg.num_params) != buf.buf_len) + return -EINVAL; + + if (arg.num_params) { + params = kcalloc(arg.num_params, sizeof(struct tee_param), + GFP_KERNEL); + if (!params) + return -ENOMEM; + uparams = (struct tee_ioctl_param __user *)(uarg + 1); + rc = params_from_user(ctx, params, arg.num_params, uparams); + if (rc) + goto out; + } + + rc = ctx->teedev->desc->ops->invoke_func(ctx, &arg, params); + if (rc) + goto out; + + if (put_user(arg.ret, &uarg->ret) || + put_user(arg.ret_origin, &uarg->ret_origin)) { + rc = -EFAULT; + goto out; + } + rc = params_to_user(uparams, arg.num_params, params); +out: + if (params) { + /* Decrease ref count for all valid shared memory pointers */ + for (n = 0; n < arg.num_params; n++) + if (tee_param_is_memref(params + n) && + params[n].u.memref.shm) + tee_shm_put(params[n].u.memref.shm); + kfree(params); + } + return rc; +} + +static int tee_ioctl_cancel(struct tee_context *ctx, + struct tee_ioctl_cancel_arg __user *uarg) +{ + struct tee_ioctl_cancel_arg arg; + + if (!ctx->teedev->desc->ops->cancel_req) + return -EINVAL; + + if (copy_from_user(&arg, uarg, sizeof(arg))) + return -EFAULT; + + return ctx->teedev->desc->ops->cancel_req(ctx, arg.cancel_id, + arg.session); +} + +static int +tee_ioctl_close_session(struct tee_context *ctx, + struct tee_ioctl_close_session_arg __user *uarg) +{ + struct tee_ioctl_close_session_arg arg; + + if (!ctx->teedev->desc->ops->close_session) + return -EINVAL; + + if (copy_from_user(&arg, uarg, sizeof(arg))) + return -EFAULT; + + return ctx->teedev->desc->ops->close_session(ctx, arg.session); +} + +static int params_to_supp(struct tee_context *ctx, + struct tee_ioctl_param __user *uparams, + size_t num_params, struct tee_param *params) +{ + size_t n; + + for (n = 0; n < num_params; n++) { + struct tee_ioctl_param ip; + struct tee_param *p = params + n; + + ip.attr = p->attr; + switch (p->attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) { + case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT: + case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INOUT: + ip.u.value.a = p->u.value.a; + ip.u.value.b = p->u.value.b; + ip.u.value.c = p->u.value.c; + break; + case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT: + case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_OUTPUT: + case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INOUT: + ip.u.memref.size = p->u.memref.size; + if (!p->u.memref.shm) { + ip.u.memref.shm_offs = 0; + ip.u.memref.shm_id = -1; + break; + } + ip.u.memref.shm_offs = p->u.memref.shm_offs; + ip.u.memref.shm_id = p->u.memref.shm->id; + break; + default: + memset(&ip.u, 0, sizeof(ip.u)); + break; + } + + if (copy_to_user(uparams + n, &ip, sizeof(ip))) + return -EFAULT; + } + + return 0; +} + +static int tee_ioctl_supp_recv(struct tee_context *ctx, + struct tee_ioctl_buf_data __user *ubuf) +{ + int rc; + struct tee_ioctl_buf_data buf; + struct tee_iocl_supp_recv_arg __user *uarg; + struct tee_param *params; + struct tee_ioctl_param __user *uparams; + u32 num_params; + u32 func; + + if (!ctx->teedev->desc->ops->supp_recv) + return -EINVAL; + + if (copy_from_user(&buf, ubuf, sizeof(buf))) + return -EFAULT; + + if (buf.buf_len > TEE_MAX_ARG_SIZE || + buf.buf_len < sizeof(struct tee_iocl_supp_recv_arg)) + return -EINVAL; + + uarg = (struct tee_iocl_supp_recv_arg __user *)(unsigned long) + buf.buf_ptr; + if (get_user(num_params, &uarg->num_params)) + return -EFAULT; + + if (sizeof(*uarg) + TEE_IOCTL_PARAM_SIZE(num_params) != buf.buf_len) + return -EINVAL; + uparams = (struct tee_ioctl_param __user *)(uarg + 1); + + params = kcalloc(num_params, sizeof(struct tee_param), GFP_KERNEL); + if (!params) + return -ENOMEM; + + rc = params_from_user(ctx, params, num_params, uparams); + if (rc) + goto out; + + rc = ctx->teedev->desc->ops->supp_recv(ctx, &func, &num_params, params); + if (rc) + goto out; + + if (put_user(func, &uarg->func) || + put_user(num_params, &uarg->num_params)) { + rc = -EFAULT; + goto out; + } + + rc = params_to_supp(ctx, uparams, num_params, params); +out: + kfree(params); + return rc; +} + +static int params_from_supp(struct tee_param *params, size_t num_params, + struct tee_ioctl_param __user *uparams) +{ + size_t n; + + for (n = 0; n < num_params; n++) { + struct tee_param *p = params + n; + struct tee_ioctl_param ip; + + if (copy_from_user(&ip, uparams + n, sizeof(ip))) + return -EFAULT; + + /* All unused attribute bits has to be zero */ + if (ip.attr & ~TEE_IOCTL_PARAM_ATTR_MASK) + return -EINVAL; + + p->attr = ip.attr; + switch (ip.attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) { + case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_OUTPUT: + case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INOUT: + /* Only out and in/out values can be updated */ + p->u.value.a = ip.u.value.a; + p->u.value.b = ip.u.value.b; + p->u.value.c = ip.u.value.c; + break; + case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_OUTPUT: + case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INOUT: + /* + * Only the size of the memref can be updated. + * Since we don't have access to the original + * parameters here, only store the supplied size. + * The driver will copy the updated size into the + * original parameters. + */ + p->u.memref.shm = NULL; + p->u.memref.shm_offs = 0; + p->u.memref.size = ip.u.memref.size; + break; + default: + memset(&p->u, 0, sizeof(p->u)); + break; + } + } + return 0; +} + +static int tee_ioctl_supp_send(struct tee_context *ctx, + struct tee_ioctl_buf_data __user *ubuf) +{ + long rc; + struct tee_ioctl_buf_data buf; + struct tee_iocl_supp_send_arg __user *uarg; + struct tee_param *params; + struct tee_ioctl_param __user *uparams; + u32 num_params; + u32 ret; + + /* Not valid for this driver */ + if (!ctx->teedev->desc->ops->supp_send) + return -EINVAL; + + if (copy_from_user(&buf, ubuf, sizeof(buf))) + return -EFAULT; + + if (buf.buf_len > TEE_MAX_ARG_SIZE || + buf.buf_len < sizeof(struct tee_iocl_supp_send_arg)) + return -EINVAL; + + uarg = (struct tee_iocl_supp_send_arg __user *)(unsigned long) + buf.buf_ptr; + if (get_user(ret, &uarg->ret) || + get_user(num_params, &uarg->num_params)) + return -EFAULT; + + if (sizeof(*uarg) + TEE_IOCTL_PARAM_SIZE(num_params) > buf.buf_len) + return -EINVAL; + + params = kcalloc(num_params, sizeof(struct tee_param), GFP_KERNEL); + if (!params) + return -ENOMEM; + + uparams = (struct tee_ioctl_param __user *)(uarg + 1); + rc = params_from_supp(params, num_params, uparams); + if (rc) + goto out; + + rc = ctx->teedev->desc->ops->supp_send(ctx, ret, num_params, params); +out: + kfree(params); + return rc; +} + +static long tee_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + struct tee_context *ctx = filp->private_data; + void __user *uarg = (void __user *)arg; + + switch (cmd) { + case TEE_IOC_VERSION: + return tee_ioctl_version(ctx, uarg); + case TEE_IOC_SHM_ALLOC: + return tee_ioctl_shm_alloc(ctx, uarg); + case TEE_IOC_SHM_REGISTER_FD: + return tee_ioctl_shm_register_fd(ctx, uarg); + case TEE_IOC_OPEN_SESSION: + return tee_ioctl_open_session(ctx, uarg); + case TEE_IOC_INVOKE: + return tee_ioctl_invoke(ctx, uarg); + case TEE_IOC_CANCEL: + return tee_ioctl_cancel(ctx, uarg); + case TEE_IOC_CLOSE_SESSION: + return tee_ioctl_close_session(ctx, uarg); + case TEE_IOC_SUPPL_RECV: + return tee_ioctl_supp_recv(ctx, uarg); + case TEE_IOC_SUPPL_SEND: + return tee_ioctl_supp_send(ctx, uarg); + default: + return -EINVAL; + } +} + +static const struct file_operations tee_fops = { + .owner = THIS_MODULE, + .open = tee_open, + .release = tee_release, + .unlocked_ioctl = tee_ioctl, + .compat_ioctl = tee_ioctl, +}; + +static void tee_release_device(struct device *dev) +{ + struct tee_device *teedev = container_of(dev, struct tee_device, dev); + + spin_lock(&driver_lock); + clear_bit(teedev->id, dev_mask); + spin_unlock(&driver_lock); + mutex_destroy(&teedev->mutex); + kfree(teedev); +} + +/** + * tee_device_alloc() - Allocate a new struct tee_device instance + * @teedesc: Descriptor for this driver + * @dev: Parent device for this device + * @pool: Shared memory pool, NULL if not used + * @driver_data: Private driver data for this device + * + * Allocates a new struct tee_device instance. The device is + * removed by tee_device_unregister(). + * + * @returns a pointer to a 'struct tee_device' or an ERR_PTR on failure + */ +struct tee_device *tee_device_alloc(const struct tee_desc *teedesc, + struct device *dev, + struct tee_shm_pool *pool, + void *driver_data) +{ + struct tee_device *teedev; + void *ret; + int rc; + int offs = 0; + + if (!teedesc || !teedesc->name || !teedesc->ops || + !teedesc->ops->get_version || !teedesc->ops->open || + !teedesc->ops->release || !dev || !pool) + return ERR_PTR(-EINVAL); + + teedev = kzalloc(sizeof(*teedev), GFP_KERNEL); + if (!teedev) { + ret = ERR_PTR(-ENOMEM); + goto err; + } + + if (teedesc->flags & TEE_DESC_PRIVILEGED) + offs = TEE_NUM_DEVICES / 2; + + spin_lock(&driver_lock); + teedev->id = find_next_zero_bit(dev_mask, TEE_NUM_DEVICES, offs); + if (teedev->id < TEE_NUM_DEVICES) + set_bit(teedev->id, dev_mask); + spin_unlock(&driver_lock); + + if (teedev->id >= TEE_NUM_DEVICES) { + ret = ERR_PTR(-ENOMEM); + goto err; + } + + snprintf(teedev->name, sizeof(teedev->name), "tee%s%d", + teedesc->flags & TEE_DESC_PRIVILEGED ? "priv" : "", + teedev->id - offs); + + teedev->dev.class = tee_class; + teedev->dev.release = tee_release_device; + teedev->dev.parent = dev; + + teedev->dev.devt = MKDEV(MAJOR(tee_devt), teedev->id); + + rc = dev_set_name(&teedev->dev, "%s", teedev->name); + if (rc) { + ret = ERR_PTR(rc); + goto err_devt; + } + + cdev_init(&teedev->cdev, &tee_fops); + teedev->cdev.owner = teedesc->owner; + teedev->cdev.kobj.parent = &teedev->dev.kobj; + + dev_set_drvdata(&teedev->dev, driver_data); + device_initialize(&teedev->dev); + + /* 1 as tee_device_unregister() does one final tee_device_put() */ + teedev->num_users = 1; + init_completion(&teedev->c_no_users); + mutex_init(&teedev->mutex); + + teedev->desc = teedesc; + teedev->pool = pool; + + return teedev; +err_devt: + unregister_chrdev_region(teedev->dev.devt, 1); +err: + dev_err(dev, "could not register %s driver\n", + teedesc->flags & TEE_DESC_PRIVILEGED ? "privileged" : "client"); + if (teedev && teedev->id < TEE_NUM_DEVICES) { + spin_lock(&driver_lock); + clear_bit(teedev->id, dev_mask); + spin_unlock(&driver_lock); + } + kfree(teedev); + return ret; +} +EXPORT_SYMBOL_GPL(tee_device_alloc); + +static ssize_t implementation_id_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct tee_device *teedev = container_of(dev, struct tee_device, dev); + struct tee_ioctl_version_data vers; + + teedev->desc->ops->get_version(teedev, &vers); + return scnprintf(buf, PAGE_SIZE, "%d\n", vers.impl_id); +} +static DEVICE_ATTR_RO(implementation_id); + +static struct attribute *tee_dev_attrs[] = { + &dev_attr_implementation_id.attr, + NULL +}; + +static const struct attribute_group tee_dev_group = { + .attrs = tee_dev_attrs, +}; + +/** + * tee_device_register() - Registers a TEE device + * @teedev: Device to register + * + * tee_device_unregister() need to be called to remove the @teedev if + * this function fails. + * + * @returns < 0 on failure + */ +int tee_device_register(struct tee_device *teedev) +{ + int rc; + + /* + * If the teedev already is registered, don't do it again. It's + * obviously an error to try to register twice, but if we return + * an error we'll force the driver to remove the teedev. + */ + if (teedev->flags & TEE_DEVICE_FLAG_REGISTERED) { + dev_err(&teedev->dev, "attempt to register twice\n"); + return 0; + } + + rc = cdev_add(&teedev->cdev, teedev->dev.devt, 1); + if (rc) { + dev_err(&teedev->dev, + "unable to cdev_add() %s, major %d, minor %d, err=%d\n", + teedev->name, MAJOR(teedev->dev.devt), + MINOR(teedev->dev.devt), rc); + return rc; + } + + rc = device_add(&teedev->dev); + if (rc) { + dev_err(&teedev->dev, + "unable to device_add() %s, major %d, minor %d, err=%d\n", + teedev->name, MAJOR(teedev->dev.devt), + MINOR(teedev->dev.devt), rc); + goto err_device_add; + } + + rc = sysfs_create_group(&teedev->dev.kobj, &tee_dev_group); + if (rc) { + dev_err(&teedev->dev, + "failed to create sysfs attributes, err=%d\n", rc); + goto err_sysfs_create_group; + } + + teedev->flags |= TEE_DEVICE_FLAG_REGISTERED; + return 0; + +err_sysfs_create_group: + device_del(&teedev->dev); +err_device_add: + cdev_del(&teedev->cdev); + return rc; +} +EXPORT_SYMBOL_GPL(tee_device_register); + +void tee_device_put(struct tee_device *teedev) +{ + mutex_lock(&teedev->mutex); + /* Shouldn't put in this state */ + if (!WARN_ON(!teedev->desc)) { + teedev->num_users--; + if (!teedev->num_users) { + teedev->desc = NULL; + complete(&teedev->c_no_users); + } + } + mutex_unlock(&teedev->mutex); +} + +bool tee_device_get(struct tee_device *teedev) +{ + mutex_lock(&teedev->mutex); + if (!teedev->desc) { + mutex_unlock(&teedev->mutex); + return false; + } + teedev->num_users++; + mutex_unlock(&teedev->mutex); + return true; +} + +/** + * tee_device_unregister() - Removes a TEE device + * @teedev: Device to unregister + * + * This function should be called to remove the @teedev even if + * tee_device_register() hasn't been called yet. Does nothing if + * @teedev is NULL. + */ +void tee_device_unregister(struct tee_device *teedev) +{ + if (!teedev) + return; + + if (teedev->flags & TEE_DEVICE_FLAG_REGISTERED) { + sysfs_remove_group(&teedev->dev.kobj, &tee_dev_group); + cdev_del(&teedev->cdev); + device_del(&teedev->dev); + } + + tee_device_put(teedev); + wait_for_completion(&teedev->c_no_users); + + /* + * No need to take a mutex any longer now since teedev->desc was + * set to NULL before teedev->c_no_users was completed. + */ + + teedev->pool = NULL; + + put_device(&teedev->dev); +} +EXPORT_SYMBOL_GPL(tee_device_unregister); + +/** + * tee_get_drvdata() - Return driver_data pointer + * @teedev: Device containing the driver_data pointer + * @returns the driver_data pointer supplied to tee_register(). + */ +void *tee_get_drvdata(struct tee_device *teedev) +{ + return dev_get_drvdata(&teedev->dev); +} +EXPORT_SYMBOL_GPL(tee_get_drvdata); + +struct match_dev_data { + struct tee_ioctl_version_data *vers; + const void *data; + int (*match)(struct tee_ioctl_version_data *, const void *); +}; + +static int match_dev(struct device *dev, const void *data) +{ + const struct match_dev_data *match_data = data; + struct tee_device *teedev = container_of(dev, struct tee_device, dev); + + teedev->desc->ops->get_version(teedev, match_data->vers); + return match_data->match(match_data->vers, match_data->data); +} + +struct tee_context *tee_client_open_context(struct tee_context *start, + int (*match)(struct tee_ioctl_version_data *, + const void *), + const void *data, struct tee_ioctl_version_data *vers) +{ + struct device *dev = NULL; + struct device *put_dev = NULL; + struct tee_context *ctx = NULL; + struct tee_ioctl_version_data v; + struct match_dev_data match_data = { vers ? vers : &v, data, match }; + + if (start) + dev = &start->teedev->dev; + + do { + dev = class_find_device(tee_class, dev, &match_data, match_dev); + if (!dev) { + ctx = ERR_PTR(-ENOENT); + break; + } + + put_device(put_dev); + put_dev = dev; + + ctx = teedev_open(container_of(dev, struct tee_device, dev)); + } while (IS_ERR(ctx) && PTR_ERR(ctx) != -ENOMEM); + + put_device(put_dev); + return ctx; +} +EXPORT_SYMBOL_GPL(tee_client_open_context); + +void tee_client_close_context(struct tee_context *ctx) +{ + teedev_close_context(ctx); +} + +EXPORT_SYMBOL_GPL(tee_client_close_context); + +void tee_client_get_version(struct tee_context *ctx, + struct tee_ioctl_version_data *vers) +{ + ctx->teedev->desc->ops->get_version(ctx->teedev, vers); +} +EXPORT_SYMBOL_GPL(tee_client_get_version); + + +int tee_client_open_session(struct tee_context *ctx, + struct tee_ioctl_open_session_arg *arg, + struct tee_param *param) +{ + if (!ctx->teedev->desc->ops->open_session) + return -EINVAL; + return ctx->teedev->desc->ops->open_session(ctx, arg, param); +} +EXPORT_SYMBOL_GPL(tee_client_open_session); + +int tee_client_close_session(struct tee_context *ctx, u32 session) +{ + if (!ctx->teedev->desc->ops->close_session) + return -EINVAL; + return ctx->teedev->desc->ops->close_session(ctx, session); +} +EXPORT_SYMBOL_GPL(tee_client_close_session); + +int tee_client_invoke_func(struct tee_context *ctx, + struct tee_ioctl_invoke_arg *arg, + struct tee_param *param) +{ + if (!ctx->teedev->desc->ops->invoke_func) + return -EINVAL; + return ctx->teedev->desc->ops->invoke_func(ctx, arg, param); +} +EXPORT_SYMBOL_GPL(tee_client_invoke_func); + +static int __init tee_init(void) +{ + int rc; + + tee_class = class_create(THIS_MODULE, "tee"); + if (IS_ERR(tee_class)) { + pr_err("couldn't create class\n"); + return PTR_ERR(tee_class); + } + + rc = alloc_chrdev_region(&tee_devt, 0, TEE_NUM_DEVICES, "tee"); + if (rc) { + pr_err("failed to allocate char dev region\n"); + class_destroy(tee_class); + tee_class = NULL; + } + + return rc; +} + +static void __exit tee_exit(void) +{ + class_destroy(tee_class); + tee_class = NULL; + unregister_chrdev_region(tee_devt, TEE_NUM_DEVICES); +} + +subsys_initcall(tee_init); +module_exit(tee_exit); + +MODULE_AUTHOR("Linaro"); +MODULE_DESCRIPTION("TEE Driver"); +MODULE_VERSION("1.0"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/tee/tee_private.h b/drivers/tee/tee_private.h new file mode 100644 index 00000000000000..21cb6be8bce94d --- /dev/null +++ b/drivers/tee/tee_private.h @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2015-2016, Linaro Limited + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ +#ifndef TEE_PRIVATE_H +#define TEE_PRIVATE_H + +#include +#include +#include +#include +#include +#include + +struct tee_device; + +/** + * struct tee_shm - shared memory object + * @teedev: device used to allocate the object + * @ctx: context using the object, if NULL the context is gone + * @link link element + * @paddr: physical address of the shared memory + * @kaddr: virtual address of the shared memory + * @size: size of shared memory + * @dmabuf: dmabuf used to for exporting to user space + * @flags: defined by TEE_SHM_* in tee_drv.h + * @id: unique id of a shared memory object on this device + */ +struct tee_shm { + struct tee_device *teedev; + struct tee_context *ctx; + struct list_head link; + phys_addr_t paddr; + void *kaddr; + size_t size; + struct dma_buf *dmabuf; + u32 flags; + int id; +}; + +struct tee_shm_pool_mgr; + +/** + * struct tee_shm_pool_mgr_ops - shared memory pool manager operations + * @alloc: called when allocating shared memory + * @free: called when freeing shared memory + */ +struct tee_shm_pool_mgr_ops { + int (*alloc)(struct tee_shm_pool_mgr *poolmgr, struct tee_shm *shm, + size_t size); + void (*free)(struct tee_shm_pool_mgr *poolmgr, struct tee_shm *shm); +}; + +/** + * struct tee_shm_pool_mgr - shared memory manager + * @ops: operations + * @private_data: private data for the shared memory manager + */ +struct tee_shm_pool_mgr { + const struct tee_shm_pool_mgr_ops *ops; + void *private_data; +}; + +/** + * struct tee_shm_pool - shared memory pool + * @private_mgr: pool manager for shared memory only between kernel + * and secure world + * @dma_buf_mgr: pool manager for shared memory exported to user space + * @destroy: called when destroying the pool + * @private_data: private data for the pool + */ +struct tee_shm_pool { + struct tee_shm_pool_mgr private_mgr; + struct tee_shm_pool_mgr dma_buf_mgr; + void (*destroy)(struct tee_shm_pool *pool); + void *private_data; +}; + +#define TEE_DEVICE_FLAG_REGISTERED 0x1 +#define TEE_MAX_DEV_NAME_LEN 32 + +/** + * struct tee_device - TEE Device representation + * @name: name of device + * @desc: description of device + * @id: unique id of device + * @flags: represented by TEE_DEVICE_FLAG_REGISTERED above + * @dev: embedded basic device structure + * @cdev: embedded cdev + * @num_users: number of active users of this device + * @c_no_user: completion used when unregistering the device + * @mutex: mutex protecting @num_users and @idr + * @idr: register of shared memory object allocated on this device + * @pool: shared memory pool + */ +struct tee_device { + char name[TEE_MAX_DEV_NAME_LEN]; + const struct tee_desc *desc; + int id; + unsigned int flags; + + struct device dev; + struct cdev cdev; + + size_t num_users; + struct completion c_no_users; + struct mutex mutex; /* protects num_users and idr */ + + struct idr idr; + struct tee_shm_pool *pool; +}; + +int tee_shm_init(void); + +int tee_shm_get_fd(struct tee_shm *shm); + +bool tee_device_get(struct tee_device *teedev); +void tee_device_put(struct tee_device *teedev); + +#endif /*TEE_PRIVATE_H*/ diff --git a/drivers/tee/tee_shm.c b/drivers/tee/tee_shm.c new file mode 100644 index 00000000000000..6a814968bfa3ea --- /dev/null +++ b/drivers/tee/tee_shm.c @@ -0,0 +1,475 @@ +/* + * Copyright (c) 2015-2016, Linaro Limited + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ +#include +#include +#include +#include +#include +#include +#include +#include "tee_private.h" + +/* extra references appended to shm object for registered shared memory */ +struct tee_shm_dmabuf_ref { + struct tee_shm shm; + struct dma_buf *dmabuf; + struct dma_buf_attachment *attach; + struct sg_table *sgt; +}; + +static void tee_shm_release(struct tee_shm *shm) +{ + struct tee_device *teedev = shm->teedev; + + mutex_lock(&teedev->mutex); + idr_remove(&teedev->idr, shm->id); + if (shm->ctx) + list_del(&shm->link); + mutex_unlock(&teedev->mutex); + + if (shm->flags & TEE_SHM_EXT_DMA_BUF) { + struct tee_shm_dmabuf_ref *ref; + + ref = container_of(shm, struct tee_shm_dmabuf_ref, shm); + dma_buf_unmap_attachment(ref->attach, ref->sgt, + DMA_BIDIRECTIONAL); + dma_buf_detach(shm->dmabuf, ref->attach); + dma_buf_put(ref->dmabuf); + } else { + struct tee_shm_pool_mgr *poolm; + + if (shm->flags & TEE_SHM_DMA_BUF) + poolm = &teedev->pool->dma_buf_mgr; + else + poolm = &teedev->pool->private_mgr; + + poolm->ops->free(poolm, shm); + } + + kfree(shm); + tee_device_put(teedev); +} + +static struct sg_table *tee_shm_op_map_dma_buf(struct dma_buf_attachment + *attach, enum dma_data_direction dir) +{ + return NULL; +} + +static void tee_shm_op_unmap_dma_buf(struct dma_buf_attachment *attach, + struct sg_table *table, + enum dma_data_direction dir) +{ +} + +static void tee_shm_op_release(struct dma_buf *dmabuf) +{ + struct tee_shm *shm = dmabuf->priv; + + tee_shm_release(shm); +} + +static void *tee_shm_op_kmap_atomic(struct dma_buf *dmabuf, unsigned long pgnum) +{ + return NULL; +} + +static void *tee_shm_op_kmap(struct dma_buf *dmabuf, unsigned long pgnum) +{ + return NULL; +} + +static int tee_shm_op_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) +{ + struct tee_shm *shm = dmabuf->priv; + size_t size = vma->vm_end - vma->vm_start; + + return remap_pfn_range(vma, vma->vm_start, shm->paddr >> PAGE_SHIFT, + size, vma->vm_page_prot); +} + +static struct dma_buf_ops tee_shm_dma_buf_ops = { + .map_dma_buf = tee_shm_op_map_dma_buf, + .unmap_dma_buf = tee_shm_op_unmap_dma_buf, + .release = tee_shm_op_release, + .kmap_atomic = tee_shm_op_kmap_atomic, + .kmap = tee_shm_op_kmap, + .mmap = tee_shm_op_mmap, +}; + +/** + * tee_shm_alloc() - Allocate shared memory + * @ctx: Context that allocates the shared memory + * @size: Requested size of shared memory + * @flags: Flags setting properties for the requested shared memory. + * + * Memory allocated as global shared memory is automatically freed when the + * TEE file pointer is closed. The @flags field uses the bits defined by + * TEE_SHM_* in . TEE_SHM_MAPPED must currently always be + * set. If TEE_SHM_DMA_BUF global shared memory will be allocated and + * associated with a dma-buf handle, else driver private memory. + */ +struct tee_shm *tee_shm_alloc(struct tee_context *ctx, size_t size, u32 flags) +{ + struct tee_device *teedev = ctx->teedev; + struct tee_shm_pool_mgr *poolm = NULL; + struct tee_shm *shm; + void *ret; + int rc; + + if (!(flags & TEE_SHM_MAPPED)) { + dev_err(teedev->dev.parent, + "only mapped allocations supported\n"); + return ERR_PTR(-EINVAL); + } + + if ((flags & ~(TEE_SHM_MAPPED | TEE_SHM_DMA_BUF))) { + dev_err(teedev->dev.parent, "invalid shm flags 0x%x", flags); + return ERR_PTR(-EINVAL); + } + + if (!tee_device_get(teedev)) + return ERR_PTR(-EINVAL); + + if (!teedev->pool) { + /* teedev has been detached from driver */ + ret = ERR_PTR(-EINVAL); + goto err_dev_put; + } + + shm = kzalloc(sizeof(*shm), GFP_KERNEL); + if (!shm) { + ret = ERR_PTR(-ENOMEM); + goto err_dev_put; + } + + shm->flags = flags; + shm->teedev = teedev; + shm->ctx = ctx; + if (flags & TEE_SHM_DMA_BUF) + poolm = &teedev->pool->dma_buf_mgr; + else + poolm = &teedev->pool->private_mgr; + + rc = poolm->ops->alloc(poolm, shm, size); + if (rc) { + ret = ERR_PTR(rc); + goto err_kfree; + } + + mutex_lock(&teedev->mutex); + shm->id = idr_alloc(&teedev->idr, shm, 1, 0, GFP_KERNEL); + mutex_unlock(&teedev->mutex); + if (shm->id < 0) { + ret = ERR_PTR(shm->id); + goto err_pool_free; + } + + if (flags & TEE_SHM_DMA_BUF) { + DEFINE_DMA_BUF_EXPORT_INFO(exp_info); + + exp_info.ops = &tee_shm_dma_buf_ops; + exp_info.size = shm->size; + exp_info.flags = O_RDWR; + exp_info.priv = shm; + + shm->dmabuf = dma_buf_export(&exp_info); + if (IS_ERR(shm->dmabuf)) { + ret = ERR_CAST(shm->dmabuf); + goto err_rem; + } + } + mutex_lock(&teedev->mutex); + list_add_tail(&shm->link, &ctx->list_shm); + mutex_unlock(&teedev->mutex); + + return shm; +err_rem: + mutex_lock(&teedev->mutex); + idr_remove(&teedev->idr, shm->id); + mutex_unlock(&teedev->mutex); +err_pool_free: + poolm->ops->free(poolm, shm); +err_kfree: + kfree(shm); +err_dev_put: + tee_device_put(teedev); + return ret; +} +EXPORT_SYMBOL_GPL(tee_shm_alloc); + +struct tee_shm *tee_shm_register_fd(struct tee_context *ctx, int fd) +{ + struct tee_shm_dmabuf_ref *ref; + void *rc; + DEFINE_DMA_BUF_EXPORT_INFO(exp_info); + + if (!tee_device_get(ctx->teedev)) + return ERR_PTR(-EINVAL); + + ref = kzalloc(sizeof(*ref), GFP_KERNEL); + if (!ref) { + rc = ERR_PTR(-ENOMEM); + goto err; + } + + ref->shm.ctx = ctx; + ref->shm.teedev = ctx->teedev; + ref->shm.id = -1; + + ref->dmabuf = dma_buf_get(fd); + if (!ref->dmabuf) { + rc = ERR_PTR(-EINVAL); + goto err; + } + + ref->attach = dma_buf_attach(ref->dmabuf, &ref->shm.teedev->dev); + if (IS_ERR_OR_NULL(ref->attach)) { + rc = ERR_PTR(-EINVAL); + goto err; + } + + ref->sgt = dma_buf_map_attachment(ref->attach, DMA_BIDIRECTIONAL); + if (IS_ERR_OR_NULL(ref->sgt)) { + rc = ERR_PTR(-EINVAL); + goto err; + } + + if (sg_nents(ref->sgt->sgl) != 1) { + rc = ERR_PTR(-EINVAL); + goto err; + } + + ref->shm.paddr = sg_dma_address(ref->sgt->sgl); + ref->shm.size = sg_dma_len(ref->sgt->sgl); + ref->shm.flags = TEE_SHM_DMA_BUF | TEE_SHM_EXT_DMA_BUF; + + mutex_lock(&ref->shm.teedev->mutex); + ref->shm.id = idr_alloc(&ref->shm.teedev->idr, &ref->shm, + 1, 0, GFP_KERNEL); + mutex_unlock(&ref->shm.teedev->mutex); + if (ref->shm.id < 0) { + rc = ERR_PTR(ref->shm.id); + goto err; + } + + /* export a dmabuf to later get a userland ref */ + exp_info.ops = &tee_shm_dma_buf_ops; + exp_info.size = ref->shm.size; + exp_info.flags = O_RDWR; + exp_info.priv = &ref->shm; + + ref->shm.dmabuf = dma_buf_export(&exp_info); + if (IS_ERR(ref->shm.dmabuf)) { + rc = ERR_PTR(-EINVAL); + goto err; + } + + mutex_lock(&ref->shm.teedev->mutex); + list_add_tail(&ref->shm.link, &ctx->list_shm); + mutex_unlock(&ref->shm.teedev->mutex); + + return &ref->shm; + +err: + if (ref) { + if (ref->shm.id >= 0) { + mutex_lock(&ctx->teedev->mutex); + idr_remove(&ctx->teedev->idr, ref->shm.id); + mutex_unlock(&ctx->teedev->mutex); + } + if (ref->sgt) + dma_buf_unmap_attachment(ref->attach, ref->sgt, + DMA_BIDIRECTIONAL); + if (ref->attach) + dma_buf_detach(ref->dmabuf, ref->attach); + if (ref->dmabuf) + dma_buf_put(ref->dmabuf); + } + kfree(ref); + tee_device_put(ctx->teedev); + return rc; +} +EXPORT_SYMBOL_GPL(tee_shm_register_fd); + +/** + * tee_shm_get_fd() - Increase reference count and return file descriptor + * @shm: Shared memory handle + * @returns user space file descriptor to shared memory + */ +int tee_shm_get_fd(struct tee_shm *shm) +{ + int fd; + + if (!(shm->flags & TEE_SHM_DMA_BUF)) + return -EINVAL; + + fd = dma_buf_fd(shm->dmabuf, O_CLOEXEC); + if (fd >= 0) + get_dma_buf(shm->dmabuf); + return fd; +} + +/** + * tee_shm_free() - Free shared memory + * @shm: Handle to shared memory to free + */ +void tee_shm_free(struct tee_shm *shm) +{ + /* + * dma_buf_put() decreases the dmabuf reference counter and will + * call tee_shm_release() when the last reference is gone. + * + * In the case of driver private memory we call tee_shm_release + * directly instead as it doesn't have a reference counter. + */ + if (shm->flags & TEE_SHM_DMA_BUF) + dma_buf_put(shm->dmabuf); + else + tee_shm_release(shm); +} +EXPORT_SYMBOL_GPL(tee_shm_free); + +/** + * tee_shm_va2pa() - Get physical address of a virtual address + * @shm: Shared memory handle + * @va: Virtual address to tranlsate + * @pa: Returned physical address + * @returns 0 on success and < 0 on failure + */ +int tee_shm_va2pa(struct tee_shm *shm, void *va, phys_addr_t *pa) +{ + if (!(shm->flags & TEE_SHM_MAPPED)) + return -EINVAL; + /* Check that we're in the range of the shm */ + if ((char *)va < (char *)shm->kaddr) + return -EINVAL; + if ((char *)va >= ((char *)shm->kaddr + shm->size)) + return -EINVAL; + + return tee_shm_get_pa( + shm, (unsigned long)va - (unsigned long)shm->kaddr, pa); +} +EXPORT_SYMBOL_GPL(tee_shm_va2pa); + +/** + * tee_shm_pa2va() - Get virtual address of a physical address + * @shm: Shared memory handle + * @pa: Physical address to tranlsate + * @va: Returned virtual address + * @returns 0 on success and < 0 on failure + */ +int tee_shm_pa2va(struct tee_shm *shm, phys_addr_t pa, void **va) +{ + if (!(shm->flags & TEE_SHM_MAPPED)) + return -EINVAL; + /* Check that we're in the range of the shm */ + if (pa < shm->paddr) + return -EINVAL; + if (pa >= (shm->paddr + shm->size)) + return -EINVAL; + + if (va) { + void *v = tee_shm_get_va(shm, pa - shm->paddr); + + if (IS_ERR(v)) + return PTR_ERR(v); + *va = v; + } + return 0; +} +EXPORT_SYMBOL_GPL(tee_shm_pa2va); + +/** + * tee_shm_get_va() - Get virtual address of a shared memory plus an offset + * @shm: Shared memory handle + * @offs: Offset from start of this shared memory + * @returns virtual address of the shared memory + offs if offs is within + * the bounds of this shared memory, else an ERR_PTR + */ +void *tee_shm_get_va(struct tee_shm *shm, size_t offs) +{ + if (!(shm->flags & TEE_SHM_MAPPED)) + return ERR_PTR(-EINVAL); + if (offs >= shm->size) + return ERR_PTR(-EINVAL); + return (char *)shm->kaddr + offs; +} +EXPORT_SYMBOL_GPL(tee_shm_get_va); + +/** + * tee_shm_get_pa() - Get physical address of a shared memory plus an offset + * @shm: Shared memory handle + * @offs: Offset from start of this shared memory + * @pa: Physical address to return + * @returns 0 if offs is within the bounds of this shared memory, else an + * error code. + */ +int tee_shm_get_pa(struct tee_shm *shm, size_t offs, phys_addr_t *pa) +{ + if (offs >= shm->size) + return -EINVAL; + if (pa) + *pa = shm->paddr + offs; + return 0; +} +EXPORT_SYMBOL_GPL(tee_shm_get_pa); + +/** + * tee_shm_get_from_id() - Find shared memory object and increase referece count + * @ctx: Context owning the shared memory + * @id: Id of shared memory object + * @returns a pointer to 'struct tee_shm' on success or an ERR_PTR on failure + */ +struct tee_shm *tee_shm_get_from_id(struct tee_context *ctx, int id) +{ + struct tee_device *teedev; + struct tee_shm *shm; + + if (!ctx) + return ERR_PTR(-EINVAL); + + teedev = ctx->teedev; + mutex_lock(&teedev->mutex); + shm = idr_find(&teedev->idr, id); + if (!shm || shm->ctx != ctx) + shm = ERR_PTR(-EINVAL); + else if (shm->flags & TEE_SHM_DMA_BUF) + get_dma_buf(shm->dmabuf); + mutex_unlock(&teedev->mutex); + return shm; +} +EXPORT_SYMBOL_GPL(tee_shm_get_from_id); + +/** + * tee_shm_get_id() - Get id of a shared memory object + * @shm: Shared memory handle + * @returns id + */ +int tee_shm_get_id(struct tee_shm *shm) +{ + return shm->id; +} +EXPORT_SYMBOL_GPL(tee_shm_get_id); + +/** + * tee_shm_put() - Decrease reference count on a shared memory handle + * @shm: Shared memory handle + */ +void tee_shm_put(struct tee_shm *shm) +{ + if (shm->flags & TEE_SHM_DMA_BUF) + dma_buf_put(shm->dmabuf); +} +EXPORT_SYMBOL_GPL(tee_shm_put); diff --git a/drivers/tee/tee_shm_pool.c b/drivers/tee/tee_shm_pool.c new file mode 100644 index 00000000000000..3cb0ad0ce99f8e --- /dev/null +++ b/drivers/tee/tee_shm_pool.c @@ -0,0 +1,158 @@ +/* + * Copyright (c) 2015, Linaro Limited + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ +#include +#include +#include +#include +#include +#include "tee_private.h" + +static int pool_op_gen_alloc(struct tee_shm_pool_mgr *poolm, + struct tee_shm *shm, size_t size) +{ + unsigned long va; + struct gen_pool *genpool = poolm->private_data; + size_t s = roundup(size, 1 << genpool->min_alloc_order); + + va = gen_pool_alloc(genpool, s); + if (!va) + return -ENOMEM; + + memset((void *)va, 0, s); + shm->kaddr = (void *)va; + shm->paddr = gen_pool_virt_to_phys(genpool, va); + shm->size = s; + return 0; +} + +static void pool_op_gen_free(struct tee_shm_pool_mgr *poolm, + struct tee_shm *shm) +{ + gen_pool_free(poolm->private_data, (unsigned long)shm->kaddr, + shm->size); + shm->kaddr = NULL; +} + +static const struct tee_shm_pool_mgr_ops pool_ops_generic = { + .alloc = pool_op_gen_alloc, + .free = pool_op_gen_free, +}; + +static void pool_res_mem_destroy(struct tee_shm_pool *pool) +{ + gen_pool_destroy(pool->private_mgr.private_data); + gen_pool_destroy(pool->dma_buf_mgr.private_data); +} + +static int pool_res_mem_mgr_init(struct tee_shm_pool_mgr *mgr, + struct tee_shm_pool_mem_info *info, + int min_alloc_order) +{ + size_t page_mask = PAGE_SIZE - 1; + struct gen_pool *genpool = NULL; + int rc; + + /* + * Start and end must be page aligned + */ + if ((info->vaddr & page_mask) || (info->paddr & page_mask) || + (info->size & page_mask)) + return -EINVAL; + + genpool = gen_pool_create(min_alloc_order, -1); + if (!genpool) + return -ENOMEM; + + gen_pool_set_algo(genpool, gen_pool_best_fit, NULL); + rc = gen_pool_add_virt(genpool, info->vaddr, info->paddr, info->size, + -1); + if (rc) { + gen_pool_destroy(genpool); + return rc; + } + + mgr->private_data = genpool; + mgr->ops = &pool_ops_generic; + return 0; +} + +/** + * tee_shm_pool_alloc_res_mem() - Create a shared memory pool from reserved + * memory range + * @dev: Device allocating the pool + * @priv_info: Information for driver private shared memory pool + * @dmabuf_info: Information for dma-buf shared memory pool + * + * Start and end of pools will must be page aligned. + * + * Allocation with the flag TEE_SHM_DMA_BUF set will use the range supplied + * in @dmabuf, others will use the range provided by @priv. + * + * @returns pointer to a 'struct tee_shm_pool' or an ERR_PTR on failure. + */ +struct tee_shm_pool * +tee_shm_pool_alloc_res_mem(struct device *dev, + struct tee_shm_pool_mem_info *priv_info, + struct tee_shm_pool_mem_info *dmabuf_info) +{ + struct tee_shm_pool *pool = NULL; + int ret; + + pool = kzalloc(sizeof(*pool), GFP_KERNEL); + if (!pool) { + ret = -ENOMEM; + goto err; + } + + /* + * Create the pool for driver private shared memory + */ + ret = pool_res_mem_mgr_init(&pool->private_mgr, priv_info, + 3 /* 8 byte aligned */); + if (ret) + goto err; + + /* + * Create the pool for dma_buf shared memory + */ + ret = pool_res_mem_mgr_init(&pool->dma_buf_mgr, dmabuf_info, + PAGE_SHIFT); + if (ret) + goto err; + + pool->destroy = pool_res_mem_destroy; + return pool; +err: + if (ret == -ENOMEM) + dev_err(dev, "can't allocate memory for res_mem shared memory pool\n"); + if (pool && pool->private_mgr.private_data) + gen_pool_destroy(pool->private_mgr.private_data); + kfree(pool); + return ERR_PTR(ret); +} +EXPORT_SYMBOL_GPL(tee_shm_pool_alloc_res_mem); + +/** + * tee_shm_pool_free() - Free a shared memory pool + * @pool: The shared memory pool to free + * + * There must be no remaining shared memory allocated from this pool when + * this function is called. + */ +void tee_shm_pool_free(struct tee_shm_pool *pool) +{ + pool->destroy(pool); + kfree(pool); +} +EXPORT_SYMBOL_GPL(tee_shm_pool_free); diff --git a/drivers/thermal/thermal_hwmon.c b/drivers/thermal/thermal_hwmon.c index c41c7742903ab4..2dcd4194d103bd 100644 --- a/drivers/thermal/thermal_hwmon.c +++ b/drivers/thermal/thermal_hwmon.c @@ -98,7 +98,7 @@ temp_crit_show(struct device *dev, struct device_attribute *attr, char *buf) int temperature; int ret; - ret = tz->ops->get_trip_temp(tz, 0, &temperature); + ret = tz->ops->get_crit_temp(tz, &temperature); if (ret) return ret; diff --git a/drivers/tty/serial/8250/8250_core.c b/drivers/tty/serial/8250/8250_core.c index 240a361b674fe7..e8819aa2041560 100644 --- a/drivers/tty/serial/8250/8250_core.c +++ b/drivers/tty/serial/8250/8250_core.c @@ -675,7 +675,7 @@ static struct console univ8250_console = { .device = uart_console_device, .setup = univ8250_console_setup, .match = univ8250_console_match, - .flags = CON_PRINTBUFFER | CON_ANYTIME | CON_CONSDEV, + .flags = CON_PRINTBUFFER | CON_ANYTIME, .index = -1, .data = &serial8250_reg, }; diff --git a/drivers/tty/serial/8250/8250_port.c b/drivers/tty/serial/8250/8250_port.c index 1731b98d247107..080d5a59d0a7f4 100644 --- a/drivers/tty/serial/8250/8250_port.c +++ b/drivers/tty/serial/8250/8250_port.c @@ -1411,7 +1411,7 @@ static void __do_stop_tx_rs485(struct uart_8250_port *p) * Enable previously disabled RX interrupts. */ if (!(p->port.rs485.flags & SER_RS485_RX_DURING_TX)) { - serial8250_clear_fifos(p); + serial8250_clear_and_reinit_fifos(p); p->ier |= UART_IER_RLSI | UART_IER_RDI; serial_port_out(&p->port, UART_IER, p->ier); diff --git a/drivers/tty/serial/atmel_serial.c b/drivers/tty/serial/atmel_serial.c index 168b10cad47b54..fabbe76203bb76 100644 --- a/drivers/tty/serial/atmel_serial.c +++ b/drivers/tty/serial/atmel_serial.c @@ -481,6 +481,14 @@ static void atmel_stop_tx(struct uart_port *port) /* disable PDC transmit */ atmel_uart_writel(port, ATMEL_PDC_PTCR, ATMEL_PDC_TXTDIS); } + + /* + * Disable the transmitter. + * This is mandatory when DMA is used, otherwise the DMA buffer + * is fully transmitted. + */ + atmel_uart_writel(port, ATMEL_US_CR, ATMEL_US_TXDIS); + /* Disable interrupts */ atmel_uart_writel(port, ATMEL_US_IDR, atmel_port->tx_done_mask); @@ -513,6 +521,9 @@ static void atmel_start_tx(struct uart_port *port) /* Enable interrupts */ atmel_uart_writel(port, ATMEL_US_IER, atmel_port->tx_done_mask); + + /* re-enable the transmitter */ + atmel_uart_writel(port, ATMEL_US_CR, ATMEL_US_TXEN); } /* @@ -798,6 +809,11 @@ static void atmel_complete_tx_dma(void *arg) */ if (!uart_circ_empty(xmit)) atmel_tasklet_schedule(atmel_port, &atmel_port->tasklet_tx); + else if ((port->rs485.flags & SER_RS485_ENABLED) && + !(port->rs485.flags & SER_RS485_RX_DURING_TX)) { + /* DMA done, stop TX, start RX for RS485 */ + atmel_start_rx(port); + } spin_unlock_irqrestore(&port->lock, flags); } @@ -900,12 +916,6 @@ static void atmel_tx_dma(struct uart_port *port) desc->callback = atmel_complete_tx_dma; desc->callback_param = atmel_port; atmel_port->cookie_tx = dmaengine_submit(desc); - - } else { - if (port->rs485.flags & SER_RS485_ENABLED) { - /* DMA done, stop TX, start RX for RS485 */ - atmel_start_rx(port); - } } if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS) diff --git a/drivers/tty/serial/sc16is7xx.c b/drivers/tty/serial/sc16is7xx.c index fb0672554123a1..793395451982d8 100644 --- a/drivers/tty/serial/sc16is7xx.c +++ b/drivers/tty/serial/sc16is7xx.c @@ -1264,7 +1264,7 @@ static int sc16is7xx_probe(struct device *dev, /* Setup interrupt */ ret = devm_request_irq(dev, irq, sc16is7xx_irq, - IRQF_ONESHOT | flags, dev_name(dev), s); + flags, dev_name(dev), s); if (!ret) return 0; diff --git a/drivers/tty/serial/serial_core.c b/drivers/tty/serial/serial_core.c index f2303f390345e1..2e2b88aa3004b2 100644 --- a/drivers/tty/serial/serial_core.c +++ b/drivers/tty/serial/serial_core.c @@ -131,6 +131,9 @@ static void __uart_start(struct tty_struct *tty) struct uart_state *state = tty->driver_data; struct uart_port *port = state->uart_port; + if (port && port->ops->wake_peer) + port->ops->wake_peer(port); + if (port && !uart_tx_stopped(port)) port->ops->start_tx(port); } diff --git a/drivers/tty/sysrq.c b/drivers/tty/sysrq.c index 52bbd27e93ae7a..701c085bb19b8e 100644 --- a/drivers/tty/sysrq.c +++ b/drivers/tty/sysrq.c @@ -946,8 +946,8 @@ static const struct input_device_id sysrq_ids[] = { { .flags = INPUT_DEVICE_ID_MATCH_EVBIT | INPUT_DEVICE_ID_MATCH_KEYBIT, - .evbit = { BIT_MASK(EV_KEY) }, - .keybit = { BIT_MASK(KEY_LEFTALT) }, + .evbit = { [BIT_WORD(EV_KEY)] = BIT_MASK(EV_KEY) }, + .keybit = { [BIT_WORD(KEY_LEFTALT)] = BIT_MASK(KEY_LEFTALT) }, }, { }, }; diff --git a/drivers/tty/vt/keyboard.c b/drivers/tty/vt/keyboard.c index 0f8caae4267df8..ece10e6b731baa 100644 --- a/drivers/tty/vt/keyboard.c +++ b/drivers/tty/vt/keyboard.c @@ -982,7 +982,7 @@ static void kbd_led_trigger_activate(struct led_classdev *cdev) KBD_LED_TRIGGER((_led_bit) + 8, _name) static struct kbd_led_trigger kbd_led_triggers[] = { - KBD_LED_TRIGGER(VC_SCROLLOCK, "kbd-scrollock"), + KBD_LED_TRIGGER(VC_SCROLLOCK, "kbd-scrolllock"), KBD_LED_TRIGGER(VC_NUMLOCK, "kbd-numlock"), KBD_LED_TRIGGER(VC_CAPSLOCK, "kbd-capslock"), KBD_LED_TRIGGER(VC_KANALOCK, "kbd-kanalock"), diff --git a/drivers/usb/class/cdc-acm.c b/drivers/usb/class/cdc-acm.c index fada988512a162..c5ff13f22b2473 100644 --- a/drivers/usb/class/cdc-acm.c +++ b/drivers/usb/class/cdc-acm.c @@ -1719,6 +1719,7 @@ static const struct usb_device_id acm_ids[] = { { USB_DEVICE(0x20df, 0x0001), /* Simtec Electronics Entropy Key */ .driver_info = QUIRK_CONTROL_LINE_STATE, }, { USB_DEVICE(0x2184, 0x001c) }, /* GW Instek AFG-2225 */ + { USB_DEVICE(0x2184, 0x0036) }, /* GW Instek AFG-125 */ { USB_DEVICE(0x22b8, 0x6425), /* Motorola MOTOMAGX phones */ }, /* Motorola H24 HSPA module: */ diff --git a/drivers/usb/core/config.c b/drivers/usb/core/config.c index a2d90aca779fa1..1f7036c8f57b5f 100644 --- a/drivers/usb/core/config.c +++ b/drivers/usb/core/config.c @@ -234,6 +234,16 @@ static int usb_parse_endpoint(struct device *ddev, int cfgno, int inum, if (ifp->desc.bNumEndpoints >= num_ep) goto skip_to_next_endpoint_or_interface_descriptor; + /* Check for duplicate endpoint addresses */ + for (i = 0; i < ifp->desc.bNumEndpoints; ++i) { + if (ifp->endpoint[i].desc.bEndpointAddress == + d->bEndpointAddress) { + dev_warn(ddev, "config %d interface %d altsetting %d has a duplicate endpoint with address 0x%X, skipping\n", + cfgno, inum, asnum, d->bEndpointAddress); + goto skip_to_next_endpoint_or_interface_descriptor; + } + } + endpoint = &ifp->endpoint[ifp->desc.bNumEndpoints]; ++ifp->desc.bNumEndpoints; diff --git a/drivers/usb/core/hub.c b/drivers/usb/core/hub.c index cbb146736f577d..aef81a16e2c853 100644 --- a/drivers/usb/core/hub.c +++ b/drivers/usb/core/hub.c @@ -101,6 +101,7 @@ EXPORT_SYMBOL_GPL(ehci_cf_port_reset_rwsem); static void hub_release(struct kref *kref); static int usb_reset_and_verify_device(struct usb_device *udev); +static int hub_port_disable(struct usb_hub *hub, int port1, int set_state); static inline char *portspeed(struct usb_hub *hub, int portstatus) { @@ -898,88 +899,6 @@ static int hub_set_port_link_state(struct usb_hub *hub, int port1, USB_PORT_FEAT_LINK_STATE); } -/* - * If USB 3.0 ports are placed into the Disabled state, they will no longer - * detect any device connects or disconnects. This is generally not what the - * USB core wants, since it expects a disabled port to produce a port status - * change event when a new device connects. - * - * Instead, set the link state to Disabled, wait for the link to settle into - * that state, clear any change bits, and then put the port into the RxDetect - * state. - */ -static int hub_usb3_port_disable(struct usb_hub *hub, int port1) -{ - int ret; - int total_time; - u16 portchange, portstatus; - - if (!hub_is_superspeed(hub->hdev)) - return -EINVAL; - - ret = hub_port_status(hub, port1, &portstatus, &portchange); - if (ret < 0) - return ret; - - /* - * USB controller Advanced Micro Devices, Inc. [AMD] FCH USB XHCI - * Controller [1022:7814] will have spurious result making the following - * usb 3.0 device hotplugging route to the 2.0 root hub and recognized - * as high-speed device if we set the usb 3.0 port link state to - * Disabled. Since it's already in USB_SS_PORT_LS_RX_DETECT state, we - * check the state here to avoid the bug. - */ - if ((portstatus & USB_PORT_STAT_LINK_STATE) == - USB_SS_PORT_LS_RX_DETECT) { - dev_dbg(&hub->ports[port1 - 1]->dev, - "Not disabling port; link state is RxDetect\n"); - return ret; - } - - ret = hub_set_port_link_state(hub, port1, USB_SS_PORT_LS_SS_DISABLED); - if (ret) - return ret; - - /* Wait for the link to enter the disabled state. */ - for (total_time = 0; ; total_time += HUB_DEBOUNCE_STEP) { - ret = hub_port_status(hub, port1, &portstatus, &portchange); - if (ret < 0) - return ret; - - if ((portstatus & USB_PORT_STAT_LINK_STATE) == - USB_SS_PORT_LS_SS_DISABLED) - break; - if (total_time >= HUB_DEBOUNCE_TIMEOUT) - break; - msleep(HUB_DEBOUNCE_STEP); - } - if (total_time >= HUB_DEBOUNCE_TIMEOUT) - dev_warn(&hub->ports[port1 - 1]->dev, - "Could not disable after %d ms\n", total_time); - - return hub_set_port_link_state(hub, port1, USB_SS_PORT_LS_RX_DETECT); -} - -static int hub_port_disable(struct usb_hub *hub, int port1, int set_state) -{ - struct usb_port *port_dev = hub->ports[port1 - 1]; - struct usb_device *hdev = hub->hdev; - int ret = 0; - - if (port_dev->child && set_state) - usb_set_device_state(port_dev->child, USB_STATE_NOTATTACHED); - if (!hub->error) { - if (hub_is_superspeed(hub->hdev)) - ret = hub_usb3_port_disable(hub, port1); - else - ret = usb_clear_port_feature(hdev, port1, - USB_PORT_FEAT_ENABLE); - } - if (ret && ret != -ENODEV) - dev_err(&port_dev->dev, "cannot disable (err = %d)\n", ret); - return ret; -} - /* * Disable a port and mark a logical connect-change event, so that some * time later hub_wq will disconnect() any existing usb_device on the port @@ -4140,6 +4059,26 @@ void usb_unlocked_enable_lpm(struct usb_device *udev) } EXPORT_SYMBOL_GPL(usb_unlocked_enable_lpm); +/* usb3 devices use U3 for disabled, make sure remote wakeup is disabled */ +static void hub_usb3_port_prepare_disable(struct usb_hub *hub, + struct usb_port *port_dev) +{ + struct usb_device *udev = port_dev->child; + int ret; + + if (udev && udev->port_is_suspended && udev->do_remote_wakeup) { + ret = hub_set_port_link_state(hub, port_dev->portnum, + USB_SS_PORT_LS_U0); + if (!ret) { + msleep(USB_RESUME_TIMEOUT); + ret = usb_disable_remote_wakeup(udev); + } + if (ret) + dev_warn(&udev->dev, + "Port disable: can't disable remote wake\n"); + udev->do_remote_wakeup = 0; + } +} #else /* CONFIG_PM */ @@ -4147,6 +4086,9 @@ EXPORT_SYMBOL_GPL(usb_unlocked_enable_lpm); #define hub_resume NULL #define hub_reset_resume NULL +static inline void hub_usb3_port_prepare_disable(struct usb_hub *hub, + struct usb_port *port_dev) { } + int usb_disable_lpm(struct usb_device *udev) { return 0; @@ -4182,6 +4124,34 @@ static int hub_handle_remote_wakeup(struct usb_hub *hub, unsigned int port, #endif /* CONFIG_PM */ +/* + * USB-3 does not have a similar link state as USB-2 that will avoid negotiating + * a connection with a plugged-in cable but will signal the host when the cable + * is unplugged. Disable remote wake and set link state to U3 for USB-3 devices + */ +static int hub_port_disable(struct usb_hub *hub, int port1, int set_state) +{ + struct usb_port *port_dev = hub->ports[port1 - 1]; + struct usb_device *hdev = hub->hdev; + int ret = 0; + + if (!hub->error) { + if (hub_is_superspeed(hub->hdev)) { + hub_usb3_port_prepare_disable(hub, port_dev); + ret = hub_set_port_link_state(hub, port_dev->portnum, + USB_SS_PORT_LS_U3); + } else { + ret = usb_clear_port_feature(hdev, port1, + USB_PORT_FEAT_ENABLE); + } + } + if (port_dev->child && set_state) + usb_set_device_state(port_dev->child, USB_STATE_NOTATTACHED); + if (ret && ret != -ENODEV) + dev_err(&port_dev->dev, "cannot disable (err = %d)\n", ret); + return ret; +} + /* USB 2.0 spec, 7.1.7.3 / fig 7-29: * diff --git a/drivers/usb/core/ledtrig-usbport.c b/drivers/usb/core/ledtrig-usbport.c index 3ed5162677ad56..1713248ab15ad4 100644 --- a/drivers/usb/core/ledtrig-usbport.c +++ b/drivers/usb/core/ledtrig-usbport.c @@ -74,8 +74,7 @@ static void usbport_trig_update_count(struct usbport_trig_data *usbport_data) usbport_data->count = 0; usb_for_each_dev(usbport_data, usbport_trig_usb_dev_check); - led_cdev->brightness_set(led_cdev, - usbport_data->count ? LED_FULL : LED_OFF); + led_set_brightness(led_cdev, usbport_data->count ? LED_FULL : LED_OFF); } /*************************************** @@ -228,12 +227,12 @@ static int usbport_trig_notify(struct notifier_block *nb, unsigned long action, case USB_DEVICE_ADD: usbport_trig_add_usb_dev_ports(usb_dev, usbport_data); if (observed && usbport_data->count++ == 0) - led_cdev->brightness_set(led_cdev, LED_FULL); + led_set_brightness(led_cdev, LED_FULL); return NOTIFY_OK; case USB_DEVICE_REMOVE: usbport_trig_remove_usb_dev_ports(usbport_data, usb_dev); if (observed && --usbport_data->count == 0) - led_cdev->brightness_set(led_cdev, LED_OFF); + led_set_brightness(led_cdev, LED_OFF); return NOTIFY_OK; } diff --git a/drivers/usb/core/quirks.c b/drivers/usb/core/quirks.c index d2e50a27140c92..24f9f98968a5d8 100644 --- a/drivers/usb/core/quirks.c +++ b/drivers/usb/core/quirks.c @@ -37,6 +37,10 @@ static const struct usb_device_id usb_quirk_list[] = { /* CBM - Flash disk */ { USB_DEVICE(0x0204, 0x6025), .driver_info = USB_QUIRK_RESET_RESUME }, + /* WORLDE easy key (easykey.25) MIDI controller */ + { USB_DEVICE(0x0218, 0x0401), .driver_info = + USB_QUIRK_CONFIG_INTF_STRINGS }, + /* HP 5300/5370C scanner */ { USB_DEVICE(0x03f0, 0x0701), .driver_info = USB_QUIRK_STRING_FETCH_255 }, diff --git a/drivers/usb/dwc2/core.c b/drivers/usb/dwc2/core.c index 4c0fa0b1735388..2e83545776c97b 100644 --- a/drivers/usb/dwc2/core.c +++ b/drivers/usb/dwc2/core.c @@ -313,7 +313,7 @@ static bool dwc2_iddig_filter_enabled(struct dwc2_hsotg *hsotg) * Do core a soft reset of the core. Be careful with this because it * resets all the internal state machines of the core. */ -int dwc2_core_reset(struct dwc2_hsotg *hsotg) +int dwc2_core_reset(struct dwc2_hsotg *hsotg, bool skip_wait) { u32 greset; int count = 0; @@ -369,7 +369,7 @@ int dwc2_core_reset(struct dwc2_hsotg *hsotg) } } while (!(greset & GRSTCTL_AHBIDLE)); - if (wait_for_host_mode) + if (wait_for_host_mode && !skip_wait) dwc2_wait_for_mode(hsotg, true); return 0; @@ -500,7 +500,7 @@ int dwc2_core_reset_and_force_dr_mode(struct dwc2_hsotg *hsotg) { int retval; - retval = dwc2_core_reset(hsotg); + retval = dwc2_core_reset(hsotg, false); if (retval) return retval; diff --git a/drivers/usb/dwc2/core.h b/drivers/usb/dwc2/core.h index 2a21a0414b1d38..bc723c5eb2af2f 100644 --- a/drivers/usb/dwc2/core.h +++ b/drivers/usb/dwc2/core.h @@ -417,6 +417,11 @@ enum dwc2_ep0_state { * needed. * 0 - No (default) * 1 - Yes + * @change_speed_quirk: Change speed configuration to DWC2_SPEED_PARAM_FULL + * while full&low speed device connect. And change speed + * back to DWC2_SPEED_PARAM_HIGH while device is gone. + * 0 - No (default) + * 1 - Yes * * The following parameters may be specified when starting the module. These * parameters define how the DWC_otg controller should be configured. A @@ -457,6 +462,7 @@ struct dwc2_core_params { int uframe_sched; int external_id_pin_ctl; int hibernation; + int change_speed_quirk; }; /** @@ -623,6 +629,19 @@ struct dwc2_hregs_backup { bool valid; }; +/* + * struct dwc2_extcon - Holds data related to extcon connections + * + * @nb: Notifier block for callbacks + * @extcon: Pointer to extcon device + * @state: State of extcon connection + */ +struct dwc2_extcon { + struct notifier_block nb; + struct extcon_dev *extcon; + int state; +}; + /* * Constants related to high speed periodic scheduling * @@ -996,6 +1015,10 @@ struct dwc2_hsotg { u32 g_np_g_tx_fifo_sz; u32 g_tx_fifo_sz[MAX_EPS_CHANNELS]; #endif /* CONFIG_USB_DWC2_PERIPHERAL || CONFIG_USB_DWC2_DUAL_ROLE */ + + struct dwc2_extcon extcon_vbus; + struct dwc2_extcon extcon_id; + struct delayed_work extcon_work; }; /* Reasons for halting a host channel */ @@ -1020,7 +1043,7 @@ enum dwc2_halt_status { * The following functions support initialization of the core driver component * and the DWC_otg controller */ -extern int dwc2_core_reset(struct dwc2_hsotg *hsotg); +extern int dwc2_core_reset(struct dwc2_hsotg *hsotg, bool skip_wait); extern int dwc2_core_reset_and_force_dr_mode(struct dwc2_hsotg *hsotg); extern int dwc2_enter_hibernation(struct dwc2_hsotg *hsotg); extern int dwc2_exit_hibernation(struct dwc2_hsotg *hsotg, bool restore); @@ -1041,6 +1064,11 @@ extern void dwc2_flush_rx_fifo(struct dwc2_hsotg *hsotg); extern void dwc2_enable_global_interrupts(struct dwc2_hsotg *hcd); extern void dwc2_disable_global_interrupts(struct dwc2_hsotg *hcd); +extern int dwc2_extcon_vbus_notifier(struct notifier_block *nb, + unsigned long event, void *ptr); +extern int dwc2_extcon_id_notifier(struct notifier_block *nb, + unsigned long event, void *ptr); + /* This function should be called on every hardware interrupt. */ extern irqreturn_t dwc2_handle_common_intr(int irq, void *dev); @@ -1298,6 +1326,7 @@ extern int dwc2_hsotg_remove(struct dwc2_hsotg *hsotg); extern int dwc2_hsotg_suspend(struct dwc2_hsotg *dwc2); extern int dwc2_hsotg_resume(struct dwc2_hsotg *dwc2); extern int dwc2_gadget_init(struct dwc2_hsotg *hsotg, int irq); +extern void dwc2_gadget_notify(struct dwc2_hsotg *hsotg); extern void dwc2_hsotg_core_init_disconnected(struct dwc2_hsotg *dwc2, bool reset); extern void dwc2_hsotg_core_connect(struct dwc2_hsotg *hsotg); @@ -1315,6 +1344,7 @@ static inline int dwc2_hsotg_resume(struct dwc2_hsotg *dwc2) { return 0; } static inline int dwc2_gadget_init(struct dwc2_hsotg *hsotg, int irq) { return 0; } +static inline void dwc2_gadget_notify(struct dwc2_hsotg *hsotg) {} static inline void dwc2_hsotg_core_init_disconnected(struct dwc2_hsotg *dwc2, bool reset) {} static inline void dwc2_hsotg_core_connect(struct dwc2_hsotg *hsotg) {} diff --git a/drivers/usb/dwc2/core_intr.c b/drivers/usb/dwc2/core_intr.c index d85c5c9f96c1dc..0120bc75441b73 100644 --- a/drivers/usb/dwc2/core_intr.c +++ b/drivers/usb/dwc2/core_intr.c @@ -479,6 +479,28 @@ static void dwc2_handle_usb_suspend_intr(struct dwc2_hsotg *hsotg) } } +int dwc2_extcon_vbus_notifier(struct notifier_block *nb, unsigned long event, + void *ptr) +{ + struct dwc2_extcon *vbus = container_of(nb, struct dwc2_extcon, nb); + struct dwc2_hsotg *hsotg = container_of(vbus, struct dwc2_hsotg, + extcon_vbus); + + schedule_delayed_work(&hsotg->extcon_work, msecs_to_jiffies(100)); + return NOTIFY_DONE; +} + +int dwc2_extcon_id_notifier(struct notifier_block *nb, unsigned long event, + void *ptr) +{ + struct dwc2_extcon *id = container_of(nb, struct dwc2_extcon, nb); + struct dwc2_hsotg *hsotg = container_of(id, struct dwc2_hsotg, + extcon_id); + + schedule_delayed_work(&hsotg->extcon_work, msecs_to_jiffies(100)); + return NOTIFY_DONE; +} + #define GINTMSK_COMMON (GINTSTS_WKUPINT | GINTSTS_SESSREQINT | \ GINTSTS_CONIDSTSCHNG | GINTSTS_OTGINT | \ GINTSTS_MODEMIS | GINTSTS_DISCONNINT | \ diff --git a/drivers/usb/dwc2/gadget.c b/drivers/usb/dwc2/gadget.c index 24fbebc9b40905..54f57bb6808f2d 100644 --- a/drivers/usb/dwc2/gadget.c +++ b/drivers/usb/dwc2/gadget.c @@ -72,6 +72,14 @@ static inline struct dwc2_hsotg_ep *index_to_ep(struct dwc2_hsotg *hsotg, /* forward declaration of functions */ static void dwc2_hsotg_dump(struct dwc2_hsotg *hsotg); +void dwc2_gadget_notify(struct dwc2_hsotg *hsotg) +{ + if (hsotg->extcon_id.state) + usb_gadget_vbus_connect(&hsotg->gadget); + else + usb_gadget_vbus_disconnect(&hsotg->gadget); +} + /** * using_dma - return the DMA status of the driver. * @hsotg: The driver state. @@ -2521,7 +2529,7 @@ void dwc2_hsotg_core_init_disconnected(struct dwc2_hsotg *hsotg, kill_all_requests(hsotg, hsotg->eps_out[0], -ECONNRESET); if (!is_usb_reset) - if (dwc2_core_reset(hsotg)) + if (dwc2_core_reset(hsotg, true)) return; /* diff --git a/drivers/usb/dwc2/hcd.c b/drivers/usb/dwc2/hcd.c index df5a065780054f..f4a510bcad37dc 100644 --- a/drivers/usb/dwc2/hcd.c +++ b/drivers/usb/dwc2/hcd.c @@ -47,6 +47,7 @@ #include #include #include +#include #include #include @@ -54,6 +55,8 @@ #include "core.h" #include "hcd.h" +static void dwc2_port_resume(struct dwc2_hsotg *hsotg); + /* * ========================================================================= * Host Core Layer Functions @@ -3199,17 +3202,25 @@ static void dwc2_conn_id_status_change(struct work_struct *work) dev_dbg(hsotg->dev, "gotgctl=%0x\n", gotgctl); dev_dbg(hsotg->dev, "gotgctl.b.conidsts=%d\n", !!(gotgctl & GOTGCTL_CONID_B)); - +again: /* B-Device connector (Device Mode) */ if (gotgctl & GOTGCTL_CONID_B) { /* Wait for switch to device mode */ dev_dbg(hsotg->dev, "connId B\n"); + if (hsotg->bus_suspended) { + dev_info(hsotg->dev, + "Do port resume before switching to device mode\n"); + dwc2_port_resume(hsotg); + } while (!dwc2_is_device_mode(hsotg)) { dev_info(hsotg->dev, "Waiting for Peripheral Mode, Mode=%s\n", dwc2_is_host_mode(hsotg) ? "Host" : "Peripheral"); usleep_range(20000, 40000); + gotgctl = dwc2_readl(hsotg->regs + GOTGCTL); + if (!(gotgctl & GOTGCTL_CONID_B)) + goto again; if (++count > 250) break; } @@ -3246,6 +3257,23 @@ static void dwc2_conn_id_status_change(struct work_struct *work) } } +static void dwc2_hcd_extcon_func(struct work_struct *work) +{ + struct dwc2_hsotg *hsotg = container_of(work, struct dwc2_hsotg, + extcon_work.work); + + if (!IS_ERR(hsotg->extcon_vbus.extcon)) + hsotg->extcon_vbus.state = + extcon_get_state(hsotg->extcon_vbus.extcon, + EXTCON_USB); + if (!IS_ERR(hsotg->extcon_id.extcon)) + hsotg->extcon_id.state = + extcon_get_state(hsotg->extcon_id.extcon, + EXTCON_USB_HOST); + + dwc2_gadget_notify(hsotg); +} + static void dwc2_wakeup_detected(unsigned long data) { struct dwc2_hsotg *hsotg = (struct dwc2_hsotg *)data; @@ -4365,6 +4393,9 @@ static int _dwc2_hcd_suspend(struct usb_hcd *hcd) if (!HCD_HW_ACCESSIBLE(hcd)) goto unlock; + if (hsotg->op_state == OTG_STATE_B_PERIPHERAL) + goto unlock; + if (!hsotg->core_params->hibernation) goto skip_power_saving; @@ -4860,6 +4891,61 @@ static void _dwc2_hcd_clear_tt_buffer_complete(struct usb_hcd *hcd, spin_unlock_irqrestore(&hsotg->lock, flags); } +/* + * HPRT0_SPD_HIGH_SPEED: high speed + * HPRT0_SPD_FULL_SPEED: full speed + */ +static void dwc2_change_bus_speed(struct usb_hcd *hcd, int speed) +{ + struct dwc2_hsotg *hsotg = dwc2_hcd_to_hsotg(hcd); + + if (hsotg->core_params->speed == speed) + return; + + hsotg->core_params->speed = speed; + queue_work(hsotg->wq_otg, &hsotg->wf_otg); +} + +static void dwc2_free_dev(struct usb_hcd *hcd, struct usb_device *udev) +{ + struct dwc2_hsotg *hsotg = dwc2_hcd_to_hsotg(hcd); + + if (!hsotg->core_params->change_speed_quirk) + return; + + /* + * On removal, set speed to default high-speed. + */ + if (udev->parent && udev->parent->speed > USB_SPEED_UNKNOWN && + udev->parent->speed < USB_SPEED_HIGH) { + dev_info(hsotg->dev, "Set speed to default high-speed\n"); + dwc2_change_bus_speed(hcd, HPRT0_SPD_HIGH_SPEED); + } +} + +static int dwc2_reset_device(struct usb_hcd *hcd, struct usb_device *udev) +{ + struct dwc2_hsotg *hsotg = dwc2_hcd_to_hsotg(hcd); + + if (!hsotg->core_params->change_speed_quirk) + return 0; + + if (udev->speed == USB_SPEED_HIGH) { + dev_info(hsotg->dev, "Set speed to high-speed\n"); + dwc2_change_bus_speed(hcd, HPRT0_SPD_HIGH_SPEED); + } else if ((udev->speed == USB_SPEED_FULL || + udev->speed == USB_SPEED_LOW)) { + /* + * Change speed setting to full-speed if there's + * a full-speed or low-speed device plugged in. + */ + dev_info(hsotg->dev, "Set speed to full-speed\n"); + dwc2_change_bus_speed(hcd, HPRT0_SPD_FULL_SPEED); + } + + return 0; +} + static struct hc_driver dwc2_hc_driver = { .description = "dwc2_hsotg", .product_desc = "DWC OTG Controller", @@ -5015,6 +5101,11 @@ int dwc2_hcd_init(struct dwc2_hsotg *hsotg, int irq) dev_warn(hsotg->dev, "can't set coherent DMA mask\n"); } + if (hsotg->core_params->change_speed_quirk) { + dwc2_hc_driver.free_dev = dwc2_free_dev; + dwc2_hc_driver.reset_device = dwc2_reset_device; + } + hcd = usb_create_hcd(&dwc2_hc_driver, hsotg->dev, dev_name(hsotg->dev)); if (!hcd) goto error1; @@ -5085,6 +5176,8 @@ int dwc2_hcd_init(struct dwc2_hsotg *hsotg, int irq) /* Initialize port reset work */ INIT_DELAYED_WORK(&hsotg->reset_work, dwc2_hcd_reset_func); + INIT_DELAYED_WORK(&hsotg->extcon_work, dwc2_hcd_extcon_func); + /* * Allocate space for storing data on status transactions. Normally no * data is sent, but this space acts as a bit bucket. This must be diff --git a/drivers/usb/dwc2/platform.c b/drivers/usb/dwc2/platform.c index 8e1728b39a497e..62a4bb259d8320 100644 --- a/drivers/usb/dwc2/platform.c +++ b/drivers/usb/dwc2/platform.c @@ -46,6 +46,7 @@ #include #include #include +#include #include @@ -85,6 +86,7 @@ static const struct dwc2_core_params params_hi6220 = { .uframe_sched = 0, .external_id_pin_ctl = -1, .hibernation = -1, + .change_speed_quirk = 1, }; static const struct dwc2_core_params params_bcm2835 = { @@ -543,6 +545,7 @@ static int dwc2_driver_probe(struct platform_device *dev) struct dwc2_core_params defparams; struct dwc2_hsotg *hsotg; struct resource *res; + struct extcon_dev *ext_id, *ext_vbus; int retval; match = of_match_device(dwc2_of_match_table, &dev->dev); @@ -620,6 +623,51 @@ static int dwc2_driver_probe(struct platform_device *dev) if (retval) goto error; + ext_id = ERR_PTR(-ENODEV); + ext_vbus = ERR_PTR(-ENODEV); + /* Each one of them is not mandatory */ + ext_vbus = extcon_get_edev_by_phandle(&dev->dev, 0); + if (IS_ERR(ext_vbus)) { + if (PTR_ERR(ext_vbus) != -ENODEV) + return PTR_ERR(ext_vbus); + ext_vbus = NULL; + } + + ext_id = extcon_get_edev_by_phandle(&dev->dev, 1); + if (IS_ERR(ext_id)) { + if (PTR_ERR(ext_id) != -ENODEV) + return PTR_ERR(ext_id); + ext_id = NULL; + } + + hsotg->extcon_vbus.extcon = ext_vbus; + if (ext_vbus) { + hsotg->extcon_vbus.nb.notifier_call = dwc2_extcon_vbus_notifier; + retval = devm_extcon_register_notifier(&dev->dev, ext_vbus, + EXTCON_USB, + &hsotg->extcon_vbus.nb); + if (retval < 0) { + dev_err(&dev->dev, "register VBUS notifier failed\n"); + return retval; + } + hsotg->extcon_vbus.state = extcon_get_state(ext_vbus, + EXTCON_USB); + } + + hsotg->extcon_id.extcon = ext_id; + if (ext_id) { + hsotg->extcon_id.nb.notifier_call = dwc2_extcon_id_notifier; + retval = devm_extcon_register_notifier(&dev->dev, ext_id, + EXTCON_USB_HOST, + &hsotg->extcon_id.nb); + if (retval < 0) { + dev_err(&dev->dev, "register ID notifier failed\n"); + return retval; + } + hsotg->extcon_id.state = extcon_get_state(ext_id, + EXTCON_USB_HOST); + } + /* * Reset before dwc2_get_hwparams() then it could get power-on real * reset value form registers. diff --git a/drivers/usb/dwc3/core.h b/drivers/usb/dwc3/core.h index 6b60e42626a235..884c43714456d5 100644 --- a/drivers/usb/dwc3/core.h +++ b/drivers/usb/dwc3/core.h @@ -43,9 +43,7 @@ #define DWC3_XHCI_RESOURCES_NUM 2 #define DWC3_SCRATCHBUF_SIZE 4096 /* each buffer is assumed to be 4KiB */ -#define DWC3_EVENT_SIZE 4 /* bytes */ -#define DWC3_EVENT_MAX_NUM 64 /* 2 events/endpoint */ -#define DWC3_EVENT_BUFFERS_SIZE (DWC3_EVENT_SIZE * DWC3_EVENT_MAX_NUM) +#define DWC3_EVENT_BUFFERS_SIZE 4096 #define DWC3_EVENT_TYPE_MASK 0xfe #define DWC3_EVENT_TYPE_DEV 0 @@ -303,9 +301,8 @@ #define DWC3_DCFG_SUPERSPEED_PLUS (5 << 0) /* DWC_usb31 only */ #define DWC3_DCFG_SUPERSPEED (4 << 0) #define DWC3_DCFG_HIGHSPEED (0 << 0) -#define DWC3_DCFG_FULLSPEED2 (1 << 0) +#define DWC3_DCFG_FULLSPEED (1 << 0) #define DWC3_DCFG_LOWSPEED (2 << 0) -#define DWC3_DCFG_FULLSPEED1 (3 << 0) #define DWC3_DCFG_NUMP_SHIFT 17 #define DWC3_DCFG_NUMP(n) (((n) >> DWC3_DCFG_NUMP_SHIFT) & 0x1f) @@ -397,9 +394,8 @@ #define DWC3_DSTS_SUPERSPEED_PLUS (5 << 0) /* DWC_usb31 only */ #define DWC3_DSTS_SUPERSPEED (4 << 0) #define DWC3_DSTS_HIGHSPEED (0 << 0) -#define DWC3_DSTS_FULLSPEED2 (1 << 0) +#define DWC3_DSTS_FULLSPEED (1 << 0) #define DWC3_DSTS_LOWSPEED (2 << 0) -#define DWC3_DSTS_FULLSPEED1 (3 << 0) /* Device Generic Command Register */ #define DWC3_DGCMD_SET_LMP 0x01 diff --git a/drivers/usb/dwc3/dwc3-pci.c b/drivers/usb/dwc3/dwc3-pci.c index 6df0f5dad9a4cd..427291a19e6dee 100644 --- a/drivers/usb/dwc3/dwc3-pci.c +++ b/drivers/usb/dwc3/dwc3-pci.c @@ -38,6 +38,7 @@ #define PCI_DEVICE_ID_INTEL_BXT_M 0x1aaa #define PCI_DEVICE_ID_INTEL_APL 0x5aaa #define PCI_DEVICE_ID_INTEL_KBP 0xa2b0 +#define PCI_DEVICE_ID_INTEL_GLK 0x31aa static const struct acpi_gpio_params reset_gpios = { 0, 0, false }; static const struct acpi_gpio_params cs_gpios = { 1, 0, false }; @@ -81,7 +82,7 @@ static int dwc3_pci_quirks(struct pci_dev *pdev, struct platform_device *dwc3) int ret; struct property_entry properties[] = { - PROPERTY_ENTRY_STRING("dr-mode", "peripheral"), + PROPERTY_ENTRY_STRING("dr_mode", "peripheral"), { } }; @@ -229,6 +230,7 @@ static const struct pci_device_id dwc3_pci_id_table[] = { { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_BXT_M), }, { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_APL), }, { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_KBP), }, + { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_GLK), }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_NL_USB), }, { } /* Terminating Entry */ }; diff --git a/drivers/usb/dwc3/ep0.c b/drivers/usb/dwc3/ep0.c index fe79d771dee4c4..2331469f943d2a 100644 --- a/drivers/usb/dwc3/ep0.c +++ b/drivers/usb/dwc3/ep0.c @@ -55,20 +55,13 @@ static const char *dwc3_ep0_state_string(enum dwc3_ep0_state state) } } -static int dwc3_ep0_start_trans(struct dwc3 *dwc, u8 epnum, dma_addr_t buf_dma, - u32 len, u32 type, bool chain) +static void dwc3_ep0_prepare_one_trb(struct dwc3 *dwc, u8 epnum, + dma_addr_t buf_dma, u32 len, u32 type, bool chain) { - struct dwc3_gadget_ep_cmd_params params; struct dwc3_trb *trb; struct dwc3_ep *dep; - int ret; - dep = dwc->eps[epnum]; - if (dep->flags & DWC3_EP_BUSY) { - dwc3_trace(trace_dwc3_ep0, "%s still busy", dep->name); - return 0; - } trb = &dwc->ep0_trb[dep->trb_enqueue]; @@ -89,15 +82,25 @@ static int dwc3_ep0_start_trans(struct dwc3 *dwc, u8 epnum, dma_addr_t buf_dma, trb->ctrl |= (DWC3_TRB_CTRL_IOC | DWC3_TRB_CTRL_LST); - if (chain) + trace_dwc3_prepare_trb(dep, trb); +} + +static int dwc3_ep0_start_trans(struct dwc3 *dwc, u8 epnum) +{ + struct dwc3_gadget_ep_cmd_params params; + struct dwc3_ep *dep; + int ret; + + dep = dwc->eps[epnum]; + if (dep->flags & DWC3_EP_BUSY) { + dwc3_trace(trace_dwc3_ep0, "%s still busy", dep->name); return 0; + } memset(¶ms, 0, sizeof(params)); params.param0 = upper_32_bits(dwc->ep0_trb_addr); params.param1 = lower_32_bits(dwc->ep0_trb_addr); - trace_dwc3_prepare_trb(dep, trb); - ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_STARTTRANSFER, ¶ms); if (ret < 0) { dwc3_trace(trace_dwc3_ep0, "%s STARTTRANSFER failed", @@ -308,8 +311,9 @@ void dwc3_ep0_out_start(struct dwc3 *dwc) { int ret; - ret = dwc3_ep0_start_trans(dwc, 0, dwc->ctrl_req_addr, 8, + dwc3_ep0_prepare_one_trb(dwc, 0, dwc->ctrl_req_addr, 8, DWC3_TRBCTL_CONTROL_SETUP, false); + ret = dwc3_ep0_start_trans(dwc, 0); WARN_ON(ret < 0); } @@ -880,9 +884,9 @@ static void dwc3_ep0_complete_data(struct dwc3 *dwc, dwc->ep0_next_event = DWC3_EP0_COMPLETE; - ret = dwc3_ep0_start_trans(dwc, epnum, - dwc->ctrl_req_addr, 0, - DWC3_TRBCTL_CONTROL_DATA, false); + dwc3_ep0_prepare_one_trb(dwc, epnum, dwc->ctrl_req_addr, + 0, DWC3_TRBCTL_CONTROL_DATA, false); + ret = dwc3_ep0_start_trans(dwc, epnum); WARN_ON(ret < 0); } } @@ -966,9 +970,10 @@ static void __dwc3_ep0_do_control_data(struct dwc3 *dwc, req->direction = !!dep->number; if (req->request.length == 0) { - ret = dwc3_ep0_start_trans(dwc, dep->number, + dwc3_ep0_prepare_one_trb(dwc, dep->number, dwc->ctrl_req_addr, 0, DWC3_TRBCTL_CONTROL_DATA, false); + ret = dwc3_ep0_start_trans(dwc, dep->number); } else if (!IS_ALIGNED(req->request.length, dep->endpoint.maxpacket) && (dep->number == 0)) { u32 transfer_size = 0; @@ -986,7 +991,7 @@ static void __dwc3_ep0_do_control_data(struct dwc3 *dwc, if (req->request.length > DWC3_EP0_BOUNCE_SIZE) { transfer_size = ALIGN(req->request.length - maxpacket, maxpacket); - ret = dwc3_ep0_start_trans(dwc, dep->number, + dwc3_ep0_prepare_one_trb(dwc, dep->number, req->request.dma, transfer_size, DWC3_TRBCTL_CONTROL_DATA, @@ -998,9 +1003,10 @@ static void __dwc3_ep0_do_control_data(struct dwc3 *dwc, dwc->ep0_bounced = true; - ret = dwc3_ep0_start_trans(dwc, dep->number, + dwc3_ep0_prepare_one_trb(dwc, dep->number, dwc->ep0_bounce_addr, transfer_size, DWC3_TRBCTL_CONTROL_DATA, false); + ret = dwc3_ep0_start_trans(dwc, dep->number); } else { ret = usb_gadget_map_request(&dwc->gadget, &req->request, dep->number); @@ -1009,9 +1015,10 @@ static void __dwc3_ep0_do_control_data(struct dwc3 *dwc, return; } - ret = dwc3_ep0_start_trans(dwc, dep->number, req->request.dma, + dwc3_ep0_prepare_one_trb(dwc, dep->number, req->request.dma, req->request.length, DWC3_TRBCTL_CONTROL_DATA, false); + ret = dwc3_ep0_start_trans(dwc, dep->number); } WARN_ON(ret < 0); @@ -1025,8 +1032,9 @@ static int dwc3_ep0_start_control_status(struct dwc3_ep *dep) type = dwc->three_stage_setup ? DWC3_TRBCTL_CONTROL_STATUS3 : DWC3_TRBCTL_CONTROL_STATUS2; - return dwc3_ep0_start_trans(dwc, dep->number, + dwc3_ep0_prepare_one_trb(dwc, dep->number, dwc->ctrl_req_addr, 0, type, false); + return dwc3_ep0_start_trans(dwc, dep->number); } static void __dwc3_ep0_do_control_status(struct dwc3 *dwc, struct dwc3_ep *dep) diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c index 1dfa56a5f1c511..d2b860ebfe13bd 100644 --- a/drivers/usb/dwc3/gadget.c +++ b/drivers/usb/dwc3/gadget.c @@ -182,11 +182,11 @@ void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req, if (req->request.status == -EINPROGRESS) req->request.status = status; - if (dwc->ep0_bounced && dep->number == 0) + if (dwc->ep0_bounced && dep->number <= 1) dwc->ep0_bounced = false; - else - usb_gadget_unmap_request(&dwc->gadget, &req->request, - req->direction); + + usb_gadget_unmap_request(&dwc->gadget, &req->request, + req->direction); trace_dwc3_gadget_giveback(req); @@ -771,6 +771,9 @@ static void dwc3_prepare_one_trb(struct dwc3_ep *dep, unsigned length, unsigned chain, unsigned node) { struct dwc3_trb *trb; + struct dwc3 *dwc = dep->dwc; + struct usb_gadget *gadget = &dwc->gadget; + enum usb_device_speed speed = gadget->speed; dwc3_trace(trace_dwc3_gadget, "%s: req %p dma %08llx length %d%s", dep->name, req, (unsigned long long) dma, @@ -798,10 +801,16 @@ static void dwc3_prepare_one_trb(struct dwc3_ep *dep, break; case USB_ENDPOINT_XFER_ISOC: - if (!node) + if (!node) { trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST; - else + + if (speed == USB_SPEED_HIGH) { + struct usb_ep *ep = &dep->endpoint; + trb->size |= DWC3_TRB_SIZE_PCM1(ep->mult - 1); + } + } else { trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS; + } /* always enable Interrupt on Missed ISOC */ trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI; @@ -1597,7 +1606,7 @@ static int __dwc3_gadget_start(struct dwc3 *dwc) reg |= DWC3_DCFG_LOWSPEED; break; case USB_SPEED_FULL: - reg |= DWC3_DCFG_FULLSPEED1; + reg |= DWC3_DCFG_FULLSPEED; break; case USB_SPEED_HIGH: reg |= DWC3_DCFG_HIGHSPEED; @@ -2456,8 +2465,7 @@ static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc) dwc->gadget.ep0->maxpacket = 64; dwc->gadget.speed = USB_SPEED_HIGH; break; - case DWC3_DSTS_FULLSPEED2: - case DWC3_DSTS_FULLSPEED1: + case DWC3_DSTS_FULLSPEED: dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64); dwc->gadget.ep0->maxpacket = 64; dwc->gadget.speed = USB_SPEED_FULL; diff --git a/drivers/usb/gadget/Kconfig b/drivers/usb/gadget/Kconfig index 8ad203296079f6..3684775677eb8c 100644 --- a/drivers/usb/gadget/Kconfig +++ b/drivers/usb/gadget/Kconfig @@ -209,6 +209,18 @@ config USB_F_PRINTER config USB_F_TCM tristate +config USB_F_MTP + tristate + +config USB_F_PTP + tristate + +config USB_F_AUDIO_SRC + tristate + +config USB_F_ACC + tristate + # this first set of drivers all depend on bulk-capable hardware. config USB_CONFIGFS @@ -362,6 +374,44 @@ config USB_CONFIGFS_F_FS implemented in kernel space (for instance Ethernet, serial or mass storage) and other are implemented in user space. +config USB_CONFIGFS_F_MTP + boolean "MTP gadget" + depends on USB_CONFIGFS + select USB_F_MTP + help + USB gadget MTP support + +config USB_CONFIGFS_F_PTP + boolean "PTP gadget" + depends on USB_CONFIGFS && USB_CONFIGFS_F_MTP + select USB_F_PTP + help + USB gadget PTP support + +config USB_CONFIGFS_F_ACC + boolean "Accessory gadget" + depends on USB_CONFIGFS + select USB_F_ACC + help + USB gadget Accessory support + +config USB_CONFIGFS_F_AUDIO_SRC + boolean "Audio Source gadget" + depends on USB_CONFIGFS && USB_CONFIGFS_F_ACC + depends on SND + select SND_PCM + select USB_F_AUDIO_SRC + help + USB gadget Audio Source support + +config USB_CONFIGFS_UEVENT + boolean "Uevent notification of Gadget state" + depends on USB_CONFIGFS + help + Enable uevent notifications to userspace when the gadget + state changes. The gadget can be in any of the following + three states: "CONNECTED/DISCONNECTED/CONFIGURED" + config USB_CONFIGFS_F_UAC1 bool "Audio Class 1.0" depends on USB_CONFIGFS diff --git a/drivers/usb/gadget/composite.c b/drivers/usb/gadget/composite.c index 32176f77986162..deece23dc58b65 100644 --- a/drivers/usb/gadget/composite.c +++ b/drivers/usb/gadget/composite.c @@ -197,11 +197,16 @@ int config_ep_by_speed(struct usb_gadget *g, ep_found: /* commit results */ - _ep->maxpacket = usb_endpoint_maxp(chosen_desc); + _ep->maxpacket = usb_endpoint_maxp(chosen_desc) & 0x7ff; _ep->desc = chosen_desc; _ep->comp_desc = NULL; _ep->maxburst = 0; - _ep->mult = 0; + _ep->mult = 1; + + if (g->speed == USB_SPEED_HIGH && (usb_endpoint_xfer_isoc(_ep->desc) || + usb_endpoint_xfer_int(_ep->desc))) + _ep->mult = ((usb_endpoint_maxp(_ep->desc) & 0x1800) >> 11) + 1; + if (!want_comp_desc) return 0; @@ -218,7 +223,7 @@ int config_ep_by_speed(struct usb_gadget *g, switch (usb_endpoint_type(_ep->desc)) { case USB_ENDPOINT_XFER_ISOC: /* mult: bits 1:0 of bmAttributes */ - _ep->mult = comp_desc->bmAttributes & 0x3; + _ep->mult = (comp_desc->bmAttributes & 0x3) + 1; case USB_ENDPOINT_XFER_BULK: case USB_ENDPOINT_XFER_INT: _ep->maxburst = comp_desc->bMaxBurst + 1; @@ -1689,9 +1694,7 @@ composite_setup(struct usb_gadget *gadget, const struct usb_ctrlrequest *ctrl) value = min(w_length, (u16) 1); break; - /* function drivers must handle get/set altsetting; if there's - * no get() method, we know only altsetting zero works. - */ + /* function drivers must handle get/set altsetting */ case USB_REQ_SET_INTERFACE: if (ctrl->bRequestType != USB_RECIP_INTERFACE) goto unknown; @@ -1700,7 +1703,13 @@ composite_setup(struct usb_gadget *gadget, const struct usb_ctrlrequest *ctrl) f = cdev->config->interface[intf]; if (!f) break; - if (w_value && !f->set_alt) + + /* + * If there's no get_alt() method, we know only altsetting zero + * works. There is no need to check if set_alt() is not NULL + * as we check this in usb_add_function(). + */ + if (w_value && !f->get_alt) break; value = f->set_alt(f, w_index, w_value); if (value == USB_GADGET_DELAYED_STATUS) { @@ -1983,6 +1992,12 @@ void composite_disconnect(struct usb_gadget *gadget) struct usb_composite_dev *cdev = get_gadget_data(gadget); unsigned long flags; + if (cdev == NULL) { + WARN(1, "%s: Calling disconnect on a Gadget that is \ + not connected\n", __func__); + return; + } + /* REVISIT: should we have config and device level * disconnect callbacks? */ diff --git a/drivers/usb/gadget/configfs.c b/drivers/usb/gadget/configfs.c index 3984787f8e9720..7461e644a8a998 100644 --- a/drivers/usb/gadget/configfs.c +++ b/drivers/usb/gadget/configfs.c @@ -9,6 +9,31 @@ #include "u_f.h" #include "u_os_desc.h" +#ifdef CONFIG_USB_CONFIGFS_UEVENT +#include +#include +#include + +#ifdef CONFIG_USB_CONFIGFS_F_ACC +extern int acc_ctrlrequest(struct usb_composite_dev *cdev, + const struct usb_ctrlrequest *ctrl); +void acc_disconnect(void); +#endif +static struct class *android_class; +static struct device *android_device; +static int index; + +struct device *create_function_device(char *name) +{ + if (android_device && !IS_ERR(android_device)) + return device_create(android_class, android_device, + MKDEV(0, index++), NULL, name); + else + return ERR_PTR(-EINVAL); +} +EXPORT_SYMBOL_GPL(create_function_device); +#endif + int check_user_usb_string(const char *name, struct usb_gadget_strings *stringtab_dev) { @@ -60,6 +85,12 @@ struct gadget_info { bool use_os_desc; char b_vendor_code; char qw_sign[OS_STRING_QW_SIGN_LEN]; +#ifdef CONFIG_USB_CONFIGFS_UEVENT + bool connected; + bool sw_connected; + struct work_struct work; + struct device *dev; +#endif }; static inline struct gadget_info *to_gadget_info(struct config_item *item) @@ -265,7 +296,7 @@ static ssize_t gadget_dev_desc_UDC_store(struct config_item *item, mutex_lock(&gi->lock); - if (!strlen(name)) { + if (!strlen(name) || strcmp(name, "none") == 0) { ret = unregister_gadget(gi); if (ret) goto err; @@ -1367,6 +1398,60 @@ static int configfs_composite_bind(struct usb_gadget *gadget, return ret; } +#ifdef CONFIG_USB_CONFIGFS_UEVENT +static void android_work(struct work_struct *data) +{ + struct gadget_info *gi = container_of(data, struct gadget_info, work); + struct usb_composite_dev *cdev = &gi->cdev; + char *disconnected[2] = { "USB_STATE=DISCONNECTED", NULL }; + char *connected[2] = { "USB_STATE=CONNECTED", NULL }; + char *configured[2] = { "USB_STATE=CONFIGURED", NULL }; + /* 0-connected 1-configured 2-disconnected*/ + bool status[3] = { false, false, false }; + unsigned long flags; + bool uevent_sent = false; + + spin_lock_irqsave(&cdev->lock, flags); + if (cdev->config) + status[1] = true; + + if (gi->connected != gi->sw_connected) { + if (gi->connected) + status[0] = true; + else + status[2] = true; + gi->sw_connected = gi->connected; + } + spin_unlock_irqrestore(&cdev->lock, flags); + + if (status[0]) { + kobject_uevent_env(&android_device->kobj, + KOBJ_CHANGE, connected); + pr_info("%s: sent uevent %s\n", __func__, connected[0]); + uevent_sent = true; + } + + if (status[1]) { + kobject_uevent_env(&android_device->kobj, + KOBJ_CHANGE, configured); + pr_info("%s: sent uevent %s\n", __func__, configured[0]); + uevent_sent = true; + } + + if (status[2]) { + kobject_uevent_env(&android_device->kobj, + KOBJ_CHANGE, disconnected); + pr_info("%s: sent uevent %s\n", __func__, disconnected[0]); + uevent_sent = true; + } + + if (!uevent_sent) { + pr_info("%s: did not send uevent (%d %d %p)\n", __func__, + gi->connected, gi->sw_connected, cdev->config); + } +} +#endif + static void configfs_composite_unbind(struct usb_gadget *gadget) { struct usb_composite_dev *cdev; @@ -1386,14 +1471,79 @@ static void configfs_composite_unbind(struct usb_gadget *gadget) set_gadget_data(gadget, NULL); } +#ifdef CONFIG_USB_CONFIGFS_UEVENT +static int android_setup(struct usb_gadget *gadget, + const struct usb_ctrlrequest *c) +{ + struct usb_composite_dev *cdev = get_gadget_data(gadget); + unsigned long flags; + struct gadget_info *gi = container_of(cdev, struct gadget_info, cdev); + int value = -EOPNOTSUPP; + struct usb_function_instance *fi; + + spin_lock_irqsave(&cdev->lock, flags); + if (!gi->connected) { + gi->connected = 1; + schedule_work(&gi->work); + } + spin_unlock_irqrestore(&cdev->lock, flags); + list_for_each_entry(fi, &gi->available_func, cfs_list) { + if (fi != NULL && fi->f != NULL && fi->f->setup != NULL) { + value = fi->f->setup(fi->f, c); + if (value >= 0) + break; + } + } + +#ifdef CONFIG_USB_CONFIGFS_F_ACC + if (value < 0) + value = acc_ctrlrequest(cdev, c); +#endif + + if (value < 0) + value = composite_setup(gadget, c); + + spin_lock_irqsave(&cdev->lock, flags); + if (c->bRequest == USB_REQ_SET_CONFIGURATION && + cdev->config) { + schedule_work(&gi->work); + } + spin_unlock_irqrestore(&cdev->lock, flags); + + return value; +} + +static void android_disconnect(struct usb_gadget *gadget) +{ + struct usb_composite_dev *cdev = get_gadget_data(gadget); + struct gadget_info *gi = container_of(cdev, struct gadget_info, cdev); + + /* accessory HID support can be active while the + accessory function is not actually enabled, + so we need to inform it when we are disconnected. + */ + +#ifdef CONFIG_USB_CONFIGFS_F_ACC + acc_disconnect(); +#endif + gi->connected = 0; + schedule_work(&gi->work); + composite_disconnect(gadget); +} +#endif + static const struct usb_gadget_driver configfs_driver_template = { .bind = configfs_composite_bind, .unbind = configfs_composite_unbind, - +#ifdef CONFIG_USB_CONFIGFS_UEVENT + .setup = android_setup, + .reset = android_disconnect, + .disconnect = android_disconnect, +#else .setup = composite_setup, .reset = composite_disconnect, .disconnect = composite_disconnect, - +#endif .suspend = composite_suspend, .resume = composite_resume, @@ -1405,6 +1555,89 @@ static const struct usb_gadget_driver configfs_driver_template = { .match_existing_only = 1, }; +#ifdef CONFIG_USB_CONFIGFS_UEVENT +static ssize_t state_show(struct device *pdev, struct device_attribute *attr, + char *buf) +{ + struct gadget_info *dev = dev_get_drvdata(pdev); + struct usb_composite_dev *cdev; + char *state = "DISCONNECTED"; + unsigned long flags; + + if (!dev) + goto out; + + cdev = &dev->cdev; + + if (!cdev) + goto out; + + spin_lock_irqsave(&cdev->lock, flags); + if (cdev->config) + state = "CONFIGURED"; + else if (dev->connected) + state = "CONNECTED"; + spin_unlock_irqrestore(&cdev->lock, flags); +out: + return sprintf(buf, "%s\n", state); +} + +static DEVICE_ATTR(state, S_IRUGO, state_show, NULL); + +static struct device_attribute *android_usb_attributes[] = { + &dev_attr_state, + NULL +}; + +static int android_device_create(struct gadget_info *gi) +{ + struct device_attribute **attrs; + struct device_attribute *attr; + + INIT_WORK(&gi->work, android_work); + android_device = device_create(android_class, NULL, + MKDEV(0, 0), NULL, "android0"); + if (IS_ERR(android_device)) + return PTR_ERR(android_device); + + dev_set_drvdata(android_device, gi); + + attrs = android_usb_attributes; + while ((attr = *attrs++)) { + int err; + + err = device_create_file(android_device, attr); + if (err) { + device_destroy(android_device->class, + android_device->devt); + return err; + } + } + + return 0; +} + +static void android_device_destroy(void) +{ + struct device_attribute **attrs; + struct device_attribute *attr; + + attrs = android_usb_attributes; + while ((attr = *attrs++)) + device_remove_file(android_device, attr); + device_destroy(android_device->class, android_device->devt); +} +#else +static inline int android_device_create(struct gadget_info *gi) +{ + return 0; +} + +static inline void android_device_destroy(void) +{ +} +#endif + static struct config_group *gadgets_make( struct config_group *group, const char *name) @@ -1456,7 +1689,11 @@ static struct config_group *gadgets_make( if (!gi->composite.gadget_driver.function) goto err; + if (android_device_create(gi) < 0) + goto err; + return &gi->group; + err: kfree(gi); return ERR_PTR(-ENOMEM); @@ -1465,6 +1702,7 @@ static struct config_group *gadgets_make( static void gadgets_drop(struct config_group *group, struct config_item *item) { config_item_put(item); + android_device_destroy(); } static struct configfs_group_operations gadgets_ops = { @@ -1504,6 +1742,13 @@ static int __init gadget_cfs_init(void) config_group_init(&gadget_subsys.su_group); ret = configfs_register_subsystem(&gadget_subsys); + +#ifdef CONFIG_USB_CONFIGFS_UEVENT + android_class = class_create(THIS_MODULE, "android_usb"); + if (IS_ERR(android_class)) + return PTR_ERR(android_class); +#endif + return ret; } module_init(gadget_cfs_init); @@ -1511,5 +1756,10 @@ module_init(gadget_cfs_init); static void __exit gadget_cfs_exit(void) { configfs_unregister_subsystem(&gadget_subsys); +#ifdef CONFIG_USB_CONFIGFS_UEVENT + if (!IS_ERR(android_class)) + class_destroy(android_class); +#endif + } module_exit(gadget_cfs_exit); diff --git a/drivers/usb/gadget/function/Makefile b/drivers/usb/gadget/function/Makefile index cb8c225e854925..78682d5e4dc786 100644 --- a/drivers/usb/gadget/function/Makefile +++ b/drivers/usb/gadget/function/Makefile @@ -46,3 +46,11 @@ usb_f_printer-y := f_printer.o obj-$(CONFIG_USB_F_PRINTER) += usb_f_printer.o usb_f_tcm-y := f_tcm.o obj-$(CONFIG_USB_F_TCM) += usb_f_tcm.o +usb_f_mtp-y := f_mtp.o +obj-$(CONFIG_USB_F_MTP) += usb_f_mtp.o +usb_f_ptp-y := f_ptp.o +obj-$(CONFIG_USB_F_PTP) += usb_f_ptp.o +usb_f_audio_source-y := f_audio_source.o +obj-$(CONFIG_USB_F_AUDIO_SRC) += usb_f_audio_source.o +usb_f_accessory-y := f_accessory.o +obj-$(CONFIG_USB_F_ACC) += usb_f_accessory.o diff --git a/drivers/usb/gadget/function/f_accessory.c b/drivers/usb/gadget/function/f_accessory.c new file mode 100644 index 00000000000000..9d3ec0e3747561 --- /dev/null +++ b/drivers/usb/gadget/function/f_accessory.c @@ -0,0 +1,1328 @@ +/* + * Gadget Function Driver for Android USB accessories + * + * Copyright (C) 2011 Google, Inc. + * Author: Mike Lockwood + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +/* #define DEBUG */ +/* #define VERBOSE_DEBUG */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +#define MAX_INST_NAME_LEN 40 +#define BULK_BUFFER_SIZE 16384 +#define ACC_STRING_SIZE 256 + +#define PROTOCOL_VERSION 2 + +/* String IDs */ +#define INTERFACE_STRING_INDEX 0 + +/* number of tx and rx requests to allocate */ +#define TX_REQ_MAX 4 +#define RX_REQ_MAX 2 + +struct acc_hid_dev { + struct list_head list; + struct hid_device *hid; + struct acc_dev *dev; + /* accessory defined ID */ + int id; + /* HID report descriptor */ + u8 *report_desc; + /* length of HID report descriptor */ + int report_desc_len; + /* number of bytes of report_desc we have received so far */ + int report_desc_offset; +}; + +struct acc_dev { + struct usb_function function; + struct usb_composite_dev *cdev; + spinlock_t lock; + + struct usb_ep *ep_in; + struct usb_ep *ep_out; + + /* set to 1 when we connect */ + int online:1; + /* Set to 1 when we disconnect. + * Not cleared until our file is closed. + */ + int disconnected:1; + + /* strings sent by the host */ + char manufacturer[ACC_STRING_SIZE]; + char model[ACC_STRING_SIZE]; + char description[ACC_STRING_SIZE]; + char version[ACC_STRING_SIZE]; + char uri[ACC_STRING_SIZE]; + char serial[ACC_STRING_SIZE]; + + /* for acc_complete_set_string */ + int string_index; + + /* set to 1 if we have a pending start request */ + int start_requested; + + int audio_mode; + + /* synchronize access to our device file */ + atomic_t open_excl; + + struct list_head tx_idle; + + wait_queue_head_t read_wq; + wait_queue_head_t write_wq; + struct usb_request *rx_req[RX_REQ_MAX]; + int rx_done; + + /* delayed work for handling ACCESSORY_START */ + struct delayed_work start_work; + + /* worker for registering and unregistering hid devices */ + struct work_struct hid_work; + + /* list of active HID devices */ + struct list_head hid_list; + + /* list of new HID devices to register */ + struct list_head new_hid_list; + + /* list of dead HID devices to unregister */ + struct list_head dead_hid_list; +}; + +static struct usb_interface_descriptor acc_interface_desc = { + .bLength = USB_DT_INTERFACE_SIZE, + .bDescriptorType = USB_DT_INTERFACE, + .bInterfaceNumber = 0, + .bNumEndpoints = 2, + .bInterfaceClass = USB_CLASS_VENDOR_SPEC, + .bInterfaceSubClass = USB_SUBCLASS_VENDOR_SPEC, + .bInterfaceProtocol = 0, +}; + +static struct usb_endpoint_descriptor acc_highspeed_in_desc = { + .bLength = USB_DT_ENDPOINT_SIZE, + .bDescriptorType = USB_DT_ENDPOINT, + .bEndpointAddress = USB_DIR_IN, + .bmAttributes = USB_ENDPOINT_XFER_BULK, + .wMaxPacketSize = __constant_cpu_to_le16(512), +}; + +static struct usb_endpoint_descriptor acc_highspeed_out_desc = { + .bLength = USB_DT_ENDPOINT_SIZE, + .bDescriptorType = USB_DT_ENDPOINT, + .bEndpointAddress = USB_DIR_OUT, + .bmAttributes = USB_ENDPOINT_XFER_BULK, + .wMaxPacketSize = __constant_cpu_to_le16(512), +}; + +static struct usb_endpoint_descriptor acc_fullspeed_in_desc = { + .bLength = USB_DT_ENDPOINT_SIZE, + .bDescriptorType = USB_DT_ENDPOINT, + .bEndpointAddress = USB_DIR_IN, + .bmAttributes = USB_ENDPOINT_XFER_BULK, +}; + +static struct usb_endpoint_descriptor acc_fullspeed_out_desc = { + .bLength = USB_DT_ENDPOINT_SIZE, + .bDescriptorType = USB_DT_ENDPOINT, + .bEndpointAddress = USB_DIR_OUT, + .bmAttributes = USB_ENDPOINT_XFER_BULK, +}; + +static struct usb_descriptor_header *fs_acc_descs[] = { + (struct usb_descriptor_header *) &acc_interface_desc, + (struct usb_descriptor_header *) &acc_fullspeed_in_desc, + (struct usb_descriptor_header *) &acc_fullspeed_out_desc, + NULL, +}; + +static struct usb_descriptor_header *hs_acc_descs[] = { + (struct usb_descriptor_header *) &acc_interface_desc, + (struct usb_descriptor_header *) &acc_highspeed_in_desc, + (struct usb_descriptor_header *) &acc_highspeed_out_desc, + NULL, +}; + +static struct usb_string acc_string_defs[] = { + [INTERFACE_STRING_INDEX].s = "Android Accessory Interface", + { }, /* end of list */ +}; + +static struct usb_gadget_strings acc_string_table = { + .language = 0x0409, /* en-US */ + .strings = acc_string_defs, +}; + +static struct usb_gadget_strings *acc_strings[] = { + &acc_string_table, + NULL, +}; + +/* temporary variable used between acc_open() and acc_gadget_bind() */ +static struct acc_dev *_acc_dev; + +struct acc_instance { + struct usb_function_instance func_inst; + const char *name; +}; + +static inline struct acc_dev *func_to_dev(struct usb_function *f) +{ + return container_of(f, struct acc_dev, function); +} + +static struct usb_request *acc_request_new(struct usb_ep *ep, int buffer_size) +{ + struct usb_request *req = usb_ep_alloc_request(ep, GFP_KERNEL); + + if (!req) + return NULL; + + /* now allocate buffers for the requests */ + req->buf = kmalloc(buffer_size, GFP_KERNEL); + if (!req->buf) { + usb_ep_free_request(ep, req); + return NULL; + } + + return req; +} + +static void acc_request_free(struct usb_request *req, struct usb_ep *ep) +{ + if (req) { + kfree(req->buf); + usb_ep_free_request(ep, req); + } +} + +/* add a request to the tail of a list */ +static void req_put(struct acc_dev *dev, struct list_head *head, + struct usb_request *req) +{ + unsigned long flags; + + spin_lock_irqsave(&dev->lock, flags); + list_add_tail(&req->list, head); + spin_unlock_irqrestore(&dev->lock, flags); +} + +/* remove a request from the head of a list */ +static struct usb_request *req_get(struct acc_dev *dev, struct list_head *head) +{ + unsigned long flags; + struct usb_request *req; + + spin_lock_irqsave(&dev->lock, flags); + if (list_empty(head)) { + req = 0; + } else { + req = list_first_entry(head, struct usb_request, list); + list_del(&req->list); + } + spin_unlock_irqrestore(&dev->lock, flags); + return req; +} + +static void acc_set_disconnected(struct acc_dev *dev) +{ + dev->online = 0; + dev->disconnected = 1; +} + +static void acc_complete_in(struct usb_ep *ep, struct usb_request *req) +{ + struct acc_dev *dev = _acc_dev; + + if (req->status == -ESHUTDOWN) { + pr_debug("acc_complete_in set disconnected"); + acc_set_disconnected(dev); + } + + req_put(dev, &dev->tx_idle, req); + + wake_up(&dev->write_wq); +} + +static void acc_complete_out(struct usb_ep *ep, struct usb_request *req) +{ + struct acc_dev *dev = _acc_dev; + + dev->rx_done = 1; + if (req->status == -ESHUTDOWN) { + pr_debug("acc_complete_out set disconnected"); + acc_set_disconnected(dev); + } + + wake_up(&dev->read_wq); +} + +static void acc_complete_set_string(struct usb_ep *ep, struct usb_request *req) +{ + struct acc_dev *dev = ep->driver_data; + char *string_dest = NULL; + int length = req->actual; + + if (req->status != 0) { + pr_err("acc_complete_set_string, err %d\n", req->status); + return; + } + + switch (dev->string_index) { + case ACCESSORY_STRING_MANUFACTURER: + string_dest = dev->manufacturer; + break; + case ACCESSORY_STRING_MODEL: + string_dest = dev->model; + break; + case ACCESSORY_STRING_DESCRIPTION: + string_dest = dev->description; + break; + case ACCESSORY_STRING_VERSION: + string_dest = dev->version; + break; + case ACCESSORY_STRING_URI: + string_dest = dev->uri; + break; + case ACCESSORY_STRING_SERIAL: + string_dest = dev->serial; + break; + } + if (string_dest) { + unsigned long flags; + + if (length >= ACC_STRING_SIZE) + length = ACC_STRING_SIZE - 1; + + spin_lock_irqsave(&dev->lock, flags); + memcpy(string_dest, req->buf, length); + /* ensure zero termination */ + string_dest[length] = 0; + spin_unlock_irqrestore(&dev->lock, flags); + } else { + pr_err("unknown accessory string index %d\n", + dev->string_index); + } +} + +static void acc_complete_set_hid_report_desc(struct usb_ep *ep, + struct usb_request *req) +{ + struct acc_hid_dev *hid = req->context; + struct acc_dev *dev = hid->dev; + int length = req->actual; + + if (req->status != 0) { + pr_err("acc_complete_set_hid_report_desc, err %d\n", + req->status); + return; + } + + memcpy(hid->report_desc + hid->report_desc_offset, req->buf, length); + hid->report_desc_offset += length; + if (hid->report_desc_offset == hid->report_desc_len) { + /* After we have received the entire report descriptor + * we schedule work to initialize the HID device + */ + schedule_work(&dev->hid_work); + } +} + +static void acc_complete_send_hid_event(struct usb_ep *ep, + struct usb_request *req) +{ + struct acc_hid_dev *hid = req->context; + int length = req->actual; + + if (req->status != 0) { + pr_err("acc_complete_send_hid_event, err %d\n", req->status); + return; + } + + hid_report_raw_event(hid->hid, HID_INPUT_REPORT, req->buf, length, 1); +} + +static int acc_hid_parse(struct hid_device *hid) +{ + struct acc_hid_dev *hdev = hid->driver_data; + + hid_parse_report(hid, hdev->report_desc, hdev->report_desc_len); + return 0; +} + +static int acc_hid_start(struct hid_device *hid) +{ + return 0; +} + +static void acc_hid_stop(struct hid_device *hid) +{ +} + +static int acc_hid_open(struct hid_device *hid) +{ + return 0; +} + +static void acc_hid_close(struct hid_device *hid) +{ +} + +static int acc_hid_raw_request(struct hid_device *hid, unsigned char reportnum, + __u8 *buf, size_t len, unsigned char rtype, int reqtype) +{ + return 0; +} + +static struct hid_ll_driver acc_hid_ll_driver = { + .parse = acc_hid_parse, + .start = acc_hid_start, + .stop = acc_hid_stop, + .open = acc_hid_open, + .close = acc_hid_close, + .raw_request = acc_hid_raw_request, +}; + +static struct acc_hid_dev *acc_hid_new(struct acc_dev *dev, + int id, int desc_len) +{ + struct acc_hid_dev *hdev; + + hdev = kzalloc(sizeof(*hdev), GFP_ATOMIC); + if (!hdev) + return NULL; + hdev->report_desc = kzalloc(desc_len, GFP_ATOMIC); + if (!hdev->report_desc) { + kfree(hdev); + return NULL; + } + hdev->dev = dev; + hdev->id = id; + hdev->report_desc_len = desc_len; + + return hdev; +} + +static struct acc_hid_dev *acc_hid_get(struct list_head *list, int id) +{ + struct acc_hid_dev *hid; + + list_for_each_entry(hid, list, list) { + if (hid->id == id) + return hid; + } + return NULL; +} + +static int acc_register_hid(struct acc_dev *dev, int id, int desc_length) +{ + struct acc_hid_dev *hid; + unsigned long flags; + + /* report descriptor length must be > 0 */ + if (desc_length <= 0) + return -EINVAL; + + spin_lock_irqsave(&dev->lock, flags); + /* replace HID if one already exists with this ID */ + hid = acc_hid_get(&dev->hid_list, id); + if (!hid) + hid = acc_hid_get(&dev->new_hid_list, id); + if (hid) + list_move(&hid->list, &dev->dead_hid_list); + + hid = acc_hid_new(dev, id, desc_length); + if (!hid) { + spin_unlock_irqrestore(&dev->lock, flags); + return -ENOMEM; + } + + list_add(&hid->list, &dev->new_hid_list); + spin_unlock_irqrestore(&dev->lock, flags); + + /* schedule work to register the HID device */ + schedule_work(&dev->hid_work); + return 0; +} + +static int acc_unregister_hid(struct acc_dev *dev, int id) +{ + struct acc_hid_dev *hid; + unsigned long flags; + + spin_lock_irqsave(&dev->lock, flags); + hid = acc_hid_get(&dev->hid_list, id); + if (!hid) + hid = acc_hid_get(&dev->new_hid_list, id); + if (!hid) { + spin_unlock_irqrestore(&dev->lock, flags); + return -EINVAL; + } + + list_move(&hid->list, &dev->dead_hid_list); + spin_unlock_irqrestore(&dev->lock, flags); + + schedule_work(&dev->hid_work); + return 0; +} + +static int create_bulk_endpoints(struct acc_dev *dev, + struct usb_endpoint_descriptor *in_desc, + struct usb_endpoint_descriptor *out_desc) +{ + struct usb_composite_dev *cdev = dev->cdev; + struct usb_request *req; + struct usb_ep *ep; + int i; + + DBG(cdev, "create_bulk_endpoints dev: %p\n", dev); + + ep = usb_ep_autoconfig(cdev->gadget, in_desc); + if (!ep) { + DBG(cdev, "usb_ep_autoconfig for ep_in failed\n"); + return -ENODEV; + } + DBG(cdev, "usb_ep_autoconfig for ep_in got %s\n", ep->name); + ep->driver_data = dev; /* claim the endpoint */ + dev->ep_in = ep; + + ep = usb_ep_autoconfig(cdev->gadget, out_desc); + if (!ep) { + DBG(cdev, "usb_ep_autoconfig for ep_out failed\n"); + return -ENODEV; + } + DBG(cdev, "usb_ep_autoconfig for ep_out got %s\n", ep->name); + ep->driver_data = dev; /* claim the endpoint */ + dev->ep_out = ep; + + /* now allocate requests for our endpoints */ + for (i = 0; i < TX_REQ_MAX; i++) { + req = acc_request_new(dev->ep_in, BULK_BUFFER_SIZE); + if (!req) + goto fail; + req->complete = acc_complete_in; + req_put(dev, &dev->tx_idle, req); + } + for (i = 0; i < RX_REQ_MAX; i++) { + req = acc_request_new(dev->ep_out, BULK_BUFFER_SIZE); + if (!req) + goto fail; + req->complete = acc_complete_out; + dev->rx_req[i] = req; + } + + return 0; + +fail: + pr_err("acc_bind() could not allocate requests\n"); + while ((req = req_get(dev, &dev->tx_idle))) + acc_request_free(req, dev->ep_in); + for (i = 0; i < RX_REQ_MAX; i++) + acc_request_free(dev->rx_req[i], dev->ep_out); + return -1; +} + +static ssize_t acc_read(struct file *fp, char __user *buf, + size_t count, loff_t *pos) +{ + struct acc_dev *dev = fp->private_data; + struct usb_request *req; + ssize_t r = count; + unsigned xfer; + int ret = 0; + + pr_debug("acc_read(%zu)\n", count); + + if (dev->disconnected) { + pr_debug("acc_read disconnected"); + return -ENODEV; + } + + if (count > BULK_BUFFER_SIZE) + count = BULK_BUFFER_SIZE; + + /* we will block until we're online */ + pr_debug("acc_read: waiting for online\n"); + ret = wait_event_interruptible(dev->read_wq, dev->online); + if (ret < 0) { + r = ret; + goto done; + } + + if (dev->rx_done) { + // last req cancelled. try to get it. + req = dev->rx_req[0]; + goto copy_data; + } + +requeue_req: + /* queue a request */ + req = dev->rx_req[0]; + req->length = count; + dev->rx_done = 0; + ret = usb_ep_queue(dev->ep_out, req, GFP_KERNEL); + if (ret < 0) { + r = -EIO; + goto done; + } else { + pr_debug("rx %p queue\n", req); + } + + /* wait for a request to complete */ + ret = wait_event_interruptible(dev->read_wq, dev->rx_done); + if (ret < 0) { + r = ret; + ret = usb_ep_dequeue(dev->ep_out, req); + if (ret != 0) { + // cancel failed. There can be a data already received. + // it will be retrieved in the next read. + pr_debug("acc_read: cancelling failed %d", ret); + } + goto done; + } + +copy_data: + dev->rx_done = 0; + if (dev->online) { + /* If we got a 0-len packet, throw it back and try again. */ + if (req->actual == 0) + goto requeue_req; + + pr_debug("rx %p %u\n", req, req->actual); + xfer = (req->actual < count) ? req->actual : count; + r = xfer; + if (copy_to_user(buf, req->buf, xfer)) + r = -EFAULT; + } else + r = -EIO; + +done: + pr_debug("acc_read returning %zd\n", r); + return r; +} + +static ssize_t acc_write(struct file *fp, const char __user *buf, + size_t count, loff_t *pos) +{ + struct acc_dev *dev = fp->private_data; + struct usb_request *req = 0; + ssize_t r = count; + unsigned xfer; + int ret; + + pr_debug("acc_write(%zu)\n", count); + + if (!dev->online || dev->disconnected) { + pr_debug("acc_write disconnected or not online"); + return -ENODEV; + } + + while (count > 0) { + if (!dev->online) { + pr_debug("acc_write dev->error\n"); + r = -EIO; + break; + } + + /* get an idle tx request to use */ + req = 0; + ret = wait_event_interruptible(dev->write_wq, + ((req = req_get(dev, &dev->tx_idle)) || !dev->online)); + if (!req) { + r = ret; + break; + } + + if (count > BULK_BUFFER_SIZE) { + xfer = BULK_BUFFER_SIZE; + /* ZLP, They will be more TX requests so not yet. */ + req->zero = 0; + } else { + xfer = count; + /* If the data length is a multple of the + * maxpacket size then send a zero length packet(ZLP). + */ + req->zero = ((xfer % dev->ep_in->maxpacket) == 0); + } + if (copy_from_user(req->buf, buf, xfer)) { + r = -EFAULT; + break; + } + + req->length = xfer; + ret = usb_ep_queue(dev->ep_in, req, GFP_KERNEL); + if (ret < 0) { + pr_debug("acc_write: xfer error %d\n", ret); + r = -EIO; + break; + } + + buf += xfer; + count -= xfer; + + /* zero this so we don't try to free it on error exit */ + req = 0; + } + + if (req) + req_put(dev, &dev->tx_idle, req); + + pr_debug("acc_write returning %zd\n", r); + return r; +} + +static long acc_ioctl(struct file *fp, unsigned code, unsigned long value) +{ + struct acc_dev *dev = fp->private_data; + char *src = NULL; + int ret; + + switch (code) { + case ACCESSORY_GET_STRING_MANUFACTURER: + src = dev->manufacturer; + break; + case ACCESSORY_GET_STRING_MODEL: + src = dev->model; + break; + case ACCESSORY_GET_STRING_DESCRIPTION: + src = dev->description; + break; + case ACCESSORY_GET_STRING_VERSION: + src = dev->version; + break; + case ACCESSORY_GET_STRING_URI: + src = dev->uri; + break; + case ACCESSORY_GET_STRING_SERIAL: + src = dev->serial; + break; + case ACCESSORY_IS_START_REQUESTED: + return dev->start_requested; + case ACCESSORY_GET_AUDIO_MODE: + return dev->audio_mode; + } + if (!src) + return -EINVAL; + + ret = strlen(src) + 1; + if (copy_to_user((void __user *)value, src, ret)) + ret = -EFAULT; + return ret; +} + +static int acc_open(struct inode *ip, struct file *fp) +{ + printk(KERN_INFO "acc_open\n"); + if (atomic_xchg(&_acc_dev->open_excl, 1)) + return -EBUSY; + + _acc_dev->disconnected = 0; + fp->private_data = _acc_dev; + return 0; +} + +static int acc_release(struct inode *ip, struct file *fp) +{ + printk(KERN_INFO "acc_release\n"); + + WARN_ON(!atomic_xchg(&_acc_dev->open_excl, 0)); + _acc_dev->disconnected = 0; + return 0; +} + +/* file operations for /dev/usb_accessory */ +static const struct file_operations acc_fops = { + .owner = THIS_MODULE, + .read = acc_read, + .write = acc_write, + .unlocked_ioctl = acc_ioctl, + .open = acc_open, + .release = acc_release, +}; + +static int acc_hid_probe(struct hid_device *hdev, + const struct hid_device_id *id) +{ + int ret; + + ret = hid_parse(hdev); + if (ret) + return ret; + return hid_hw_start(hdev, HID_CONNECT_DEFAULT); +} + +static struct miscdevice acc_device = { + .minor = MISC_DYNAMIC_MINOR, + .name = "usb_accessory", + .fops = &acc_fops, +}; + +static const struct hid_device_id acc_hid_table[] = { + { HID_USB_DEVICE(HID_ANY_ID, HID_ANY_ID) }, + { } +}; + +static struct hid_driver acc_hid_driver = { + .name = "USB accessory", + .id_table = acc_hid_table, + .probe = acc_hid_probe, +}; + +int acc_ctrlrequest(struct usb_composite_dev *cdev, + const struct usb_ctrlrequest *ctrl) +{ + struct acc_dev *dev = _acc_dev; + int value = -EOPNOTSUPP; + struct acc_hid_dev *hid; + int offset; + u8 b_requestType = ctrl->bRequestType; + u8 b_request = ctrl->bRequest; + u16 w_index = le16_to_cpu(ctrl->wIndex); + u16 w_value = le16_to_cpu(ctrl->wValue); + u16 w_length = le16_to_cpu(ctrl->wLength); + unsigned long flags; + +/* + printk(KERN_INFO "acc_ctrlrequest " + "%02x.%02x v%04x i%04x l%u\n", + b_requestType, b_request, + w_value, w_index, w_length); +*/ + + if (b_requestType == (USB_DIR_OUT | USB_TYPE_VENDOR)) { + if (b_request == ACCESSORY_START) { + dev->start_requested = 1; + schedule_delayed_work( + &dev->start_work, msecs_to_jiffies(10)); + value = 0; + } else if (b_request == ACCESSORY_SEND_STRING) { + dev->string_index = w_index; + cdev->gadget->ep0->driver_data = dev; + cdev->req->complete = acc_complete_set_string; + value = w_length; + } else if (b_request == ACCESSORY_SET_AUDIO_MODE && + w_index == 0 && w_length == 0) { + dev->audio_mode = w_value; + value = 0; + } else if (b_request == ACCESSORY_REGISTER_HID) { + value = acc_register_hid(dev, w_value, w_index); + } else if (b_request == ACCESSORY_UNREGISTER_HID) { + value = acc_unregister_hid(dev, w_value); + } else if (b_request == ACCESSORY_SET_HID_REPORT_DESC) { + spin_lock_irqsave(&dev->lock, flags); + hid = acc_hid_get(&dev->new_hid_list, w_value); + spin_unlock_irqrestore(&dev->lock, flags); + if (!hid) { + value = -EINVAL; + goto err; + } + offset = w_index; + if (offset != hid->report_desc_offset + || offset + w_length > hid->report_desc_len) { + value = -EINVAL; + goto err; + } + cdev->req->context = hid; + cdev->req->complete = acc_complete_set_hid_report_desc; + value = w_length; + } else if (b_request == ACCESSORY_SEND_HID_EVENT) { + spin_lock_irqsave(&dev->lock, flags); + hid = acc_hid_get(&dev->hid_list, w_value); + spin_unlock_irqrestore(&dev->lock, flags); + if (!hid) { + value = -EINVAL; + goto err; + } + cdev->req->context = hid; + cdev->req->complete = acc_complete_send_hid_event; + value = w_length; + } + } else if (b_requestType == (USB_DIR_IN | USB_TYPE_VENDOR)) { + if (b_request == ACCESSORY_GET_PROTOCOL) { + *((u16 *)cdev->req->buf) = PROTOCOL_VERSION; + value = sizeof(u16); + + /* clear any string left over from a previous session */ + memset(dev->manufacturer, 0, sizeof(dev->manufacturer)); + memset(dev->model, 0, sizeof(dev->model)); + memset(dev->description, 0, sizeof(dev->description)); + memset(dev->version, 0, sizeof(dev->version)); + memset(dev->uri, 0, sizeof(dev->uri)); + memset(dev->serial, 0, sizeof(dev->serial)); + dev->start_requested = 0; + dev->audio_mode = 0; + } + } + + if (value >= 0) { + cdev->req->zero = 0; + cdev->req->length = value; + value = usb_ep_queue(cdev->gadget->ep0, cdev->req, GFP_ATOMIC); + if (value < 0) + ERROR(cdev, "%s setup response queue error\n", + __func__); + } + +err: + if (value == -EOPNOTSUPP) + VDBG(cdev, + "unknown class-specific control req " + "%02x.%02x v%04x i%04x l%u\n", + ctrl->bRequestType, ctrl->bRequest, + w_value, w_index, w_length); + return value; +} +EXPORT_SYMBOL_GPL(acc_ctrlrequest); + +static int +__acc_function_bind(struct usb_configuration *c, + struct usb_function *f, bool configfs) +{ + struct usb_composite_dev *cdev = c->cdev; + struct acc_dev *dev = func_to_dev(f); + int id; + int ret; + + DBG(cdev, "acc_function_bind dev: %p\n", dev); + + if (configfs) { + if (acc_string_defs[INTERFACE_STRING_INDEX].id == 0) { + ret = usb_string_id(c->cdev); + if (ret < 0) + return ret; + acc_string_defs[INTERFACE_STRING_INDEX].id = ret; + acc_interface_desc.iInterface = ret; + } + dev->cdev = c->cdev; + } + ret = hid_register_driver(&acc_hid_driver); + if (ret) + return ret; + + dev->start_requested = 0; + + /* allocate interface ID(s) */ + id = usb_interface_id(c, f); + if (id < 0) + return id; + acc_interface_desc.bInterfaceNumber = id; + + /* allocate endpoints */ + ret = create_bulk_endpoints(dev, &acc_fullspeed_in_desc, + &acc_fullspeed_out_desc); + if (ret) + return ret; + + /* support high speed hardware */ + if (gadget_is_dualspeed(c->cdev->gadget)) { + acc_highspeed_in_desc.bEndpointAddress = + acc_fullspeed_in_desc.bEndpointAddress; + acc_highspeed_out_desc.bEndpointAddress = + acc_fullspeed_out_desc.bEndpointAddress; + } + + DBG(cdev, "%s speed %s: IN/%s, OUT/%s\n", + gadget_is_dualspeed(c->cdev->gadget) ? "dual" : "full", + f->name, dev->ep_in->name, dev->ep_out->name); + return 0; +} + +static int +acc_function_bind_configfs(struct usb_configuration *c, + struct usb_function *f) { + return __acc_function_bind(c, f, true); +} + +static void +kill_all_hid_devices(struct acc_dev *dev) +{ + struct acc_hid_dev *hid; + struct list_head *entry, *temp; + unsigned long flags; + + /* do nothing if usb accessory device doesn't exist */ + if (!dev) + return; + + spin_lock_irqsave(&dev->lock, flags); + list_for_each_safe(entry, temp, &dev->hid_list) { + hid = list_entry(entry, struct acc_hid_dev, list); + list_del(&hid->list); + list_add(&hid->list, &dev->dead_hid_list); + } + list_for_each_safe(entry, temp, &dev->new_hid_list) { + hid = list_entry(entry, struct acc_hid_dev, list); + list_del(&hid->list); + list_add(&hid->list, &dev->dead_hid_list); + } + spin_unlock_irqrestore(&dev->lock, flags); + + schedule_work(&dev->hid_work); +} + +static void +acc_hid_unbind(struct acc_dev *dev) +{ + hid_unregister_driver(&acc_hid_driver); + kill_all_hid_devices(dev); +} + +static void +acc_function_unbind(struct usb_configuration *c, struct usb_function *f) +{ + struct acc_dev *dev = func_to_dev(f); + struct usb_request *req; + int i; + + while ((req = req_get(dev, &dev->tx_idle))) + acc_request_free(req, dev->ep_in); + for (i = 0; i < RX_REQ_MAX; i++) + acc_request_free(dev->rx_req[i], dev->ep_out); + + acc_hid_unbind(dev); +} + +static void acc_start_work(struct work_struct *data) +{ + char *envp[2] = { "ACCESSORY=START", NULL }; + + kobject_uevent_env(&acc_device.this_device->kobj, KOBJ_CHANGE, envp); +} + +static int acc_hid_init(struct acc_hid_dev *hdev) +{ + struct hid_device *hid; + int ret; + + hid = hid_allocate_device(); + if (IS_ERR(hid)) + return PTR_ERR(hid); + + hid->ll_driver = &acc_hid_ll_driver; + hid->dev.parent = acc_device.this_device; + + hid->bus = BUS_USB; + hid->vendor = HID_ANY_ID; + hid->product = HID_ANY_ID; + hid->driver_data = hdev; + ret = hid_add_device(hid); + if (ret) { + pr_err("can't add hid device: %d\n", ret); + hid_destroy_device(hid); + return ret; + } + + hdev->hid = hid; + return 0; +} + +static void acc_hid_delete(struct acc_hid_dev *hid) +{ + kfree(hid->report_desc); + kfree(hid); +} + +static void acc_hid_work(struct work_struct *data) +{ + struct acc_dev *dev = _acc_dev; + struct list_head *entry, *temp; + struct acc_hid_dev *hid; + struct list_head new_list, dead_list; + unsigned long flags; + + INIT_LIST_HEAD(&new_list); + + spin_lock_irqsave(&dev->lock, flags); + + /* copy hids that are ready for initialization to new_list */ + list_for_each_safe(entry, temp, &dev->new_hid_list) { + hid = list_entry(entry, struct acc_hid_dev, list); + if (hid->report_desc_offset == hid->report_desc_len) + list_move(&hid->list, &new_list); + } + + if (list_empty(&dev->dead_hid_list)) { + INIT_LIST_HEAD(&dead_list); + } else { + /* move all of dev->dead_hid_list to dead_list */ + dead_list.prev = dev->dead_hid_list.prev; + dead_list.next = dev->dead_hid_list.next; + dead_list.next->prev = &dead_list; + dead_list.prev->next = &dead_list; + INIT_LIST_HEAD(&dev->dead_hid_list); + } + + spin_unlock_irqrestore(&dev->lock, flags); + + /* register new HID devices */ + list_for_each_safe(entry, temp, &new_list) { + hid = list_entry(entry, struct acc_hid_dev, list); + if (acc_hid_init(hid)) { + pr_err("can't add HID device %p\n", hid); + acc_hid_delete(hid); + } else { + spin_lock_irqsave(&dev->lock, flags); + list_move(&hid->list, &dev->hid_list); + spin_unlock_irqrestore(&dev->lock, flags); + } + } + + /* remove dead HID devices */ + list_for_each_safe(entry, temp, &dead_list) { + hid = list_entry(entry, struct acc_hid_dev, list); + list_del(&hid->list); + if (hid->hid) + hid_destroy_device(hid->hid); + acc_hid_delete(hid); + } +} + +static int acc_function_set_alt(struct usb_function *f, + unsigned intf, unsigned alt) +{ + struct acc_dev *dev = func_to_dev(f); + struct usb_composite_dev *cdev = f->config->cdev; + int ret; + + DBG(cdev, "acc_function_set_alt intf: %d alt: %d\n", intf, alt); + + ret = config_ep_by_speed(cdev->gadget, f, dev->ep_in); + if (ret) + return ret; + + ret = usb_ep_enable(dev->ep_in); + if (ret) + return ret; + + ret = config_ep_by_speed(cdev->gadget, f, dev->ep_out); + if (ret) + return ret; + + ret = usb_ep_enable(dev->ep_out); + if (ret) { + usb_ep_disable(dev->ep_in); + return ret; + } + + dev->online = 1; + + /* readers may be blocked waiting for us to go online */ + wake_up(&dev->read_wq); + return 0; +} + +static void acc_function_disable(struct usb_function *f) +{ + struct acc_dev *dev = func_to_dev(f); + struct usb_composite_dev *cdev = dev->cdev; + + DBG(cdev, "acc_function_disable\n"); + acc_set_disconnected(dev); + usb_ep_disable(dev->ep_in); + usb_ep_disable(dev->ep_out); + + /* readers may be blocked waiting for us to go online */ + wake_up(&dev->read_wq); + + VDBG(cdev, "%s disabled\n", dev->function.name); +} + +static int acc_setup(void) +{ + struct acc_dev *dev; + int ret; + + dev = kzalloc(sizeof(*dev), GFP_KERNEL); + if (!dev) + return -ENOMEM; + + spin_lock_init(&dev->lock); + init_waitqueue_head(&dev->read_wq); + init_waitqueue_head(&dev->write_wq); + atomic_set(&dev->open_excl, 0); + INIT_LIST_HEAD(&dev->tx_idle); + INIT_LIST_HEAD(&dev->hid_list); + INIT_LIST_HEAD(&dev->new_hid_list); + INIT_LIST_HEAD(&dev->dead_hid_list); + INIT_DELAYED_WORK(&dev->start_work, acc_start_work); + INIT_WORK(&dev->hid_work, acc_hid_work); + + /* _acc_dev must be set before calling usb_gadget_register_driver */ + _acc_dev = dev; + + ret = misc_register(&acc_device); + if (ret) + goto err; + + return 0; + +err: + kfree(dev); + pr_err("USB accessory gadget driver failed to initialize\n"); + return ret; +} + +void acc_disconnect(void) +{ + /* unregister all HID devices if USB is disconnected */ + kill_all_hid_devices(_acc_dev); +} +EXPORT_SYMBOL_GPL(acc_disconnect); + +static void acc_cleanup(void) +{ + misc_deregister(&acc_device); + kfree(_acc_dev); + _acc_dev = NULL; +} +static struct acc_instance *to_acc_instance(struct config_item *item) +{ + return container_of(to_config_group(item), struct acc_instance, + func_inst.group); +} + +static void acc_attr_release(struct config_item *item) +{ + struct acc_instance *fi_acc = to_acc_instance(item); + + usb_put_function_instance(&fi_acc->func_inst); +} + +static struct configfs_item_operations acc_item_ops = { + .release = acc_attr_release, +}; + +static struct config_item_type acc_func_type = { + .ct_item_ops = &acc_item_ops, + .ct_owner = THIS_MODULE, +}; + +static struct acc_instance *to_fi_acc(struct usb_function_instance *fi) +{ + return container_of(fi, struct acc_instance, func_inst); +} + +static int acc_set_inst_name(struct usb_function_instance *fi, const char *name) +{ + struct acc_instance *fi_acc; + char *ptr; + int name_len; + + name_len = strlen(name) + 1; + if (name_len > MAX_INST_NAME_LEN) + return -ENAMETOOLONG; + + ptr = kstrndup(name, name_len, GFP_KERNEL); + if (!ptr) + return -ENOMEM; + + fi_acc = to_fi_acc(fi); + fi_acc->name = ptr; + return 0; +} + +static void acc_free_inst(struct usb_function_instance *fi) +{ + struct acc_instance *fi_acc; + + fi_acc = to_fi_acc(fi); + kfree(fi_acc->name); + acc_cleanup(); +} + +static struct usb_function_instance *acc_alloc_inst(void) +{ + struct acc_instance *fi_acc; + struct acc_dev *dev; + int err; + + fi_acc = kzalloc(sizeof(*fi_acc), GFP_KERNEL); + if (!fi_acc) + return ERR_PTR(-ENOMEM); + fi_acc->func_inst.set_inst_name = acc_set_inst_name; + fi_acc->func_inst.free_func_inst = acc_free_inst; + + err = acc_setup(); + if (err) { + kfree(fi_acc); + pr_err("Error setting ACCESSORY\n"); + return ERR_PTR(err); + } + + config_group_init_type_name(&fi_acc->func_inst.group, + "", &acc_func_type); + dev = _acc_dev; + return &fi_acc->func_inst; +} + +static void acc_free(struct usb_function *f) +{ +/*NO-OP: no function specific resource allocation in mtp_alloc*/ +} + +int acc_ctrlrequest_configfs(struct usb_function *f, + const struct usb_ctrlrequest *ctrl) { + if (f->config != NULL && f->config->cdev != NULL) + return acc_ctrlrequest(f->config->cdev, ctrl); + else + return -1; +} + +static struct usb_function *acc_alloc(struct usb_function_instance *fi) +{ + struct acc_dev *dev = _acc_dev; + + pr_info("acc_alloc\n"); + + dev->function.name = "accessory"; + dev->function.strings = acc_strings, + dev->function.fs_descriptors = fs_acc_descs; + dev->function.hs_descriptors = hs_acc_descs; + dev->function.bind = acc_function_bind_configfs; + dev->function.unbind = acc_function_unbind; + dev->function.set_alt = acc_function_set_alt; + dev->function.disable = acc_function_disable; + dev->function.free_func = acc_free; + dev->function.setup = acc_ctrlrequest_configfs; + + return &dev->function; +} +DECLARE_USB_FUNCTION_INIT(accessory, acc_alloc_inst, acc_alloc); +MODULE_LICENSE("GPL"); diff --git a/drivers/usb/gadget/function/f_audio_source.c b/drivers/usb/gadget/function/f_audio_source.c new file mode 100644 index 00000000000000..db7903d19c435e --- /dev/null +++ b/drivers/usb/gadget/function/f_audio_source.c @@ -0,0 +1,1060 @@ +/* + * Gadget Function Driver for USB audio source device + * + * Copyright (C) 2012 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#define SAMPLE_RATE 44100 +#define FRAMES_PER_MSEC (SAMPLE_RATE / 1000) + +#define IN_EP_MAX_PACKET_SIZE 256 + +/* Number of requests to allocate */ +#define IN_EP_REQ_COUNT 4 + +#define AUDIO_AC_INTERFACE 0 +#define AUDIO_AS_INTERFACE 1 +#define AUDIO_NUM_INTERFACES 2 +#define MAX_INST_NAME_LEN 40 + +/* B.3.1 Standard AC Interface Descriptor */ +static struct usb_interface_descriptor ac_interface_desc = { + .bLength = USB_DT_INTERFACE_SIZE, + .bDescriptorType = USB_DT_INTERFACE, + .bNumEndpoints = 0, + .bInterfaceClass = USB_CLASS_AUDIO, + .bInterfaceSubClass = USB_SUBCLASS_AUDIOCONTROL, +}; + +DECLARE_UAC_AC_HEADER_DESCRIPTOR(2); + +#define UAC_DT_AC_HEADER_LENGTH UAC_DT_AC_HEADER_SIZE(AUDIO_NUM_INTERFACES) +/* 1 input terminal, 1 output terminal and 1 feature unit */ +#define UAC_DT_TOTAL_LENGTH (UAC_DT_AC_HEADER_LENGTH \ + + UAC_DT_INPUT_TERMINAL_SIZE + UAC_DT_OUTPUT_TERMINAL_SIZE \ + + UAC_DT_FEATURE_UNIT_SIZE(0)) +/* B.3.2 Class-Specific AC Interface Descriptor */ +static struct uac1_ac_header_descriptor_2 ac_header_desc = { + .bLength = UAC_DT_AC_HEADER_LENGTH, + .bDescriptorType = USB_DT_CS_INTERFACE, + .bDescriptorSubtype = UAC_HEADER, + .bcdADC = __constant_cpu_to_le16(0x0100), + .wTotalLength = __constant_cpu_to_le16(UAC_DT_TOTAL_LENGTH), + .bInCollection = AUDIO_NUM_INTERFACES, + .baInterfaceNr = { + [0] = AUDIO_AC_INTERFACE, + [1] = AUDIO_AS_INTERFACE, + } +}; + +#define INPUT_TERMINAL_ID 1 +static struct uac_input_terminal_descriptor input_terminal_desc = { + .bLength = UAC_DT_INPUT_TERMINAL_SIZE, + .bDescriptorType = USB_DT_CS_INTERFACE, + .bDescriptorSubtype = UAC_INPUT_TERMINAL, + .bTerminalID = INPUT_TERMINAL_ID, + .wTerminalType = UAC_INPUT_TERMINAL_MICROPHONE, + .bAssocTerminal = 0, + .wChannelConfig = 0x3, +}; + +DECLARE_UAC_FEATURE_UNIT_DESCRIPTOR(0); + +#define FEATURE_UNIT_ID 2 +static struct uac_feature_unit_descriptor_0 feature_unit_desc = { + .bLength = UAC_DT_FEATURE_UNIT_SIZE(0), + .bDescriptorType = USB_DT_CS_INTERFACE, + .bDescriptorSubtype = UAC_FEATURE_UNIT, + .bUnitID = FEATURE_UNIT_ID, + .bSourceID = INPUT_TERMINAL_ID, + .bControlSize = 2, +}; + +#define OUTPUT_TERMINAL_ID 3 +static struct uac1_output_terminal_descriptor output_terminal_desc = { + .bLength = UAC_DT_OUTPUT_TERMINAL_SIZE, + .bDescriptorType = USB_DT_CS_INTERFACE, + .bDescriptorSubtype = UAC_OUTPUT_TERMINAL, + .bTerminalID = OUTPUT_TERMINAL_ID, + .wTerminalType = UAC_TERMINAL_STREAMING, + .bAssocTerminal = FEATURE_UNIT_ID, + .bSourceID = FEATURE_UNIT_ID, +}; + +/* B.4.1 Standard AS Interface Descriptor */ +static struct usb_interface_descriptor as_interface_alt_0_desc = { + .bLength = USB_DT_INTERFACE_SIZE, + .bDescriptorType = USB_DT_INTERFACE, + .bAlternateSetting = 0, + .bNumEndpoints = 0, + .bInterfaceClass = USB_CLASS_AUDIO, + .bInterfaceSubClass = USB_SUBCLASS_AUDIOSTREAMING, +}; + +static struct usb_interface_descriptor as_interface_alt_1_desc = { + .bLength = USB_DT_INTERFACE_SIZE, + .bDescriptorType = USB_DT_INTERFACE, + .bAlternateSetting = 1, + .bNumEndpoints = 1, + .bInterfaceClass = USB_CLASS_AUDIO, + .bInterfaceSubClass = USB_SUBCLASS_AUDIOSTREAMING, +}; + +/* B.4.2 Class-Specific AS Interface Descriptor */ +static struct uac1_as_header_descriptor as_header_desc = { + .bLength = UAC_DT_AS_HEADER_SIZE, + .bDescriptorType = USB_DT_CS_INTERFACE, + .bDescriptorSubtype = UAC_AS_GENERAL, + .bTerminalLink = INPUT_TERMINAL_ID, + .bDelay = 1, + .wFormatTag = UAC_FORMAT_TYPE_I_PCM, +}; + +DECLARE_UAC_FORMAT_TYPE_I_DISCRETE_DESC(1); + +static struct uac_format_type_i_discrete_descriptor_1 as_type_i_desc = { + .bLength = UAC_FORMAT_TYPE_I_DISCRETE_DESC_SIZE(1), + .bDescriptorType = USB_DT_CS_INTERFACE, + .bDescriptorSubtype = UAC_FORMAT_TYPE, + .bFormatType = UAC_FORMAT_TYPE_I, + .bSubframeSize = 2, + .bBitResolution = 16, + .bSamFreqType = 1, +}; + +/* Standard ISO IN Endpoint Descriptor for highspeed */ +static struct usb_endpoint_descriptor hs_as_in_ep_desc = { + .bLength = USB_DT_ENDPOINT_AUDIO_SIZE, + .bDescriptorType = USB_DT_ENDPOINT, + .bEndpointAddress = USB_DIR_IN, + .bmAttributes = USB_ENDPOINT_SYNC_SYNC + | USB_ENDPOINT_XFER_ISOC, + .wMaxPacketSize = __constant_cpu_to_le16(IN_EP_MAX_PACKET_SIZE), + .bInterval = 4, /* poll 1 per millisecond */ +}; + +/* Standard ISO IN Endpoint Descriptor for highspeed */ +static struct usb_endpoint_descriptor fs_as_in_ep_desc = { + .bLength = USB_DT_ENDPOINT_AUDIO_SIZE, + .bDescriptorType = USB_DT_ENDPOINT, + .bEndpointAddress = USB_DIR_IN, + .bmAttributes = USB_ENDPOINT_SYNC_SYNC + | USB_ENDPOINT_XFER_ISOC, + .wMaxPacketSize = __constant_cpu_to_le16(IN_EP_MAX_PACKET_SIZE), + .bInterval = 1, /* poll 1 per millisecond */ +}; + +/* Class-specific AS ISO OUT Endpoint Descriptor */ +static struct uac_iso_endpoint_descriptor as_iso_in_desc = { + .bLength = UAC_ISO_ENDPOINT_DESC_SIZE, + .bDescriptorType = USB_DT_CS_ENDPOINT, + .bDescriptorSubtype = UAC_EP_GENERAL, + .bmAttributes = 1, + .bLockDelayUnits = 1, + .wLockDelay = __constant_cpu_to_le16(1), +}; + +static struct usb_descriptor_header *hs_audio_desc[] = { + (struct usb_descriptor_header *)&ac_interface_desc, + (struct usb_descriptor_header *)&ac_header_desc, + + (struct usb_descriptor_header *)&input_terminal_desc, + (struct usb_descriptor_header *)&output_terminal_desc, + (struct usb_descriptor_header *)&feature_unit_desc, + + (struct usb_descriptor_header *)&as_interface_alt_0_desc, + (struct usb_descriptor_header *)&as_interface_alt_1_desc, + (struct usb_descriptor_header *)&as_header_desc, + + (struct usb_descriptor_header *)&as_type_i_desc, + + (struct usb_descriptor_header *)&hs_as_in_ep_desc, + (struct usb_descriptor_header *)&as_iso_in_desc, + NULL, +}; + +static struct usb_descriptor_header *fs_audio_desc[] = { + (struct usb_descriptor_header *)&ac_interface_desc, + (struct usb_descriptor_header *)&ac_header_desc, + + (struct usb_descriptor_header *)&input_terminal_desc, + (struct usb_descriptor_header *)&output_terminal_desc, + (struct usb_descriptor_header *)&feature_unit_desc, + + (struct usb_descriptor_header *)&as_interface_alt_0_desc, + (struct usb_descriptor_header *)&as_interface_alt_1_desc, + (struct usb_descriptor_header *)&as_header_desc, + + (struct usb_descriptor_header *)&as_type_i_desc, + + (struct usb_descriptor_header *)&fs_as_in_ep_desc, + (struct usb_descriptor_header *)&as_iso_in_desc, + NULL, +}; + +static struct snd_pcm_hardware audio_hw_info = { + .info = SNDRV_PCM_INFO_MMAP | + SNDRV_PCM_INFO_MMAP_VALID | + SNDRV_PCM_INFO_BATCH | + SNDRV_PCM_INFO_INTERLEAVED | + SNDRV_PCM_INFO_BLOCK_TRANSFER, + + .formats = SNDRV_PCM_FMTBIT_S16_LE, + .channels_min = 2, + .channels_max = 2, + .rate_min = SAMPLE_RATE, + .rate_max = SAMPLE_RATE, + + .buffer_bytes_max = 1024 * 1024, + .period_bytes_min = 64, + .period_bytes_max = 512 * 1024, + .periods_min = 2, + .periods_max = 1024, +}; + +/*-------------------------------------------------------------------------*/ + +struct audio_source_config { + int card; + int device; +}; + +struct audio_dev { + struct usb_function func; + struct snd_card *card; + struct snd_pcm *pcm; + struct snd_pcm_substream *substream; + + struct list_head idle_reqs; + struct usb_ep *in_ep; + + spinlock_t lock; + + /* beginning, end and current position in our buffer */ + void *buffer_start; + void *buffer_end; + void *buffer_pos; + + /* byte size of a "period" */ + unsigned int period; + /* bytes sent since last call to snd_pcm_period_elapsed */ + unsigned int period_offset; + /* time we started playing */ + ktime_t start_time; + /* number of frames sent since start_time */ + s64 frames_sent; + struct audio_source_config *config; +}; + +static inline struct audio_dev *func_to_audio(struct usb_function *f) +{ + return container_of(f, struct audio_dev, func); +} + +/*-------------------------------------------------------------------------*/ + +struct audio_source_instance { + struct usb_function_instance func_inst; + const char *name; + struct audio_source_config *config; + struct device *audio_device; +}; + +static void audio_source_attr_release(struct config_item *item); + +static struct configfs_item_operations audio_source_item_ops = { + .release = audio_source_attr_release, +}; + +static struct config_item_type audio_source_func_type = { + .ct_item_ops = &audio_source_item_ops, + .ct_owner = THIS_MODULE, +}; + +static ssize_t audio_source_pcm_show(struct device *dev, + struct device_attribute *attr, char *buf); + +static DEVICE_ATTR(pcm, S_IRUGO, audio_source_pcm_show, NULL); + +static struct device_attribute *audio_source_function_attributes[] = { + &dev_attr_pcm, + NULL +}; + +/*--------------------------------------------------------------------------*/ + +static struct usb_request *audio_request_new(struct usb_ep *ep, int buffer_size) +{ + struct usb_request *req = usb_ep_alloc_request(ep, GFP_KERNEL); + + if (!req) + return NULL; + + req->buf = kmalloc(buffer_size, GFP_KERNEL); + if (!req->buf) { + usb_ep_free_request(ep, req); + return NULL; + } + req->length = buffer_size; + return req; +} + +static void audio_request_free(struct usb_request *req, struct usb_ep *ep) +{ + if (req) { + kfree(req->buf); + usb_ep_free_request(ep, req); + } +} + +static void audio_req_put(struct audio_dev *audio, struct usb_request *req) +{ + unsigned long flags; + + spin_lock_irqsave(&audio->lock, flags); + list_add_tail(&req->list, &audio->idle_reqs); + spin_unlock_irqrestore(&audio->lock, flags); +} + +static struct usb_request *audio_req_get(struct audio_dev *audio) +{ + unsigned long flags; + struct usb_request *req; + + spin_lock_irqsave(&audio->lock, flags); + if (list_empty(&audio->idle_reqs)) { + req = 0; + } else { + req = list_first_entry(&audio->idle_reqs, struct usb_request, + list); + list_del(&req->list); + } + spin_unlock_irqrestore(&audio->lock, flags); + return req; +} + +/* send the appropriate number of packets to match our bitrate */ +static void audio_send(struct audio_dev *audio) +{ + struct snd_pcm_runtime *runtime; + struct usb_request *req; + int length, length1, length2, ret; + s64 msecs; + s64 frames; + ktime_t now; + + /* audio->substream will be null if we have been closed */ + if (!audio->substream) + return; + /* audio->buffer_pos will be null if we have been stopped */ + if (!audio->buffer_pos) + return; + + runtime = audio->substream->runtime; + + /* compute number of frames to send */ + now = ktime_get(); + msecs = div_s64((ktime_to_ns(now) - ktime_to_ns(audio->start_time)), + 1000000); + frames = div_s64((msecs * SAMPLE_RATE), 1000); + + /* Readjust our frames_sent if we fall too far behind. + * If we get too far behind it is better to drop some frames than + * to keep sending data too fast in an attempt to catch up. + */ + if (frames - audio->frames_sent > 10 * FRAMES_PER_MSEC) + audio->frames_sent = frames - FRAMES_PER_MSEC; + + frames -= audio->frames_sent; + + /* We need to send something to keep the pipeline going */ + if (frames <= 0) + frames = FRAMES_PER_MSEC; + + while (frames > 0) { + req = audio_req_get(audio); + if (!req) + break; + + length = frames_to_bytes(runtime, frames); + if (length > IN_EP_MAX_PACKET_SIZE) + length = IN_EP_MAX_PACKET_SIZE; + + if (audio->buffer_pos + length > audio->buffer_end) + length1 = audio->buffer_end - audio->buffer_pos; + else + length1 = length; + memcpy(req->buf, audio->buffer_pos, length1); + if (length1 < length) { + /* Wrap around and copy remaining length + * at beginning of buffer. + */ + length2 = length - length1; + memcpy(req->buf + length1, audio->buffer_start, + length2); + audio->buffer_pos = audio->buffer_start + length2; + } else { + audio->buffer_pos += length1; + if (audio->buffer_pos >= audio->buffer_end) + audio->buffer_pos = audio->buffer_start; + } + + req->length = length; + ret = usb_ep_queue(audio->in_ep, req, GFP_ATOMIC); + if (ret < 0) { + pr_err("usb_ep_queue failed ret: %d\n", ret); + audio_req_put(audio, req); + break; + } + + frames -= bytes_to_frames(runtime, length); + audio->frames_sent += bytes_to_frames(runtime, length); + } +} + +static void audio_control_complete(struct usb_ep *ep, struct usb_request *req) +{ + /* nothing to do here */ +} + +static void audio_data_complete(struct usb_ep *ep, struct usb_request *req) +{ + struct audio_dev *audio = req->context; + + pr_debug("audio_data_complete req->status %d req->actual %d\n", + req->status, req->actual); + + audio_req_put(audio, req); + + if (!audio->buffer_start || req->status) + return; + + audio->period_offset += req->actual; + if (audio->period_offset >= audio->period) { + snd_pcm_period_elapsed(audio->substream); + audio->period_offset = 0; + } + audio_send(audio); +} + +static int audio_set_endpoint_req(struct usb_function *f, + const struct usb_ctrlrequest *ctrl) +{ + int value = -EOPNOTSUPP; + u16 ep = le16_to_cpu(ctrl->wIndex); + u16 len = le16_to_cpu(ctrl->wLength); + u16 w_value = le16_to_cpu(ctrl->wValue); + + pr_debug("bRequest 0x%x, w_value 0x%04x, len %d, endpoint %d\n", + ctrl->bRequest, w_value, len, ep); + + switch (ctrl->bRequest) { + case UAC_SET_CUR: + case UAC_SET_MIN: + case UAC_SET_MAX: + case UAC_SET_RES: + value = len; + break; + default: + break; + } + + return value; +} + +static int audio_get_endpoint_req(struct usb_function *f, + const struct usb_ctrlrequest *ctrl) +{ + struct usb_composite_dev *cdev = f->config->cdev; + int value = -EOPNOTSUPP; + u8 ep = ((le16_to_cpu(ctrl->wIndex) >> 8) & 0xFF); + u16 len = le16_to_cpu(ctrl->wLength); + u16 w_value = le16_to_cpu(ctrl->wValue); + u8 *buf = cdev->req->buf; + + pr_debug("bRequest 0x%x, w_value 0x%04x, len %d, endpoint %d\n", + ctrl->bRequest, w_value, len, ep); + + if (w_value == UAC_EP_CS_ATTR_SAMPLE_RATE << 8) { + switch (ctrl->bRequest) { + case UAC_GET_CUR: + case UAC_GET_MIN: + case UAC_GET_MAX: + case UAC_GET_RES: + /* return our sample rate */ + buf[0] = (u8)SAMPLE_RATE; + buf[1] = (u8)(SAMPLE_RATE >> 8); + buf[2] = (u8)(SAMPLE_RATE >> 16); + value = 3; + break; + default: + break; + } + } + + return value; +} + +static int +audio_setup(struct usb_function *f, const struct usb_ctrlrequest *ctrl) +{ + struct usb_composite_dev *cdev = f->config->cdev; + struct usb_request *req = cdev->req; + int value = -EOPNOTSUPP; + u16 w_index = le16_to_cpu(ctrl->wIndex); + u16 w_value = le16_to_cpu(ctrl->wValue); + u16 w_length = le16_to_cpu(ctrl->wLength); + + /* composite driver infrastructure handles everything; interface + * activation uses set_alt(). + */ + switch (ctrl->bRequestType) { + case USB_DIR_OUT | USB_TYPE_CLASS | USB_RECIP_ENDPOINT: + value = audio_set_endpoint_req(f, ctrl); + break; + + case USB_DIR_IN | USB_TYPE_CLASS | USB_RECIP_ENDPOINT: + value = audio_get_endpoint_req(f, ctrl); + break; + } + + /* respond with data transfer or status phase? */ + if (value >= 0) { + pr_debug("audio req%02x.%02x v%04x i%04x l%d\n", + ctrl->bRequestType, ctrl->bRequest, + w_value, w_index, w_length); + req->zero = 0; + req->length = value; + req->complete = audio_control_complete; + value = usb_ep_queue(cdev->gadget->ep0, req, GFP_ATOMIC); + if (value < 0) + pr_err("audio response on err %d\n", value); + } + + /* device either stalls (value < 0) or reports success */ + return value; +} + +static int audio_set_alt(struct usb_function *f, unsigned intf, unsigned alt) +{ + struct audio_dev *audio = func_to_audio(f); + struct usb_composite_dev *cdev = f->config->cdev; + int ret; + + pr_debug("audio_set_alt intf %d, alt %d\n", intf, alt); + + ret = config_ep_by_speed(cdev->gadget, f, audio->in_ep); + if (ret) + return ret; + + usb_ep_enable(audio->in_ep); + return 0; +} + +static void audio_disable(struct usb_function *f) +{ + struct audio_dev *audio = func_to_audio(f); + + pr_debug("audio_disable\n"); + usb_ep_disable(audio->in_ep); +} + +static void audio_free_func(struct usb_function *f) +{ + /* no-op */ +} + +/*-------------------------------------------------------------------------*/ + +static void audio_build_desc(struct audio_dev *audio) +{ + u8 *sam_freq; + int rate; + + /* Set channel numbers */ + input_terminal_desc.bNrChannels = 2; + as_type_i_desc.bNrChannels = 2; + + /* Set sample rates */ + rate = SAMPLE_RATE; + sam_freq = as_type_i_desc.tSamFreq[0]; + memcpy(sam_freq, &rate, 3); +} + + +static int snd_card_setup(struct usb_configuration *c, + struct audio_source_config *config); +static struct audio_source_instance *to_fi_audio_source( + const struct usb_function_instance *fi); + + +/* audio function driver setup/binding */ +static int +audio_bind(struct usb_configuration *c, struct usb_function *f) +{ + struct usb_composite_dev *cdev = c->cdev; + struct audio_dev *audio = func_to_audio(f); + int status; + struct usb_ep *ep; + struct usb_request *req; + int i; + int err; + + if (IS_ENABLED(CONFIG_USB_CONFIGFS)) { + struct audio_source_instance *fi_audio = + to_fi_audio_source(f->fi); + struct audio_source_config *config = + fi_audio->config; + + err = snd_card_setup(c, config); + if (err) + return err; + } + + audio_build_desc(audio); + + /* allocate instance-specific interface IDs, and patch descriptors */ + status = usb_interface_id(c, f); + if (status < 0) + goto fail; + ac_interface_desc.bInterfaceNumber = status; + + /* AUDIO_AC_INTERFACE */ + ac_header_desc.baInterfaceNr[0] = status; + + status = usb_interface_id(c, f); + if (status < 0) + goto fail; + as_interface_alt_0_desc.bInterfaceNumber = status; + as_interface_alt_1_desc.bInterfaceNumber = status; + + /* AUDIO_AS_INTERFACE */ + ac_header_desc.baInterfaceNr[1] = status; + + status = -ENODEV; + + /* allocate our endpoint */ + ep = usb_ep_autoconfig(cdev->gadget, &fs_as_in_ep_desc); + if (!ep) + goto fail; + audio->in_ep = ep; + ep->driver_data = audio; /* claim */ + + if (gadget_is_dualspeed(c->cdev->gadget)) + hs_as_in_ep_desc.bEndpointAddress = + fs_as_in_ep_desc.bEndpointAddress; + + f->fs_descriptors = fs_audio_desc; + f->hs_descriptors = hs_audio_desc; + + for (i = 0, status = 0; i < IN_EP_REQ_COUNT && status == 0; i++) { + req = audio_request_new(ep, IN_EP_MAX_PACKET_SIZE); + if (req) { + req->context = audio; + req->complete = audio_data_complete; + audio_req_put(audio, req); + } else + status = -ENOMEM; + } + +fail: + return status; +} + +static void +audio_unbind(struct usb_configuration *c, struct usb_function *f) +{ + struct audio_dev *audio = func_to_audio(f); + struct usb_request *req; + + while ((req = audio_req_get(audio))) + audio_request_free(req, audio->in_ep); + + snd_card_free_when_closed(audio->card); + audio->card = NULL; + audio->pcm = NULL; + audio->substream = NULL; + audio->in_ep = NULL; + + if (IS_ENABLED(CONFIG_USB_CONFIGFS)) { + struct audio_source_instance *fi_audio = + to_fi_audio_source(f->fi); + struct audio_source_config *config = + fi_audio->config; + + config->card = -1; + config->device = -1; + } +} + +static void audio_pcm_playback_start(struct audio_dev *audio) +{ + audio->start_time = ktime_get(); + audio->frames_sent = 0; + audio_send(audio); +} + +static void audio_pcm_playback_stop(struct audio_dev *audio) +{ + unsigned long flags; + + spin_lock_irqsave(&audio->lock, flags); + audio->buffer_start = 0; + audio->buffer_end = 0; + audio->buffer_pos = 0; + spin_unlock_irqrestore(&audio->lock, flags); +} + +static int audio_pcm_open(struct snd_pcm_substream *substream) +{ + struct snd_pcm_runtime *runtime = substream->runtime; + struct audio_dev *audio = substream->private_data; + + runtime->private_data = audio; + runtime->hw = audio_hw_info; + snd_pcm_limit_hw_rates(runtime); + runtime->hw.channels_max = 2; + + audio->substream = substream; + return 0; +} + +static int audio_pcm_close(struct snd_pcm_substream *substream) +{ + struct audio_dev *audio = substream->private_data; + unsigned long flags; + + spin_lock_irqsave(&audio->lock, flags); + audio->substream = NULL; + spin_unlock_irqrestore(&audio->lock, flags); + + return 0; +} + +static int audio_pcm_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params) +{ + unsigned int channels = params_channels(params); + unsigned int rate = params_rate(params); + + if (rate != SAMPLE_RATE) + return -EINVAL; + if (channels != 2) + return -EINVAL; + + return snd_pcm_lib_alloc_vmalloc_buffer(substream, + params_buffer_bytes(params)); +} + +static int audio_pcm_hw_free(struct snd_pcm_substream *substream) +{ + return snd_pcm_lib_free_vmalloc_buffer(substream); +} + +static int audio_pcm_prepare(struct snd_pcm_substream *substream) +{ + struct snd_pcm_runtime *runtime = substream->runtime; + struct audio_dev *audio = runtime->private_data; + + audio->period = snd_pcm_lib_period_bytes(substream); + audio->period_offset = 0; + audio->buffer_start = runtime->dma_area; + audio->buffer_end = audio->buffer_start + + snd_pcm_lib_buffer_bytes(substream); + audio->buffer_pos = audio->buffer_start; + + return 0; +} + +static snd_pcm_uframes_t audio_pcm_pointer(struct snd_pcm_substream *substream) +{ + struct snd_pcm_runtime *runtime = substream->runtime; + struct audio_dev *audio = runtime->private_data; + ssize_t bytes = audio->buffer_pos - audio->buffer_start; + + /* return offset of next frame to fill in our buffer */ + return bytes_to_frames(runtime, bytes); +} + +static int audio_pcm_playback_trigger(struct snd_pcm_substream *substream, + int cmd) +{ + struct audio_dev *audio = substream->runtime->private_data; + int ret = 0; + + switch (cmd) { + case SNDRV_PCM_TRIGGER_START: + case SNDRV_PCM_TRIGGER_RESUME: + audio_pcm_playback_start(audio); + break; + + case SNDRV_PCM_TRIGGER_STOP: + case SNDRV_PCM_TRIGGER_SUSPEND: + audio_pcm_playback_stop(audio); + break; + + default: + ret = -EINVAL; + } + + return ret; +} + +static struct audio_dev _audio_dev = { + .func = { + .name = "audio_source", + .bind = audio_bind, + .unbind = audio_unbind, + .set_alt = audio_set_alt, + .setup = audio_setup, + .disable = audio_disable, + .free_func = audio_free_func, + }, + .lock = __SPIN_LOCK_UNLOCKED(_audio_dev.lock), + .idle_reqs = LIST_HEAD_INIT(_audio_dev.idle_reqs), +}; + +static struct snd_pcm_ops audio_playback_ops = { + .open = audio_pcm_open, + .close = audio_pcm_close, + .ioctl = snd_pcm_lib_ioctl, + .hw_params = audio_pcm_hw_params, + .hw_free = audio_pcm_hw_free, + .prepare = audio_pcm_prepare, + .trigger = audio_pcm_playback_trigger, + .pointer = audio_pcm_pointer, +}; + +int audio_source_bind_config(struct usb_configuration *c, + struct audio_source_config *config) +{ + struct audio_dev *audio; + int err; + + config->card = -1; + config->device = -1; + + audio = &_audio_dev; + + err = snd_card_setup(c, config); + if (err) + return err; + + err = usb_add_function(c, &audio->func); + if (err) + goto add_fail; + + return 0; + +add_fail: + snd_card_free(audio->card); + return err; +} + +static int snd_card_setup(struct usb_configuration *c, + struct audio_source_config *config) +{ + struct audio_dev *audio; + struct snd_card *card; + struct snd_pcm *pcm; + int err; + + audio = &_audio_dev; + + err = snd_card_new(&c->cdev->gadget->dev, + SNDRV_DEFAULT_IDX1, SNDRV_DEFAULT_STR1, + THIS_MODULE, 0, &card); + if (err) + return err; + + err = snd_pcm_new(card, "USB audio source", 0, 1, 0, &pcm); + if (err) + goto pcm_fail; + + pcm->private_data = audio; + pcm->info_flags = 0; + audio->pcm = pcm; + + strlcpy(pcm->name, "USB gadget audio", sizeof(pcm->name)); + + snd_pcm_set_ops(pcm, SNDRV_PCM_STREAM_PLAYBACK, &audio_playback_ops); + snd_pcm_lib_preallocate_pages_for_all(pcm, SNDRV_DMA_TYPE_DEV, + NULL, 0, 64 * 1024); + + strlcpy(card->driver, "audio_source", sizeof(card->driver)); + strlcpy(card->shortname, card->driver, sizeof(card->shortname)); + strlcpy(card->longname, "USB accessory audio source", + sizeof(card->longname)); + + err = snd_card_register(card); + if (err) + goto register_fail; + + config->card = pcm->card->number; + config->device = pcm->device; + audio->card = card; + return 0; + +register_fail: +pcm_fail: + snd_card_free(audio->card); + return err; +} + +static struct audio_source_instance *to_audio_source_instance( + struct config_item *item) +{ + return container_of(to_config_group(item), struct audio_source_instance, + func_inst.group); +} + +static struct audio_source_instance *to_fi_audio_source( + const struct usb_function_instance *fi) +{ + return container_of(fi, struct audio_source_instance, func_inst); +} + +static void audio_source_attr_release(struct config_item *item) +{ + struct audio_source_instance *fi_audio = to_audio_source_instance(item); + + usb_put_function_instance(&fi_audio->func_inst); +} + +static int audio_source_set_inst_name(struct usb_function_instance *fi, + const char *name) +{ + struct audio_source_instance *fi_audio; + char *ptr; + int name_len; + + name_len = strlen(name) + 1; + if (name_len > MAX_INST_NAME_LEN) + return -ENAMETOOLONG; + + ptr = kstrndup(name, name_len, GFP_KERNEL); + if (!ptr) + return -ENOMEM; + + fi_audio = to_fi_audio_source(fi); + fi_audio->name = ptr; + + return 0; +} + +static void audio_source_free_inst(struct usb_function_instance *fi) +{ + struct audio_source_instance *fi_audio; + + fi_audio = to_fi_audio_source(fi); + device_destroy(fi_audio->audio_device->class, + fi_audio->audio_device->devt); + kfree(fi_audio->name); + kfree(fi_audio->config); +} + +static ssize_t audio_source_pcm_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct audio_source_instance *fi_audio = dev_get_drvdata(dev); + struct audio_source_config *config = fi_audio->config; + + /* print PCM card and device numbers */ + return sprintf(buf, "%d %d\n", config->card, config->device); +} + +struct device *create_function_device(char *name); + +static struct usb_function_instance *audio_source_alloc_inst(void) +{ + struct audio_source_instance *fi_audio; + struct device_attribute **attrs; + struct device_attribute *attr; + struct device *dev; + void *err_ptr; + int err = 0; + + fi_audio = kzalloc(sizeof(*fi_audio), GFP_KERNEL); + if (!fi_audio) + return ERR_PTR(-ENOMEM); + + fi_audio->func_inst.set_inst_name = audio_source_set_inst_name; + fi_audio->func_inst.free_func_inst = audio_source_free_inst; + + fi_audio->config = kzalloc(sizeof(struct audio_source_config), + GFP_KERNEL); + if (!fi_audio->config) { + err_ptr = ERR_PTR(-ENOMEM); + goto fail_audio; + } + + config_group_init_type_name(&fi_audio->func_inst.group, "", + &audio_source_func_type); + dev = create_function_device("f_audio_source"); + + if (IS_ERR(dev)) { + err_ptr = dev; + goto fail_audio_config; + } + + fi_audio->config->card = -1; + fi_audio->config->device = -1; + fi_audio->audio_device = dev; + + attrs = audio_source_function_attributes; + if (attrs) { + while ((attr = *attrs++) && !err) + err = device_create_file(dev, attr); + if (err) { + err_ptr = ERR_PTR(-EINVAL); + goto fail_device; + } + } + + dev_set_drvdata(dev, fi_audio); + _audio_dev.config = fi_audio->config; + + return &fi_audio->func_inst; + +fail_device: + device_destroy(dev->class, dev->devt); +fail_audio_config: + kfree(fi_audio->config); +fail_audio: + kfree(fi_audio); + return err_ptr; + +} + +static struct usb_function *audio_source_alloc(struct usb_function_instance *fi) +{ + return &_audio_dev.func; +} + +DECLARE_USB_FUNCTION_INIT(audio_source, audio_source_alloc_inst, + audio_source_alloc); +MODULE_LICENSE("GPL"); diff --git a/drivers/usb/gadget/function/f_fs.c b/drivers/usb/gadget/function/f_fs.c index 17989b72cdaec1..8d412d8b1f292f 100644 --- a/drivers/usb/gadget/function/f_fs.c +++ b/drivers/usb/gadget/function/f_fs.c @@ -2269,6 +2269,8 @@ static int __ffs_data_do_os_desc(enum ffs_os_desc_type type, if (len < sizeof(*d) || h->interface >= ffs->interfaces_count) return -EINVAL; length = le32_to_cpu(d->dwSize); + if (len < length) + return -EINVAL; type = le32_to_cpu(d->dwPropertyDataType); if (type < USB_EXT_PROP_UNICODE || type > USB_EXT_PROP_UNICODE_MULTI) { @@ -2277,6 +2279,11 @@ static int __ffs_data_do_os_desc(enum ffs_os_desc_type type, return -EINVAL; } pnl = le16_to_cpu(d->wPropertyNameLength); + if (length < 14 + pnl) { + pr_vdebug("invalid os descriptor length: %d pnl:%d (descriptor %d)\n", + length, pnl, type); + return -EINVAL; + } pdl = le32_to_cpu(*(u32 *)((u8 *)data + 10 + pnl)); if (length != 14 + pnl + pdl) { pr_vdebug("invalid os descriptor length: %d pnl:%d pdl:%d (descriptor %d)\n", @@ -2363,6 +2370,9 @@ static int __ffs_data_got_descs(struct ffs_data *ffs, } } if (flags & (1 << i)) { + if (len < 4) { + goto error; + } os_descs_count = get_unaligned_le32(data); data += 4; len -= 4; @@ -2435,7 +2445,8 @@ static int __ffs_data_got_strings(struct ffs_data *ffs, ENTER(); - if (unlikely(get_unaligned_le32(data) != FUNCTIONFS_STRINGS_MAGIC || + if (unlikely(len < 16 || + get_unaligned_le32(data) != FUNCTIONFS_STRINGS_MAGIC || get_unaligned_le32(data + 4) != len)) goto error; str_count = get_unaligned_le32(data + 8); diff --git a/drivers/usb/gadget/function/f_midi.c b/drivers/usb/gadget/function/f_midi.c index a5719f271bf0b7..a832d271739c66 100644 --- a/drivers/usb/gadget/function/f_midi.c +++ b/drivers/usb/gadget/function/f_midi.c @@ -1167,6 +1167,65 @@ static void f_midi_free_inst(struct usb_function_instance *f) kfree(opts); } +#ifdef CONFIG_USB_CONFIGFS_UEVENT +extern struct device *create_function_device(char *name); +static ssize_t alsa_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct usb_function_instance *fi_midi = dev_get_drvdata(dev); + struct f_midi *midi; + + if (!fi_midi->f) + dev_warn(dev, "f_midi: function not set\n"); + + if (fi_midi && fi_midi->f) { + midi = func_to_midi(fi_midi->f); + if (midi->rmidi && midi->rmidi->card) + return sprintf(buf, "%d %d\n", + midi->rmidi->card->number, midi->rmidi->device); + } + + /* print PCM card and device numbers */ + return sprintf(buf, "%d %d\n", -1, -1); +} + +static DEVICE_ATTR(alsa, S_IRUGO, alsa_show, NULL); + +static struct device_attribute *alsa_function_attributes[] = { + &dev_attr_alsa, + NULL +}; + +static int create_alsa_device(struct usb_function_instance *fi) +{ + struct device *dev; + struct device_attribute **attrs; + struct device_attribute *attr; + int err = 0; + + dev = create_function_device("f_midi"); + if (IS_ERR(dev)) + return PTR_ERR(dev); + + attrs = alsa_function_attributes; + if (attrs) { + while ((attr = *attrs++) && !err) + err = device_create_file(dev, attr); + if (err) { + device_destroy(dev->class, dev->devt); + return -EINVAL; + } + } + dev_set_drvdata(dev, fi); + return 0; +} +#else +static int create_alsa_device(struct usb_function_instance *fi) +{ + return 0; +} +#endif + static struct usb_function_instance *f_midi_alloc_inst(void) { struct f_midi_opts *opts; @@ -1184,6 +1243,11 @@ static struct usb_function_instance *f_midi_alloc_inst(void) opts->in_ports = 1; opts->out_ports = 1; + if (create_alsa_device(&opts->func_inst)) { + kfree(opts); + return ERR_PTR(-ENODEV); + } + config_group_init_type_name(&opts->func_inst.group, "", &midi_func_type); @@ -1201,6 +1265,7 @@ static void f_midi_free(struct usb_function *f) mutex_lock(&opts->lock); kfifo_free(&midi->in_req_fifo); kfree(midi); + opts->func_inst.f = NULL; --opts->refcnt; mutex_unlock(&opts->lock); } @@ -1280,6 +1345,7 @@ static struct usb_function *f_midi_alloc(struct usb_function_instance *fi) midi->func.disable = f_midi_disable; midi->func.free_func = f_midi_free; + fi->f = &midi->func; return &midi->func; setup_fail: diff --git a/drivers/usb/gadget/function/f_mtp.c b/drivers/usb/gadget/function/f_mtp.c new file mode 100644 index 00000000000000..5cdee7bc5d38d9 --- /dev/null +++ b/drivers/usb/gadget/function/f_mtp.c @@ -0,0 +1,1536 @@ +/* + * Gadget Function Driver for MTP + * + * Copyright (C) 2010 Google, Inc. + * Author: Mike Lockwood + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +/* #define DEBUG */ +/* #define VERBOSE_DEBUG */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "configfs.h" + +#define MTP_BULK_BUFFER_SIZE 16384 +#define INTR_BUFFER_SIZE 28 +#define MAX_INST_NAME_LEN 40 + +/* String IDs */ +#define INTERFACE_STRING_INDEX 0 + +/* values for mtp_dev.state */ +#define STATE_OFFLINE 0 /* initial state, disconnected */ +#define STATE_READY 1 /* ready for userspace calls */ +#define STATE_BUSY 2 /* processing userspace calls */ +#define STATE_CANCELED 3 /* transaction canceled by host */ +#define STATE_ERROR 4 /* error from completion routine */ + +/* number of tx and rx requests to allocate */ +#define TX_REQ_MAX 4 +#define RX_REQ_MAX 2 +#define INTR_REQ_MAX 5 + +/* ID for Microsoft MTP OS String */ +#define MTP_OS_STRING_ID 0xEE + +/* MTP class reqeusts */ +#define MTP_REQ_CANCEL 0x64 +#define MTP_REQ_GET_EXT_EVENT_DATA 0x65 +#define MTP_REQ_RESET 0x66 +#define MTP_REQ_GET_DEVICE_STATUS 0x67 + +/* constants for device status */ +#define MTP_RESPONSE_OK 0x2001 +#define MTP_RESPONSE_DEVICE_BUSY 0x2019 +#define DRIVER_NAME "mtp" + +static const char mtp_shortname[] = DRIVER_NAME "_usb"; + +struct mtp_dev { + struct usb_function function; + struct usb_composite_dev *cdev; + spinlock_t lock; + + struct usb_ep *ep_in; + struct usb_ep *ep_out; + struct usb_ep *ep_intr; + + int state; + + /* synchronize access to our device file */ + atomic_t open_excl; + /* to enforce only one ioctl at a time */ + atomic_t ioctl_excl; + + struct list_head tx_idle; + struct list_head intr_idle; + + wait_queue_head_t read_wq; + wait_queue_head_t write_wq; + wait_queue_head_t intr_wq; + struct usb_request *rx_req[RX_REQ_MAX]; + int rx_done; + + /* for processing MTP_SEND_FILE, MTP_RECEIVE_FILE and + * MTP_SEND_FILE_WITH_HEADER ioctls on a work queue + */ + struct workqueue_struct *wq; + struct work_struct send_file_work; + struct work_struct receive_file_work; + struct file *xfer_file; + loff_t xfer_file_offset; + int64_t xfer_file_length; + unsigned xfer_send_header; + uint16_t xfer_command; + uint32_t xfer_transaction_id; + int xfer_result; +}; + +static struct usb_interface_descriptor mtp_interface_desc = { + .bLength = USB_DT_INTERFACE_SIZE, + .bDescriptorType = USB_DT_INTERFACE, + .bInterfaceNumber = 0, + .bNumEndpoints = 3, + .bInterfaceClass = USB_CLASS_VENDOR_SPEC, + .bInterfaceSubClass = USB_SUBCLASS_VENDOR_SPEC, + .bInterfaceProtocol = 0, +}; + +static struct usb_interface_descriptor ptp_interface_desc = { + .bLength = USB_DT_INTERFACE_SIZE, + .bDescriptorType = USB_DT_INTERFACE, + .bInterfaceNumber = 0, + .bNumEndpoints = 3, + .bInterfaceClass = USB_CLASS_STILL_IMAGE, + .bInterfaceSubClass = 1, + .bInterfaceProtocol = 1, +}; + +static struct usb_endpoint_descriptor mtp_ss_in_desc = { + .bLength = USB_DT_ENDPOINT_SIZE, + .bDescriptorType = USB_DT_ENDPOINT, + .bEndpointAddress = USB_DIR_IN, + .bmAttributes = USB_ENDPOINT_XFER_BULK, + .wMaxPacketSize = __constant_cpu_to_le16(1024), +}; + +static struct usb_ss_ep_comp_descriptor mtp_ss_in_comp_desc = { + .bLength = sizeof(mtp_ss_in_comp_desc), + .bDescriptorType = USB_DT_SS_ENDPOINT_COMP, + /* .bMaxBurst = DYNAMIC, */ +}; + +static struct usb_endpoint_descriptor mtp_ss_out_desc = { + .bLength = USB_DT_ENDPOINT_SIZE, + .bDescriptorType = USB_DT_ENDPOINT, + .bEndpointAddress = USB_DIR_OUT, + .bmAttributes = USB_ENDPOINT_XFER_BULK, + .wMaxPacketSize = __constant_cpu_to_le16(1024), +}; + +static struct usb_ss_ep_comp_descriptor mtp_ss_out_comp_desc = { + .bLength = sizeof(mtp_ss_out_comp_desc), + .bDescriptorType = USB_DT_SS_ENDPOINT_COMP, + /* .bMaxBurst = DYNAMIC, */ +}; + +static struct usb_endpoint_descriptor mtp_highspeed_in_desc = { + .bLength = USB_DT_ENDPOINT_SIZE, + .bDescriptorType = USB_DT_ENDPOINT, + .bEndpointAddress = USB_DIR_IN, + .bmAttributes = USB_ENDPOINT_XFER_BULK, + .wMaxPacketSize = __constant_cpu_to_le16(512), +}; + +static struct usb_endpoint_descriptor mtp_highspeed_out_desc = { + .bLength = USB_DT_ENDPOINT_SIZE, + .bDescriptorType = USB_DT_ENDPOINT, + .bEndpointAddress = USB_DIR_OUT, + .bmAttributes = USB_ENDPOINT_XFER_BULK, + .wMaxPacketSize = __constant_cpu_to_le16(512), +}; + +static struct usb_endpoint_descriptor mtp_fullspeed_in_desc = { + .bLength = USB_DT_ENDPOINT_SIZE, + .bDescriptorType = USB_DT_ENDPOINT, + .bEndpointAddress = USB_DIR_IN, + .bmAttributes = USB_ENDPOINT_XFER_BULK, +}; + +static struct usb_endpoint_descriptor mtp_fullspeed_out_desc = { + .bLength = USB_DT_ENDPOINT_SIZE, + .bDescriptorType = USB_DT_ENDPOINT, + .bEndpointAddress = USB_DIR_OUT, + .bmAttributes = USB_ENDPOINT_XFER_BULK, +}; + +static struct usb_endpoint_descriptor mtp_intr_desc = { + .bLength = USB_DT_ENDPOINT_SIZE, + .bDescriptorType = USB_DT_ENDPOINT, + .bEndpointAddress = USB_DIR_IN, + .bmAttributes = USB_ENDPOINT_XFER_INT, + .wMaxPacketSize = __constant_cpu_to_le16(INTR_BUFFER_SIZE), + .bInterval = 6, +}; + +static struct usb_ss_ep_comp_descriptor mtp_intr_ss_comp_desc = { + .bLength = sizeof(mtp_intr_ss_comp_desc), + .bDescriptorType = USB_DT_SS_ENDPOINT_COMP, + .wBytesPerInterval = cpu_to_le16(INTR_BUFFER_SIZE), +}; + +static struct usb_descriptor_header *fs_mtp_descs[] = { + (struct usb_descriptor_header *) &mtp_interface_desc, + (struct usb_descriptor_header *) &mtp_fullspeed_in_desc, + (struct usb_descriptor_header *) &mtp_fullspeed_out_desc, + (struct usb_descriptor_header *) &mtp_intr_desc, + NULL, +}; + +static struct usb_descriptor_header *hs_mtp_descs[] = { + (struct usb_descriptor_header *) &mtp_interface_desc, + (struct usb_descriptor_header *) &mtp_highspeed_in_desc, + (struct usb_descriptor_header *) &mtp_highspeed_out_desc, + (struct usb_descriptor_header *) &mtp_intr_desc, + NULL, +}; + +static struct usb_descriptor_header *ss_mtp_descs[] = { + (struct usb_descriptor_header *) &mtp_interface_desc, + (struct usb_descriptor_header *) &mtp_ss_in_desc, + (struct usb_descriptor_header *) &mtp_ss_in_comp_desc, + (struct usb_descriptor_header *) &mtp_ss_out_desc, + (struct usb_descriptor_header *) &mtp_ss_out_comp_desc, + (struct usb_descriptor_header *) &mtp_intr_desc, + (struct usb_descriptor_header *) &mtp_intr_ss_comp_desc, + NULL, +}; + +static struct usb_descriptor_header *fs_ptp_descs[] = { + (struct usb_descriptor_header *) &ptp_interface_desc, + (struct usb_descriptor_header *) &mtp_fullspeed_in_desc, + (struct usb_descriptor_header *) &mtp_fullspeed_out_desc, + (struct usb_descriptor_header *) &mtp_intr_desc, + NULL, +}; + +static struct usb_descriptor_header *hs_ptp_descs[] = { + (struct usb_descriptor_header *) &ptp_interface_desc, + (struct usb_descriptor_header *) &mtp_highspeed_in_desc, + (struct usb_descriptor_header *) &mtp_highspeed_out_desc, + (struct usb_descriptor_header *) &mtp_intr_desc, + NULL, +}; + +static struct usb_descriptor_header *ss_ptp_descs[] = { + (struct usb_descriptor_header *) &ptp_interface_desc, + (struct usb_descriptor_header *) &mtp_ss_in_desc, + (struct usb_descriptor_header *) &mtp_ss_in_comp_desc, + (struct usb_descriptor_header *) &mtp_ss_out_desc, + (struct usb_descriptor_header *) &mtp_ss_out_comp_desc, + (struct usb_descriptor_header *) &mtp_intr_desc, + (struct usb_descriptor_header *) &mtp_intr_ss_comp_desc, + NULL, +}; + +static struct usb_string mtp_string_defs[] = { + /* Naming interface "MTP" so libmtp will recognize us */ + [INTERFACE_STRING_INDEX].s = "MTP", + { }, /* end of list */ +}; + +static struct usb_gadget_strings mtp_string_table = { + .language = 0x0409, /* en-US */ + .strings = mtp_string_defs, +}; + +static struct usb_gadget_strings *mtp_strings[] = { + &mtp_string_table, + NULL, +}; + +/* Microsoft MTP OS String */ +static u8 mtp_os_string[] = { + 18, /* sizeof(mtp_os_string) */ + USB_DT_STRING, + /* Signature field: "MSFT100" */ + 'M', 0, 'S', 0, 'F', 0, 'T', 0, '1', 0, '0', 0, '0', 0, + /* vendor code */ + 1, + /* padding */ + 0 +}; + +/* Microsoft Extended Configuration Descriptor Header Section */ +struct mtp_ext_config_desc_header { + __le32 dwLength; + __u16 bcdVersion; + __le16 wIndex; + __u8 bCount; + __u8 reserved[7]; +}; + +/* Microsoft Extended Configuration Descriptor Function Section */ +struct mtp_ext_config_desc_function { + __u8 bFirstInterfaceNumber; + __u8 bInterfaceCount; + __u8 compatibleID[8]; + __u8 subCompatibleID[8]; + __u8 reserved[6]; +}; + +/* MTP Extended Configuration Descriptor */ +struct { + struct mtp_ext_config_desc_header header; + struct mtp_ext_config_desc_function function; +} mtp_ext_config_desc = { + .header = { + .dwLength = __constant_cpu_to_le32(sizeof(mtp_ext_config_desc)), + .bcdVersion = __constant_cpu_to_le16(0x0100), + .wIndex = __constant_cpu_to_le16(4), + .bCount = 1, + }, + .function = { + .bFirstInterfaceNumber = 0, + .bInterfaceCount = 1, + .compatibleID = { 'M', 'T', 'P' }, + }, +}; + +struct mtp_device_status { + __le16 wLength; + __le16 wCode; +}; + +struct mtp_data_header { + /* length of packet, including this header */ + __le32 length; + /* container type (2 for data packet) */ + __le16 type; + /* MTP command code */ + __le16 command; + /* MTP transaction ID */ + __le32 transaction_id; +}; + +struct mtp_instance { + struct usb_function_instance func_inst; + const char *name; + struct mtp_dev *dev; + char mtp_ext_compat_id[16]; + struct usb_os_desc mtp_os_desc; +}; + +/* temporary variable used between mtp_open() and mtp_gadget_bind() */ +static struct mtp_dev *_mtp_dev; + +static inline struct mtp_dev *func_to_mtp(struct usb_function *f) +{ + return container_of(f, struct mtp_dev, function); +} + +static struct usb_request *mtp_request_new(struct usb_ep *ep, int buffer_size) +{ + struct usb_request *req = usb_ep_alloc_request(ep, GFP_KERNEL); + + if (!req) + return NULL; + + /* now allocate buffers for the requests */ + req->buf = kmalloc(buffer_size, GFP_KERNEL); + if (!req->buf) { + usb_ep_free_request(ep, req); + return NULL; + } + + return req; +} + +static void mtp_request_free(struct usb_request *req, struct usb_ep *ep) +{ + if (req) { + kfree(req->buf); + usb_ep_free_request(ep, req); + } +} + +static inline int mtp_lock(atomic_t *excl) +{ + if (atomic_inc_return(excl) == 1) { + return 0; + } else { + atomic_dec(excl); + return -1; + } +} + +static inline void mtp_unlock(atomic_t *excl) +{ + atomic_dec(excl); +} + +/* add a request to the tail of a list */ +static void mtp_req_put(struct mtp_dev *dev, struct list_head *head, + struct usb_request *req) +{ + unsigned long flags; + + spin_lock_irqsave(&dev->lock, flags); + list_add_tail(&req->list, head); + spin_unlock_irqrestore(&dev->lock, flags); +} + +/* remove a request from the head of a list */ +static struct usb_request +*mtp_req_get(struct mtp_dev *dev, struct list_head *head) +{ + unsigned long flags; + struct usb_request *req; + + spin_lock_irqsave(&dev->lock, flags); + if (list_empty(head)) { + req = 0; + } else { + req = list_first_entry(head, struct usb_request, list); + list_del(&req->list); + } + spin_unlock_irqrestore(&dev->lock, flags); + return req; +} + +static void mtp_complete_in(struct usb_ep *ep, struct usb_request *req) +{ + struct mtp_dev *dev = _mtp_dev; + + if (req->status != 0) + dev->state = STATE_ERROR; + + mtp_req_put(dev, &dev->tx_idle, req); + + wake_up(&dev->write_wq); +} + +static void mtp_complete_out(struct usb_ep *ep, struct usb_request *req) +{ + struct mtp_dev *dev = _mtp_dev; + + dev->rx_done = 1; + if (req->status != 0) + dev->state = STATE_ERROR; + + wake_up(&dev->read_wq); +} + +static void mtp_complete_intr(struct usb_ep *ep, struct usb_request *req) +{ + struct mtp_dev *dev = _mtp_dev; + + if (req->status != 0) + dev->state = STATE_ERROR; + + mtp_req_put(dev, &dev->intr_idle, req); + + wake_up(&dev->intr_wq); +} + +static int mtp_create_bulk_endpoints(struct mtp_dev *dev, + struct usb_endpoint_descriptor *in_desc, + struct usb_endpoint_descriptor *out_desc, + struct usb_endpoint_descriptor *intr_desc) +{ + struct usb_composite_dev *cdev = dev->cdev; + struct usb_request *req; + struct usb_ep *ep; + int i; + + DBG(cdev, "create_bulk_endpoints dev: %p\n", dev); + + ep = usb_ep_autoconfig(cdev->gadget, in_desc); + if (!ep) { + DBG(cdev, "usb_ep_autoconfig for ep_in failed\n"); + return -ENODEV; + } + DBG(cdev, "usb_ep_autoconfig for ep_in got %s\n", ep->name); + ep->driver_data = dev; /* claim the endpoint */ + dev->ep_in = ep; + + ep = usb_ep_autoconfig(cdev->gadget, out_desc); + if (!ep) { + DBG(cdev, "usb_ep_autoconfig for ep_out failed\n"); + return -ENODEV; + } + DBG(cdev, "usb_ep_autoconfig for mtp ep_out got %s\n", ep->name); + ep->driver_data = dev; /* claim the endpoint */ + dev->ep_out = ep; + + ep = usb_ep_autoconfig(cdev->gadget, intr_desc); + if (!ep) { + DBG(cdev, "usb_ep_autoconfig for ep_intr failed\n"); + return -ENODEV; + } + DBG(cdev, "usb_ep_autoconfig for mtp ep_intr got %s\n", ep->name); + ep->driver_data = dev; /* claim the endpoint */ + dev->ep_intr = ep; + + /* now allocate requests for our endpoints */ + for (i = 0; i < TX_REQ_MAX; i++) { + req = mtp_request_new(dev->ep_in, MTP_BULK_BUFFER_SIZE); + if (!req) + goto fail; + req->complete = mtp_complete_in; + mtp_req_put(dev, &dev->tx_idle, req); + } + for (i = 0; i < RX_REQ_MAX; i++) { + req = mtp_request_new(dev->ep_out, MTP_BULK_BUFFER_SIZE); + if (!req) + goto fail; + req->complete = mtp_complete_out; + dev->rx_req[i] = req; + } + for (i = 0; i < INTR_REQ_MAX; i++) { + req = mtp_request_new(dev->ep_intr, INTR_BUFFER_SIZE); + if (!req) + goto fail; + req->complete = mtp_complete_intr; + mtp_req_put(dev, &dev->intr_idle, req); + } + + return 0; + +fail: + pr_err("mtp_bind() could not allocate requests\n"); + return -1; +} + +static ssize_t mtp_read(struct file *fp, char __user *buf, + size_t count, loff_t *pos) +{ + struct mtp_dev *dev = fp->private_data; + struct usb_composite_dev *cdev = dev->cdev; + struct usb_request *req; + ssize_t r = count; + unsigned xfer; + int ret = 0; + + DBG(cdev, "mtp_read(%zu)\n", count); + + if (count > MTP_BULK_BUFFER_SIZE) + return -EINVAL; + + /* we will block until we're online */ + DBG(cdev, "mtp_read: waiting for online state\n"); + ret = wait_event_interruptible(dev->read_wq, + dev->state != STATE_OFFLINE); + if (ret < 0) { + r = ret; + goto done; + } + spin_lock_irq(&dev->lock); + if (dev->state == STATE_CANCELED) { + /* report cancelation to userspace */ + dev->state = STATE_READY; + spin_unlock_irq(&dev->lock); + return -ECANCELED; + } + dev->state = STATE_BUSY; + spin_unlock_irq(&dev->lock); + +requeue_req: + /* queue a request */ + req = dev->rx_req[0]; + req->length = count; + dev->rx_done = 0; + ret = usb_ep_queue(dev->ep_out, req, GFP_KERNEL); + if (ret < 0) { + r = -EIO; + goto done; + } else { + DBG(cdev, "rx %p queue\n", req); + } + + /* wait for a request to complete */ + ret = wait_event_interruptible(dev->read_wq, dev->rx_done); + if (ret < 0) { + r = ret; + usb_ep_dequeue(dev->ep_out, req); + goto done; + } + if (dev->state == STATE_BUSY) { + /* If we got a 0-len packet, throw it back and try again. */ + if (req->actual == 0) + goto requeue_req; + + DBG(cdev, "rx %p %d\n", req, req->actual); + xfer = (req->actual < count) ? req->actual : count; + r = xfer; + if (copy_to_user(buf, req->buf, xfer)) + r = -EFAULT; + } else + r = -EIO; + +done: + spin_lock_irq(&dev->lock); + if (dev->state == STATE_CANCELED) + r = -ECANCELED; + else if (dev->state != STATE_OFFLINE) + dev->state = STATE_READY; + spin_unlock_irq(&dev->lock); + + DBG(cdev, "mtp_read returning %zd\n", r); + return r; +} + +static ssize_t mtp_write(struct file *fp, const char __user *buf, + size_t count, loff_t *pos) +{ + struct mtp_dev *dev = fp->private_data; + struct usb_composite_dev *cdev = dev->cdev; + struct usb_request *req = 0; + ssize_t r = count; + unsigned xfer; + int sendZLP = 0; + int ret; + + DBG(cdev, "mtp_write(%zu)\n", count); + + spin_lock_irq(&dev->lock); + if (dev->state == STATE_CANCELED) { + /* report cancelation to userspace */ + dev->state = STATE_READY; + spin_unlock_irq(&dev->lock); + return -ECANCELED; + } + if (dev->state == STATE_OFFLINE) { + spin_unlock_irq(&dev->lock); + return -ENODEV; + } + dev->state = STATE_BUSY; + spin_unlock_irq(&dev->lock); + + /* we need to send a zero length packet to signal the end of transfer + * if the transfer size is aligned to a packet boundary. + */ + if ((count & (dev->ep_in->maxpacket - 1)) == 0) + sendZLP = 1; + + while (count > 0 || sendZLP) { + /* so we exit after sending ZLP */ + if (count == 0) + sendZLP = 0; + + if (dev->state != STATE_BUSY) { + DBG(cdev, "mtp_write dev->error\n"); + r = -EIO; + break; + } + + /* get an idle tx request to use */ + req = 0; + ret = wait_event_interruptible(dev->write_wq, + ((req = mtp_req_get(dev, &dev->tx_idle)) + || dev->state != STATE_BUSY)); + if (!req) { + r = ret; + break; + } + + if (count > MTP_BULK_BUFFER_SIZE) + xfer = MTP_BULK_BUFFER_SIZE; + else + xfer = count; + if (xfer && copy_from_user(req->buf, buf, xfer)) { + r = -EFAULT; + break; + } + + req->length = xfer; + ret = usb_ep_queue(dev->ep_in, req, GFP_KERNEL); + if (ret < 0) { + DBG(cdev, "mtp_write: xfer error %d\n", ret); + r = -EIO; + break; + } + + buf += xfer; + count -= xfer; + + /* zero this so we don't try to free it on error exit */ + req = 0; + } + + if (req) + mtp_req_put(dev, &dev->tx_idle, req); + + spin_lock_irq(&dev->lock); + if (dev->state == STATE_CANCELED) + r = -ECANCELED; + else if (dev->state != STATE_OFFLINE) + dev->state = STATE_READY; + spin_unlock_irq(&dev->lock); + + DBG(cdev, "mtp_write returning %zd\n", r); + return r; +} + +/* read from a local file and write to USB */ +static void send_file_work(struct work_struct *data) +{ + struct mtp_dev *dev = container_of(data, struct mtp_dev, + send_file_work); + struct usb_composite_dev *cdev = dev->cdev; + struct usb_request *req = 0; + struct mtp_data_header *header; + struct file *filp; + loff_t offset; + int64_t count; + int xfer, ret, hdr_size; + int r = 0; + int sendZLP = 0; + + /* read our parameters */ + smp_rmb(); + filp = dev->xfer_file; + offset = dev->xfer_file_offset; + count = dev->xfer_file_length; + + DBG(cdev, "send_file_work(%lld %lld)\n", offset, count); + + if (dev->xfer_send_header) { + hdr_size = sizeof(struct mtp_data_header); + count += hdr_size; + } else { + hdr_size = 0; + } + + /* we need to send a zero length packet to signal the end of transfer + * if the transfer size is aligned to a packet boundary. + */ + if ((count & (dev->ep_in->maxpacket - 1)) == 0) + sendZLP = 1; + + while (count > 0 || sendZLP) { + /* so we exit after sending ZLP */ + if (count == 0) + sendZLP = 0; + + /* get an idle tx request to use */ + req = 0; + ret = wait_event_interruptible(dev->write_wq, + (req = mtp_req_get(dev, &dev->tx_idle)) + || dev->state != STATE_BUSY); + if (dev->state == STATE_CANCELED) { + r = -ECANCELED; + break; + } + if (!req) { + r = ret; + break; + } + + if (count > MTP_BULK_BUFFER_SIZE) + xfer = MTP_BULK_BUFFER_SIZE; + else + xfer = count; + + if (hdr_size) { + /* prepend MTP data header */ + header = (struct mtp_data_header *)req->buf; + header->length = __cpu_to_le32(count); + header->type = __cpu_to_le16(2); /* data packet */ + header->command = __cpu_to_le16(dev->xfer_command); + header->transaction_id = + __cpu_to_le32(dev->xfer_transaction_id); + } + + ret = vfs_read(filp, req->buf + hdr_size, xfer - hdr_size, + &offset); + if (ret < 0) { + r = ret; + break; + } + xfer = ret + hdr_size; + hdr_size = 0; + + req->length = xfer; + ret = usb_ep_queue(dev->ep_in, req, GFP_KERNEL); + if (ret < 0) { + DBG(cdev, "send_file_work: xfer error %d\n", ret); + dev->state = STATE_ERROR; + r = -EIO; + break; + } + + count -= xfer; + + /* zero this so we don't try to free it on error exit */ + req = 0; + } + + if (req) + mtp_req_put(dev, &dev->tx_idle, req); + + DBG(cdev, "send_file_work returning %d\n", r); + /* write the result */ + dev->xfer_result = r; + smp_wmb(); +} + +/* read from USB and write to a local file */ +static void receive_file_work(struct work_struct *data) +{ + struct mtp_dev *dev = container_of(data, struct mtp_dev, + receive_file_work); + struct usb_composite_dev *cdev = dev->cdev; + struct usb_request *read_req = NULL, *write_req = NULL; + struct file *filp; + loff_t offset; + int64_t count; + int ret, cur_buf = 0; + int r = 0; + + /* read our parameters */ + smp_rmb(); + filp = dev->xfer_file; + offset = dev->xfer_file_offset; + count = dev->xfer_file_length; + + DBG(cdev, "receive_file_work(%lld)\n", count); + + while (count > 0 || write_req) { + if (count > 0) { + /* queue a request */ + read_req = dev->rx_req[cur_buf]; + cur_buf = (cur_buf + 1) % RX_REQ_MAX; + + read_req->length = (count > MTP_BULK_BUFFER_SIZE + ? MTP_BULK_BUFFER_SIZE : count); + dev->rx_done = 0; + ret = usb_ep_queue(dev->ep_out, read_req, GFP_KERNEL); + if (ret < 0) { + r = -EIO; + dev->state = STATE_ERROR; + break; + } + } + + if (write_req) { + DBG(cdev, "rx %p %d\n", write_req, write_req->actual); + ret = vfs_write(filp, write_req->buf, write_req->actual, + &offset); + DBG(cdev, "vfs_write %d\n", ret); + if (ret != write_req->actual) { + r = -EIO; + dev->state = STATE_ERROR; + break; + } + write_req = NULL; + } + + if (read_req) { + /* wait for our last read to complete */ + ret = wait_event_interruptible(dev->read_wq, + dev->rx_done || dev->state != STATE_BUSY); + if (dev->state == STATE_CANCELED) { + r = -ECANCELED; + if (!dev->rx_done) + usb_ep_dequeue(dev->ep_out, read_req); + break; + } + /* if xfer_file_length is 0xFFFFFFFF, then we read until + * we get a zero length packet + */ + if (count != 0xFFFFFFFF) + count -= read_req->actual; + if (read_req->actual < read_req->length) { + /* + * short packet is used to signal EOF for + * sizes > 4 gig + */ + DBG(cdev, "got short packet\n"); + count = 0; + } + + write_req = read_req; + read_req = NULL; + } + } + + DBG(cdev, "receive_file_work returning %d\n", r); + /* write the result */ + dev->xfer_result = r; + smp_wmb(); +} + +static int mtp_send_event(struct mtp_dev *dev, struct mtp_event *event) +{ + struct usb_request *req = NULL; + int ret; + int length = event->length; + + DBG(dev->cdev, "mtp_send_event(%zu)\n", event->length); + + if (length < 0 || length > INTR_BUFFER_SIZE) + return -EINVAL; + if (dev->state == STATE_OFFLINE) + return -ENODEV; + + ret = wait_event_interruptible_timeout(dev->intr_wq, + (req = mtp_req_get(dev, &dev->intr_idle)), + msecs_to_jiffies(1000)); + if (!req) + return -ETIME; + + if (copy_from_user(req->buf, (void __user *)event->data, length)) { + mtp_req_put(dev, &dev->intr_idle, req); + return -EFAULT; + } + req->length = length; + ret = usb_ep_queue(dev->ep_intr, req, GFP_KERNEL); + if (ret) + mtp_req_put(dev, &dev->intr_idle, req); + + return ret; +} + +static long mtp_ioctl(struct file *fp, unsigned code, unsigned long value) +{ + struct mtp_dev *dev = fp->private_data; + struct file *filp = NULL; + int ret = -EINVAL; + + if (mtp_lock(&dev->ioctl_excl)) + return -EBUSY; + + switch (code) { + case MTP_SEND_FILE: + case MTP_RECEIVE_FILE: + case MTP_SEND_FILE_WITH_HEADER: + { + struct mtp_file_range mfr; + struct work_struct *work; + + spin_lock_irq(&dev->lock); + if (dev->state == STATE_CANCELED) { + /* report cancelation to userspace */ + dev->state = STATE_READY; + spin_unlock_irq(&dev->lock); + ret = -ECANCELED; + goto out; + } + if (dev->state == STATE_OFFLINE) { + spin_unlock_irq(&dev->lock); + ret = -ENODEV; + goto out; + } + dev->state = STATE_BUSY; + spin_unlock_irq(&dev->lock); + + if (copy_from_user(&mfr, (void __user *)value, sizeof(mfr))) { + ret = -EFAULT; + goto fail; + } + /* hold a reference to the file while we are working with it */ + filp = fget(mfr.fd); + if (!filp) { + ret = -EBADF; + goto fail; + } + + /* write the parameters */ + dev->xfer_file = filp; + dev->xfer_file_offset = mfr.offset; + dev->xfer_file_length = mfr.length; + smp_wmb(); + + if (code == MTP_SEND_FILE_WITH_HEADER) { + work = &dev->send_file_work; + dev->xfer_send_header = 1; + dev->xfer_command = mfr.command; + dev->xfer_transaction_id = mfr.transaction_id; + } else if (code == MTP_SEND_FILE) { + work = &dev->send_file_work; + dev->xfer_send_header = 0; + } else { + work = &dev->receive_file_work; + } + + /* We do the file transfer on a work queue so it will run + * in kernel context, which is necessary for vfs_read and + * vfs_write to use our buffers in the kernel address space. + */ + queue_work(dev->wq, work); + /* wait for operation to complete */ + flush_workqueue(dev->wq); + fput(filp); + + /* read the result */ + smp_rmb(); + ret = dev->xfer_result; + break; + } + case MTP_SEND_EVENT: + { + struct mtp_event event; + /* return here so we don't change dev->state below, + * which would interfere with bulk transfer state. + */ + if (copy_from_user(&event, (void __user *)value, sizeof(event))) + ret = -EFAULT; + else + ret = mtp_send_event(dev, &event); + goto out; + } + } + +fail: + spin_lock_irq(&dev->lock); + if (dev->state == STATE_CANCELED) + ret = -ECANCELED; + else if (dev->state != STATE_OFFLINE) + dev->state = STATE_READY; + spin_unlock_irq(&dev->lock); +out: + mtp_unlock(&dev->ioctl_excl); + DBG(dev->cdev, "ioctl returning %d\n", ret); + return ret; +} + +static int mtp_open(struct inode *ip, struct file *fp) +{ + printk(KERN_INFO "mtp_open\n"); + if (mtp_lock(&_mtp_dev->open_excl)) + return -EBUSY; + + /* clear any error condition */ + if (_mtp_dev->state != STATE_OFFLINE) + _mtp_dev->state = STATE_READY; + + fp->private_data = _mtp_dev; + return 0; +} + +static int mtp_release(struct inode *ip, struct file *fp) +{ + printk(KERN_INFO "mtp_release\n"); + + mtp_unlock(&_mtp_dev->open_excl); + return 0; +} + +/* file operations for /dev/mtp_usb */ +static const struct file_operations mtp_fops = { + .owner = THIS_MODULE, + .read = mtp_read, + .write = mtp_write, + .unlocked_ioctl = mtp_ioctl, + .open = mtp_open, + .release = mtp_release, +}; + +static struct miscdevice mtp_device = { + .minor = MISC_DYNAMIC_MINOR, + .name = mtp_shortname, + .fops = &mtp_fops, +}; + +static int mtp_ctrlrequest(struct usb_composite_dev *cdev, + const struct usb_ctrlrequest *ctrl) +{ + struct mtp_dev *dev = _mtp_dev; + int value = -EOPNOTSUPP; + u16 w_index = le16_to_cpu(ctrl->wIndex); + u16 w_value = le16_to_cpu(ctrl->wValue); + u16 w_length = le16_to_cpu(ctrl->wLength); + unsigned long flags; + + VDBG(cdev, "mtp_ctrlrequest " + "%02x.%02x v%04x i%04x l%u\n", + ctrl->bRequestType, ctrl->bRequest, + w_value, w_index, w_length); + + /* Handle MTP OS string */ + if (ctrl->bRequestType == + (USB_DIR_IN | USB_TYPE_STANDARD | USB_RECIP_DEVICE) + && ctrl->bRequest == USB_REQ_GET_DESCRIPTOR + && (w_value >> 8) == USB_DT_STRING + && (w_value & 0xFF) == MTP_OS_STRING_ID) { + value = (w_length < sizeof(mtp_os_string) + ? w_length : sizeof(mtp_os_string)); + memcpy(cdev->req->buf, mtp_os_string, value); + } else if ((ctrl->bRequestType & USB_TYPE_MASK) == USB_TYPE_VENDOR) { + /* Handle MTP OS descriptor */ + DBG(cdev, "vendor request: %d index: %d value: %d length: %d\n", + ctrl->bRequest, w_index, w_value, w_length); + + if (ctrl->bRequest == 1 + && (ctrl->bRequestType & USB_DIR_IN) + && (w_index == 4 || w_index == 5)) { + value = (w_length < sizeof(mtp_ext_config_desc) ? + w_length : sizeof(mtp_ext_config_desc)); + memcpy(cdev->req->buf, &mtp_ext_config_desc, value); + } + } else if ((ctrl->bRequestType & USB_TYPE_MASK) == USB_TYPE_CLASS) { + DBG(cdev, "class request: %d index: %d value: %d length: %d\n", + ctrl->bRequest, w_index, w_value, w_length); + + if (ctrl->bRequest == MTP_REQ_CANCEL && w_index == 0 + && w_value == 0) { + DBG(cdev, "MTP_REQ_CANCEL\n"); + + spin_lock_irqsave(&dev->lock, flags); + if (dev->state == STATE_BUSY) { + dev->state = STATE_CANCELED; + wake_up(&dev->read_wq); + wake_up(&dev->write_wq); + } + spin_unlock_irqrestore(&dev->lock, flags); + + /* We need to queue a request to read the remaining + * bytes, but we don't actually need to look at + * the contents. + */ + value = w_length; + } else if (ctrl->bRequest == MTP_REQ_GET_DEVICE_STATUS + && w_index == 0 && w_value == 0) { + struct mtp_device_status *status = cdev->req->buf; + + status->wLength = + __constant_cpu_to_le16(sizeof(*status)); + + DBG(cdev, "MTP_REQ_GET_DEVICE_STATUS\n"); + spin_lock_irqsave(&dev->lock, flags); + /* device status is "busy" until we report + * the cancelation to userspace + */ + if (dev->state == STATE_CANCELED) + status->wCode = + __cpu_to_le16(MTP_RESPONSE_DEVICE_BUSY); + else + status->wCode = + __cpu_to_le16(MTP_RESPONSE_OK); + spin_unlock_irqrestore(&dev->lock, flags); + value = sizeof(*status); + } + } + + /* respond with data transfer or status phase? */ + if (value >= 0) { + int rc; + + cdev->req->zero = value < w_length; + cdev->req->length = value; + rc = usb_ep_queue(cdev->gadget->ep0, cdev->req, GFP_ATOMIC); + if (rc < 0) + ERROR(cdev, "%s: response queue error\n", __func__); + } + return value; +} + +static int +mtp_function_bind(struct usb_configuration *c, struct usb_function *f) +{ + struct usb_composite_dev *cdev = c->cdev; + struct mtp_dev *dev = func_to_mtp(f); + int id; + int ret; + struct mtp_instance *fi_mtp; + + dev->cdev = cdev; + DBG(cdev, "mtp_function_bind dev: %p\n", dev); + + /* allocate interface ID(s) */ + id = usb_interface_id(c, f); + if (id < 0) + return id; + mtp_interface_desc.bInterfaceNumber = id; + + if (mtp_string_defs[INTERFACE_STRING_INDEX].id == 0) { + ret = usb_string_id(c->cdev); + if (ret < 0) + return ret; + mtp_string_defs[INTERFACE_STRING_INDEX].id = ret; + mtp_interface_desc.iInterface = ret; + } + + fi_mtp = container_of(f->fi, struct mtp_instance, func_inst); + + if (cdev->use_os_string) { + f->os_desc_table = kzalloc(sizeof(*f->os_desc_table), + GFP_KERNEL); + if (!f->os_desc_table) + return -ENOMEM; + f->os_desc_n = 1; + f->os_desc_table[0].os_desc = &fi_mtp->mtp_os_desc; + } + + /* allocate endpoints */ + ret = mtp_create_bulk_endpoints(dev, &mtp_fullspeed_in_desc, + &mtp_fullspeed_out_desc, &mtp_intr_desc); + if (ret) + return ret; + + /* support high speed hardware */ + if (gadget_is_dualspeed(c->cdev->gadget)) { + mtp_highspeed_in_desc.bEndpointAddress = + mtp_fullspeed_in_desc.bEndpointAddress; + mtp_highspeed_out_desc.bEndpointAddress = + mtp_fullspeed_out_desc.bEndpointAddress; + } + /* support super speed hardware */ + if (gadget_is_superspeed(c->cdev->gadget)) { + unsigned max_burst; + + /* Calculate bMaxBurst, we know packet size is 1024 */ + max_burst = min_t(unsigned, MTP_BULK_BUFFER_SIZE / 1024, 15); + mtp_ss_in_desc.bEndpointAddress = + mtp_fullspeed_in_desc.bEndpointAddress; + mtp_ss_in_comp_desc.bMaxBurst = max_burst; + mtp_ss_out_desc.bEndpointAddress = + mtp_fullspeed_out_desc.bEndpointAddress; + mtp_ss_out_comp_desc.bMaxBurst = max_burst; + } + + DBG(cdev, "%s speed %s: IN/%s, OUT/%s\n", + gadget_is_superspeed(c->cdev->gadget) ? "super" : + (gadget_is_dualspeed(c->cdev->gadget) ? "dual" : "full"), + f->name, dev->ep_in->name, dev->ep_out->name); + return 0; +} + +static void +mtp_function_unbind(struct usb_configuration *c, struct usb_function *f) +{ + struct mtp_dev *dev = func_to_mtp(f); + struct usb_request *req; + int i; + + mtp_string_defs[INTERFACE_STRING_INDEX].id = 0; + while ((req = mtp_req_get(dev, &dev->tx_idle))) + mtp_request_free(req, dev->ep_in); + for (i = 0; i < RX_REQ_MAX; i++) + mtp_request_free(dev->rx_req[i], dev->ep_out); + while ((req = mtp_req_get(dev, &dev->intr_idle))) + mtp_request_free(req, dev->ep_intr); + dev->state = STATE_OFFLINE; + kfree(f->os_desc_table); + f->os_desc_n = 0; +} + +static int mtp_function_set_alt(struct usb_function *f, + unsigned intf, unsigned alt) +{ + struct mtp_dev *dev = func_to_mtp(f); + struct usb_composite_dev *cdev = f->config->cdev; + int ret; + + DBG(cdev, "mtp_function_set_alt intf: %d alt: %d\n", intf, alt); + + ret = config_ep_by_speed(cdev->gadget, f, dev->ep_in); + if (ret) + return ret; + + ret = usb_ep_enable(dev->ep_in); + if (ret) + return ret; + + ret = config_ep_by_speed(cdev->gadget, f, dev->ep_out); + if (ret) + return ret; + + ret = usb_ep_enable(dev->ep_out); + if (ret) { + usb_ep_disable(dev->ep_in); + return ret; + } + + ret = config_ep_by_speed(cdev->gadget, f, dev->ep_intr); + if (ret) + return ret; + + ret = usb_ep_enable(dev->ep_intr); + if (ret) { + usb_ep_disable(dev->ep_out); + usb_ep_disable(dev->ep_in); + return ret; + } + dev->state = STATE_READY; + + /* readers may be blocked waiting for us to go online */ + wake_up(&dev->read_wq); + return 0; +} + +static void mtp_function_disable(struct usb_function *f) +{ + struct mtp_dev *dev = func_to_mtp(f); + struct usb_composite_dev *cdev = dev->cdev; + + DBG(cdev, "mtp_function_disable\n"); + dev->state = STATE_OFFLINE; + usb_ep_disable(dev->ep_in); + usb_ep_disable(dev->ep_out); + usb_ep_disable(dev->ep_intr); + + /* readers may be blocked waiting for us to go online */ + wake_up(&dev->read_wq); + + VDBG(cdev, "%s disabled\n", dev->function.name); +} + +static int __mtp_setup(struct mtp_instance *fi_mtp) +{ + struct mtp_dev *dev; + int ret; + + dev = kzalloc(sizeof(*dev), GFP_KERNEL); + + if (fi_mtp != NULL) + fi_mtp->dev = dev; + + if (!dev) + return -ENOMEM; + + spin_lock_init(&dev->lock); + init_waitqueue_head(&dev->read_wq); + init_waitqueue_head(&dev->write_wq); + init_waitqueue_head(&dev->intr_wq); + atomic_set(&dev->open_excl, 0); + atomic_set(&dev->ioctl_excl, 0); + INIT_LIST_HEAD(&dev->tx_idle); + INIT_LIST_HEAD(&dev->intr_idle); + + dev->wq = create_singlethread_workqueue("f_mtp"); + if (!dev->wq) { + ret = -ENOMEM; + goto err1; + } + INIT_WORK(&dev->send_file_work, send_file_work); + INIT_WORK(&dev->receive_file_work, receive_file_work); + + _mtp_dev = dev; + + ret = misc_register(&mtp_device); + if (ret) + goto err2; + + return 0; + +err2: + destroy_workqueue(dev->wq); +err1: + _mtp_dev = NULL; + kfree(dev); + printk(KERN_ERR "mtp gadget driver failed to initialize\n"); + return ret; +} + +static int mtp_setup_configfs(struct mtp_instance *fi_mtp) +{ + return __mtp_setup(fi_mtp); +} + + +static void mtp_cleanup(void) +{ + struct mtp_dev *dev = _mtp_dev; + + if (!dev) + return; + + misc_deregister(&mtp_device); + destroy_workqueue(dev->wq); + _mtp_dev = NULL; + kfree(dev); +} + +static struct mtp_instance *to_mtp_instance(struct config_item *item) +{ + return container_of(to_config_group(item), struct mtp_instance, + func_inst.group); +} + +static void mtp_attr_release(struct config_item *item) +{ + struct mtp_instance *fi_mtp = to_mtp_instance(item); + + usb_put_function_instance(&fi_mtp->func_inst); +} + +static struct configfs_item_operations mtp_item_ops = { + .release = mtp_attr_release, +}; + +static struct config_item_type mtp_func_type = { + .ct_item_ops = &mtp_item_ops, + .ct_owner = THIS_MODULE, +}; + + +static struct mtp_instance *to_fi_mtp(struct usb_function_instance *fi) +{ + return container_of(fi, struct mtp_instance, func_inst); +} + +static int mtp_set_inst_name(struct usb_function_instance *fi, const char *name) +{ + struct mtp_instance *fi_mtp; + char *ptr; + int name_len; + + name_len = strlen(name) + 1; + if (name_len > MAX_INST_NAME_LEN) + return -ENAMETOOLONG; + + ptr = kstrndup(name, name_len, GFP_KERNEL); + if (!ptr) + return -ENOMEM; + + fi_mtp = to_fi_mtp(fi); + fi_mtp->name = ptr; + + return 0; +} + +static void mtp_free_inst(struct usb_function_instance *fi) +{ + struct mtp_instance *fi_mtp; + + fi_mtp = to_fi_mtp(fi); + kfree(fi_mtp->name); + mtp_cleanup(); + kfree(fi_mtp); +} + +struct usb_function_instance *alloc_inst_mtp_ptp(bool mtp_config) +{ + struct mtp_instance *fi_mtp; + int ret = 0; + struct usb_os_desc *descs[1]; + char *names[1]; + + fi_mtp = kzalloc(sizeof(*fi_mtp), GFP_KERNEL); + if (!fi_mtp) + return ERR_PTR(-ENOMEM); + fi_mtp->func_inst.set_inst_name = mtp_set_inst_name; + fi_mtp->func_inst.free_func_inst = mtp_free_inst; + + fi_mtp->mtp_os_desc.ext_compat_id = fi_mtp->mtp_ext_compat_id; + INIT_LIST_HEAD(&fi_mtp->mtp_os_desc.ext_prop); + descs[0] = &fi_mtp->mtp_os_desc; + names[0] = "MTP"; + + if (mtp_config) { + ret = mtp_setup_configfs(fi_mtp); + if (ret) { + kfree(fi_mtp); + pr_err("Error setting MTP\n"); + return ERR_PTR(ret); + } + } else + fi_mtp->dev = _mtp_dev; + + config_group_init_type_name(&fi_mtp->func_inst.group, + "", &mtp_func_type); + usb_os_desc_prepare_interf_dir(&fi_mtp->func_inst.group, 1, + descs, names, THIS_MODULE); + + return &fi_mtp->func_inst; +} +EXPORT_SYMBOL_GPL(alloc_inst_mtp_ptp); + +static struct usb_function_instance *mtp_alloc_inst(void) +{ + return alloc_inst_mtp_ptp(true); +} + +static int mtp_ctrlreq_configfs(struct usb_function *f, + const struct usb_ctrlrequest *ctrl) +{ + return mtp_ctrlrequest(f->config->cdev, ctrl); +} + +static void mtp_free(struct usb_function *f) +{ + /*NO-OP: no function specific resource allocation in mtp_alloc*/ +} + +struct usb_function *function_alloc_mtp_ptp(struct usb_function_instance *fi, + bool mtp_config) +{ + struct mtp_instance *fi_mtp = to_fi_mtp(fi); + struct mtp_dev *dev; + + /* + * PTP piggybacks on MTP function so make sure we have + * created MTP function before we associate this PTP + * function with a gadget configuration. + */ + if (fi_mtp->dev == NULL) { + pr_err("Error: Create MTP function before linking" + " PTP function with a gadget configuration\n"); + pr_err("\t1: Delete existing PTP function if any\n"); + pr_err("\t2: Create MTP function\n"); + pr_err("\t3: Create and symlink PTP function" + " with a gadget configuration\n"); + return ERR_PTR(-EINVAL); /* Invalid Configuration */ + } + + dev = fi_mtp->dev; + dev->function.name = DRIVER_NAME; + dev->function.strings = mtp_strings; + if (mtp_config) { + dev->function.fs_descriptors = fs_mtp_descs; + dev->function.hs_descriptors = hs_mtp_descs; + dev->function.ss_descriptors = ss_mtp_descs; + } else { + dev->function.fs_descriptors = fs_ptp_descs; + dev->function.hs_descriptors = hs_ptp_descs; + dev->function.ss_descriptors = ss_ptp_descs; + } + dev->function.bind = mtp_function_bind; + dev->function.unbind = mtp_function_unbind; + dev->function.set_alt = mtp_function_set_alt; + dev->function.disable = mtp_function_disable; + dev->function.setup = mtp_ctrlreq_configfs; + dev->function.free_func = mtp_free; + + return &dev->function; +} +EXPORT_SYMBOL_GPL(function_alloc_mtp_ptp); + +static struct usb_function *mtp_alloc(struct usb_function_instance *fi) +{ + return function_alloc_mtp_ptp(fi, true); +} + +DECLARE_USB_FUNCTION_INIT(mtp, mtp_alloc_inst, mtp_alloc); +MODULE_LICENSE("GPL"); diff --git a/drivers/usb/gadget/function/f_mtp.h b/drivers/usb/gadget/function/f_mtp.h new file mode 100644 index 00000000000000..7adb1ff08eff12 --- /dev/null +++ b/drivers/usb/gadget/function/f_mtp.h @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2014 Google, Inc. + * Author: Badhri Jagan Sridharan + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +extern struct usb_function_instance *alloc_inst_mtp_ptp(bool mtp_config); +extern struct usb_function *function_alloc_mtp_ptp( + struct usb_function_instance *fi, bool mtp_config); diff --git a/drivers/usb/gadget/function/f_ptp.c b/drivers/usb/gadget/function/f_ptp.c new file mode 100644 index 00000000000000..da3e4d53e085bc --- /dev/null +++ b/drivers/usb/gadget/function/f_ptp.c @@ -0,0 +1,38 @@ +/* + * Gadget Function Driver for PTP + * + * Copyright (C) 2014 Google, Inc. + * Author: Badhri Jagan Sridharan + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include + +#include +#include + +#include "f_mtp.h" + +static struct usb_function_instance *ptp_alloc_inst(void) +{ + return alloc_inst_mtp_ptp(false); +} + +static struct usb_function *ptp_alloc(struct usb_function_instance *fi) +{ + return function_alloc_mtp_ptp(fi, false); +} + +DECLARE_USB_FUNCTION_INIT(ptp, ptp_alloc_inst, ptp_alloc); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Badhri Jagan Sridharan"); diff --git a/drivers/usb/gadget/function/f_tcm.c b/drivers/usb/gadget/function/f_tcm.c index 197f73386fac9a..d2351139342f62 100644 --- a/drivers/usb/gadget/function/f_tcm.c +++ b/drivers/usb/gadget/function/f_tcm.c @@ -1073,7 +1073,7 @@ static struct usbg_cmd *usbg_get_cmd(struct f_uas *fu, struct usbg_cmd *cmd; int tag; - tag = percpu_ida_alloc(&se_sess->sess_tag_pool, GFP_ATOMIC); + tag = percpu_ida_alloc(&se_sess->sess_tag_pool, TASK_RUNNING); if (tag < 0) return ERR_PTR(-ENOMEM); diff --git a/drivers/usb/gadget/function/f_uac2.c b/drivers/usb/gadget/function/f_uac2.c index cd214ec8a601b4..969cfe7413808c 100644 --- a/drivers/usb/gadget/function/f_uac2.c +++ b/drivers/usb/gadget/function/f_uac2.c @@ -1067,13 +1067,13 @@ afunc_bind(struct usb_configuration *cfg, struct usb_function *fn) agdev->out_ep = usb_ep_autoconfig(gadget, &fs_epout_desc); if (!agdev->out_ep) { dev_err(dev, "%s:%d Error!\n", __func__, __LINE__); - goto err; + return ret; } agdev->in_ep = usb_ep_autoconfig(gadget, &fs_epin_desc); if (!agdev->in_ep) { dev_err(dev, "%s:%d Error!\n", __func__, __LINE__); - goto err; + return ret; } uac2->p_prm.uac2 = uac2; @@ -1091,7 +1091,7 @@ afunc_bind(struct usb_configuration *cfg, struct usb_function *fn) ret = usb_assign_descriptors(fn, fs_audio_desc, hs_audio_desc, NULL, NULL); if (ret) - goto err; + return ret; prm = &agdev->uac2.c_prm; prm->max_psize = hs_epout_desc.wMaxPacketSize; @@ -1106,19 +1106,19 @@ afunc_bind(struct usb_configuration *cfg, struct usb_function *fn) prm->rbuf = kzalloc(prm->max_psize * USB_XFERS, GFP_KERNEL); if (!prm->rbuf) { prm->max_psize = 0; - goto err_free_descs; + goto err; } ret = alsa_uac2_init(agdev); if (ret) - goto err_free_descs; + goto err; return 0; -err_free_descs: - usb_free_all_descriptors(fn); err: kfree(agdev->uac2.p_prm.rbuf); kfree(agdev->uac2.c_prm.rbuf); +err_free_descs: + usb_free_all_descriptors(fn); return -EINVAL; } diff --git a/drivers/usb/gadget/function/uvc_video.c b/drivers/usb/gadget/function/uvc_video.c index 3d0d5d94a62f2f..0f01c04d7cbd85 100644 --- a/drivers/usb/gadget/function/uvc_video.c +++ b/drivers/usb/gadget/function/uvc_video.c @@ -243,7 +243,7 @@ uvc_video_alloc_requests(struct uvc_video *video) req_size = video->ep->maxpacket * max_t(unsigned int, video->ep->maxburst, 1) - * (video->ep->mult + 1); + * (video->ep->mult); for (i = 0; i < UVC_NUM_REQUESTS; ++i) { video->req_buffer[i] = kmalloc(req_size, GFP_KERNEL); diff --git a/drivers/usb/gadget/legacy/inode.c b/drivers/usb/gadget/legacy/inode.c index bd82dd12deffd2..1468d8f085a397 100644 --- a/drivers/usb/gadget/legacy/inode.c +++ b/drivers/usb/gadget/legacy/inode.c @@ -1126,7 +1126,7 @@ ep0_write (struct file *fd, const char __user *buf, size_t len, loff_t *ptr) /* data and/or status stage for control request */ } else if (dev->state == STATE_DEV_SETUP) { - /* IN DATA+STATUS caller makes len <= wLength */ + len = min_t(size_t, len, dev->setup_wLength); if (dev->setup_in) { retval = setup_req (dev->gadget->ep0, dev->req, len); if (retval == 0) { @@ -1734,10 +1734,12 @@ static struct usb_gadget_driver gadgetfs_driver = { * such as configuration notifications. */ -static int is_valid_config (struct usb_config_descriptor *config) +static int is_valid_config(struct usb_config_descriptor *config, + unsigned int total) { return config->bDescriptorType == USB_DT_CONFIG && config->bLength == USB_DT_CONFIG_SIZE + && total >= USB_DT_CONFIG_SIZE && config->bConfigurationValue != 0 && (config->bmAttributes & USB_CONFIG_ATT_ONE) != 0 && (config->bmAttributes & USB_CONFIG_ATT_WAKEUP) == 0; @@ -1762,7 +1764,8 @@ dev_config (struct file *fd, const char __user *buf, size_t len, loff_t *ptr) } spin_unlock_irq(&dev->lock); - if (len < (USB_DT_CONFIG_SIZE + USB_DT_DEVICE_SIZE + 4)) + if ((len < (USB_DT_CONFIG_SIZE + USB_DT_DEVICE_SIZE + 4)) || + (len > PAGE_SIZE * 4)) return -EINVAL; /* we might need to change message format someday */ @@ -1786,7 +1789,8 @@ dev_config (struct file *fd, const char __user *buf, size_t len, loff_t *ptr) /* full or low speed config */ dev->config = (void *) kbuf; total = le16_to_cpu(dev->config->wTotalLength); - if (!is_valid_config (dev->config) || total >= length) + if (!is_valid_config(dev->config, total) || + total > length - USB_DT_DEVICE_SIZE) goto fail; kbuf += total; length -= total; @@ -1795,10 +1799,13 @@ dev_config (struct file *fd, const char __user *buf, size_t len, loff_t *ptr) if (kbuf [1] == USB_DT_CONFIG) { dev->hs_config = (void *) kbuf; total = le16_to_cpu(dev->hs_config->wTotalLength); - if (!is_valid_config (dev->hs_config) || total >= length) + if (!is_valid_config(dev->hs_config, total) || + total > length - USB_DT_DEVICE_SIZE) goto fail; kbuf += total; length -= total; + } else { + dev->hs_config = NULL; } /* could support multiple configs, using another encoding! */ diff --git a/drivers/usb/gadget/udc/core.c b/drivers/usb/gadget/udc/core.c index 9483489080f662..0402177f93cdde 100644 --- a/drivers/usb/gadget/udc/core.c +++ b/drivers/usb/gadget/udc/core.c @@ -1317,7 +1317,11 @@ int usb_gadget_probe_driver(struct usb_gadget_driver *driver) if (!ret) break; } - if (!ret && !udc->driver) + if (ret) + ret = -ENODEV; + else if (udc->driver) + ret = -EBUSY; + else goto found; } else { list_for_each_entry(udc, &udc_list, list) { diff --git a/drivers/usb/gadget/udc/dummy_hcd.c b/drivers/usb/gadget/udc/dummy_hcd.c index 77d07904f932e7..a81d9ab861dc05 100644 --- a/drivers/usb/gadget/udc/dummy_hcd.c +++ b/drivers/usb/gadget/udc/dummy_hcd.c @@ -330,7 +330,7 @@ static void nuke(struct dummy *dum, struct dummy_ep *ep) /* caller must hold lock */ static void stop_activity(struct dummy *dum) { - struct dummy_ep *ep; + int i; /* prevent any more requests */ dum->address = 0; @@ -338,8 +338,8 @@ static void stop_activity(struct dummy *dum) /* The timer is left running so that outstanding URBs can fail */ /* nuke any pending requests first, so driver i/o is quiesced */ - list_for_each_entry(ep, &dum->gadget.ep_list, ep.ep_list) - nuke(dum, ep); + for (i = 0; i < DUMMY_ENDPOINTS; ++i) + nuke(dum, &dum->ep[i]); /* driver now does any non-usb quiescing necessary */ } diff --git a/drivers/usb/host/uhci-pci.c b/drivers/usb/host/uhci-pci.c index 940304c3322457..02260cfdedb135 100644 --- a/drivers/usb/host/uhci-pci.c +++ b/drivers/usb/host/uhci-pci.c @@ -129,6 +129,10 @@ static int uhci_pci_init(struct usb_hcd *hcd) if (to_pci_dev(uhci_dev(uhci))->vendor == PCI_VENDOR_ID_HP) uhci->wait_for_hp = 1; + /* Intel controllers use non-PME wakeup signalling */ + if (to_pci_dev(uhci_dev(uhci))->vendor == PCI_VENDOR_ID_INTEL) + device_set_run_wake(uhci_dev(uhci), 1); + /* Set up pointers to PCI-specific functions */ uhci->reset_hc = uhci_pci_reset_hc; uhci->check_and_reset_hc = uhci_pci_check_and_reset_hc; diff --git a/drivers/usb/host/xhci-mem.c b/drivers/usb/host/xhci-mem.c index 6afe32381209d7..7064892ff4a640 100644 --- a/drivers/usb/host/xhci-mem.c +++ b/drivers/usb/host/xhci-mem.c @@ -979,6 +979,40 @@ void xhci_free_virt_device(struct xhci_hcd *xhci, int slot_id) xhci->devs[slot_id] = NULL; } +/* + * Free a virt_device structure. + * If the virt_device added a tt_info (a hub) and has children pointing to + * that tt_info, then free the child first. Recursive. + * We can't rely on udev at this point to find child-parent relationships. + */ +void xhci_free_virt_devices_depth_first(struct xhci_hcd *xhci, int slot_id) +{ + struct xhci_virt_device *vdev; + struct list_head *tt_list_head; + struct xhci_tt_bw_info *tt_info, *next; + int i; + + vdev = xhci->devs[slot_id]; + if (!vdev) + return; + + tt_list_head = &(xhci->rh_bw[vdev->real_port - 1].tts); + list_for_each_entry_safe(tt_info, next, tt_list_head, tt_list) { + /* is this a hub device that added a tt_info to the tts list */ + if (tt_info->slot_id == slot_id) { + /* are any devices using this tt_info? */ + for (i = 1; i < HCS_MAX_SLOTS(xhci->hcs_params1); i++) { + vdev = xhci->devs[i]; + if (vdev && (vdev->tt_info == tt_info)) + xhci_free_virt_devices_depth_first( + xhci, i); + } + } + } + /* we are now at a leaf device */ + xhci_free_virt_device(xhci, slot_id); +} + int xhci_alloc_virt_device(struct xhci_hcd *xhci, int slot_id, struct usb_device *udev, gfp_t flags) { @@ -1796,7 +1830,7 @@ void xhci_mem_cleanup(struct xhci_hcd *xhci) int size; int i, j, num_ports; - del_timer_sync(&xhci->cmd_timer); + cancel_delayed_work_sync(&xhci->cmd_timer); /* Free the Event Ring Segment Table and the actual Event Ring */ size = sizeof(struct xhci_erst_entry)*(xhci->erst.num_entries); @@ -1829,8 +1863,8 @@ void xhci_mem_cleanup(struct xhci_hcd *xhci) } } - for (i = 1; i < MAX_HC_SLOTS; ++i) - xhci_free_virt_device(xhci, i); + for (i = HCS_MAX_SLOTS(xhci->hcs_params1); i > 0; i--) + xhci_free_virt_devices_depth_first(xhci, i); dma_pool_destroy(xhci->segment_pool); xhci->segment_pool = NULL; @@ -2343,9 +2377,9 @@ int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags) INIT_LIST_HEAD(&xhci->cmd_list); - /* init command timeout timer */ - setup_timer(&xhci->cmd_timer, xhci_handle_command_timeout, - (unsigned long)xhci); + /* init command timeout work */ + INIT_DELAYED_WORK(&xhci->cmd_timer, xhci_handle_command_timeout); + init_completion(&xhci->cmd_ring_stop_completion); page_size = readl(&xhci->op_regs->page_size); xhci_dbg_trace(xhci, trace_xhci_dbg_init, @@ -2384,7 +2418,7 @@ int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags) * "physically contiguous and 64-byte (cache line) aligned". */ xhci->dcbaa = dma_alloc_coherent(dev, sizeof(*xhci->dcbaa), &dma, - GFP_KERNEL); + flags); if (!xhci->dcbaa) goto fail; memset(xhci->dcbaa, 0, sizeof *(xhci->dcbaa)); @@ -2480,7 +2514,7 @@ int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags) xhci->erst.entries = dma_alloc_coherent(dev, sizeof(struct xhci_erst_entry) * ERST_NUM_SEGS, &dma, - GFP_KERNEL); + flags); if (!xhci->erst.entries) goto fail; xhci_dbg_trace(xhci, trace_xhci_dbg_init, diff --git a/drivers/usb/host/xhci-mtk.c b/drivers/usb/host/xhci-mtk.c index 79959f17c38cfe..f2365a47fa4a42 100644 --- a/drivers/usb/host/xhci-mtk.c +++ b/drivers/usb/host/xhci-mtk.c @@ -560,8 +560,10 @@ static int xhci_mtk_probe(struct platform_device *pdev) goto disable_ldos; irq = platform_get_irq(pdev, 0); - if (irq < 0) + if (irq < 0) { + ret = irq; goto disable_clk; + } /* Initialize dma_mask and coherent_dma_mask to 32-bits */ ret = dma_set_coherent_mask(dev, DMA_BIT_MASK(32)); diff --git a/drivers/usb/host/xhci-pci.c b/drivers/usb/host/xhci-pci.c index e96ae80d107e94..954abfd5014d28 100644 --- a/drivers/usb/host/xhci-pci.c +++ b/drivers/usb/host/xhci-pci.c @@ -165,7 +165,8 @@ static void xhci_pci_quirks(struct device *dev, struct xhci_hcd *xhci) pdev->device == PCI_DEVICE_ID_INTEL_SUNRISEPOINT_H_XHCI || pdev->device == PCI_DEVICE_ID_INTEL_CHERRYVIEW_XHCI || pdev->device == PCI_DEVICE_ID_INTEL_BROXTON_M_XHCI || - pdev->device == PCI_DEVICE_ID_INTEL_BROXTON_B_XHCI)) { + pdev->device == PCI_DEVICE_ID_INTEL_BROXTON_B_XHCI || + pdev->device == PCI_DEVICE_ID_INTEL_APL_XHCI)) { xhci->quirks |= XHCI_PME_STUCK_QUIRK; } if (pdev->vendor == PCI_VENDOR_ID_INTEL && diff --git a/drivers/usb/host/xhci-ring.c b/drivers/usb/host/xhci-ring.c index 797137e26549b5..521c1816a26a71 100644 --- a/drivers/usb/host/xhci-ring.c +++ b/drivers/usb/host/xhci-ring.c @@ -260,23 +260,76 @@ void xhci_ring_cmd_db(struct xhci_hcd *xhci) readl(&xhci->dba->doorbell[0]); } -static int xhci_abort_cmd_ring(struct xhci_hcd *xhci) +static bool xhci_mod_cmd_timer(struct xhci_hcd *xhci, unsigned long delay) +{ + return mod_delayed_work(system_wq, &xhci->cmd_timer, delay); +} + +static struct xhci_command *xhci_next_queued_cmd(struct xhci_hcd *xhci) +{ + return list_first_entry_or_null(&xhci->cmd_list, struct xhci_command, + cmd_list); +} + +/* + * Turn all commands on command ring with status set to "aborted" to no-op trbs. + * If there are other commands waiting then restart the ring and kick the timer. + * This must be called with command ring stopped and xhci->lock held. + */ +static void xhci_handle_stopped_cmd_ring(struct xhci_hcd *xhci, + struct xhci_command *cur_cmd) +{ + struct xhci_command *i_cmd; + u32 cycle_state; + + /* Turn all aborted commands in list to no-ops, then restart */ + list_for_each_entry(i_cmd, &xhci->cmd_list, cmd_list) { + + if (i_cmd->status != COMP_CMD_ABORT) + continue; + + i_cmd->status = COMP_CMD_STOP; + + xhci_dbg(xhci, "Turn aborted command %p to no-op\n", + i_cmd->command_trb); + /* get cycle state from the original cmd trb */ + cycle_state = le32_to_cpu( + i_cmd->command_trb->generic.field[3]) & TRB_CYCLE; + /* modify the command trb to no-op command */ + i_cmd->command_trb->generic.field[0] = 0; + i_cmd->command_trb->generic.field[1] = 0; + i_cmd->command_trb->generic.field[2] = 0; + i_cmd->command_trb->generic.field[3] = cpu_to_le32( + TRB_TYPE(TRB_CMD_NOOP) | cycle_state); + + /* + * caller waiting for completion is called when command + * completion event is received for these no-op commands + */ + } + + xhci->cmd_ring_state = CMD_RING_STATE_RUNNING; + + /* ring command ring doorbell to restart the command ring */ + if ((xhci->cmd_ring->dequeue != xhci->cmd_ring->enqueue) && + !(xhci->xhc_state & XHCI_STATE_DYING)) { + xhci->current_cmd = cur_cmd; + xhci_mod_cmd_timer(xhci, XHCI_CMD_DEFAULT_TIMEOUT); + xhci_ring_cmd_db(xhci); + } +} + +/* Must be called with xhci->lock held, releases and aquires lock back */ +static int xhci_abort_cmd_ring(struct xhci_hcd *xhci, unsigned long flags) { u64 temp_64; int ret; xhci_dbg(xhci, "Abort command ring\n"); - temp_64 = xhci_read_64(xhci, &xhci->op_regs->cmd_ring); - xhci->cmd_ring_state = CMD_RING_STATE_ABORTED; + reinit_completion(&xhci->cmd_ring_stop_completion); - /* - * Writing the CMD_RING_ABORT bit should cause a cmd completion event, - * however on some host hw the CMD_RING_RUNNING bit is correctly cleared - * but the completion event in never sent. Use the cmd timeout timer to - * handle those cases. Use twice the time to cover the bit polling retry - */ - mod_timer(&xhci->cmd_timer, jiffies + (2 * XHCI_CMD_DEFAULT_TIMEOUT)); + temp_64 = xhci_read_64(xhci, &xhci->op_regs->cmd_ring); xhci_write_64(xhci, temp_64 | CMD_RING_ABORT, &xhci->op_regs->cmd_ring); @@ -296,16 +349,30 @@ static int xhci_abort_cmd_ring(struct xhci_hcd *xhci) udelay(1000); ret = xhci_handshake(&xhci->op_regs->cmd_ring, CMD_RING_RUNNING, 0, 3 * 1000 * 1000); - if (ret == 0) - return 0; - - xhci_err(xhci, "Stopped the command ring failed, " - "maybe the host is dead\n"); - del_timer(&xhci->cmd_timer); - xhci->xhc_state |= XHCI_STATE_DYING; - xhci_quiesce(xhci); - xhci_halt(xhci); - return -ESHUTDOWN; + if (ret < 0) { + xhci_err(xhci, "Stopped the command ring failed, " + "maybe the host is dead\n"); + xhci->xhc_state |= XHCI_STATE_DYING; + xhci_quiesce(xhci); + xhci_halt(xhci); + return -ESHUTDOWN; + } + } + /* + * Writing the CMD_RING_ABORT bit should cause a cmd completion event, + * however on some host hw the CMD_RING_RUNNING bit is correctly cleared + * but the completion event in never sent. Wait 2 secs (arbitrary + * number) to handle those cases after negation of CMD_RING_RUNNING. + */ + spin_unlock_irqrestore(&xhci->lock, flags); + ret = wait_for_completion_timeout(&xhci->cmd_ring_stop_completion, + msecs_to_jiffies(2000)); + spin_lock_irqsave(&xhci->lock, flags); + if (!ret) { + xhci_dbg(xhci, "No stop event for abort, ring start fail?\n"); + xhci_cleanup_command_queue(xhci); + } else { + xhci_handle_stopped_cmd_ring(xhci, xhci_next_queued_cmd(xhci)); } return 0; @@ -850,17 +917,6 @@ void xhci_stop_endpoint_command_watchdog(unsigned long arg) spin_lock_irqsave(&xhci->lock, flags); ep->stop_cmds_pending--; - if (xhci->xhc_state & XHCI_STATE_REMOVING) { - spin_unlock_irqrestore(&xhci->lock, flags); - return; - } - if (xhci->xhc_state & XHCI_STATE_DYING) { - xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, - "Stop EP timer ran, but another timer marked " - "xHCI as DYING, exiting."); - spin_unlock_irqrestore(&xhci->lock, flags); - return; - } if (!(ep->stop_cmds_pending == 0 && (ep->ep_state & EP_HALT_PENDING))) { xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, "Stop EP timer ran, but no command pending, " @@ -1211,101 +1267,62 @@ void xhci_cleanup_command_queue(struct xhci_hcd *xhci) xhci_complete_del_and_free_cmd(cur_cmd, COMP_CMD_ABORT); } -/* - * Turn all commands on command ring with status set to "aborted" to no-op trbs. - * If there are other commands waiting then restart the ring and kick the timer. - * This must be called with command ring stopped and xhci->lock held. - */ -static void xhci_handle_stopped_cmd_ring(struct xhci_hcd *xhci, - struct xhci_command *cur_cmd) -{ - struct xhci_command *i_cmd, *tmp_cmd; - u32 cycle_state; - - /* Turn all aborted commands in list to no-ops, then restart */ - list_for_each_entry_safe(i_cmd, tmp_cmd, &xhci->cmd_list, - cmd_list) { - - if (i_cmd->status != COMP_CMD_ABORT) - continue; - - i_cmd->status = COMP_CMD_STOP; - - xhci_dbg(xhci, "Turn aborted command %p to no-op\n", - i_cmd->command_trb); - /* get cycle state from the original cmd trb */ - cycle_state = le32_to_cpu( - i_cmd->command_trb->generic.field[3]) & TRB_CYCLE; - /* modify the command trb to no-op command */ - i_cmd->command_trb->generic.field[0] = 0; - i_cmd->command_trb->generic.field[1] = 0; - i_cmd->command_trb->generic.field[2] = 0; - i_cmd->command_trb->generic.field[3] = cpu_to_le32( - TRB_TYPE(TRB_CMD_NOOP) | cycle_state); - - /* - * caller waiting for completion is called when command - * completion event is received for these no-op commands - */ - } - - xhci->cmd_ring_state = CMD_RING_STATE_RUNNING; - - /* ring command ring doorbell to restart the command ring */ - if ((xhci->cmd_ring->dequeue != xhci->cmd_ring->enqueue) && - !(xhci->xhc_state & XHCI_STATE_DYING)) { - xhci->current_cmd = cur_cmd; - mod_timer(&xhci->cmd_timer, jiffies + XHCI_CMD_DEFAULT_TIMEOUT); - xhci_ring_cmd_db(xhci); - } - return; -} - - -void xhci_handle_command_timeout(unsigned long data) +void xhci_handle_command_timeout(struct work_struct *work) { struct xhci_hcd *xhci; int ret; unsigned long flags; u64 hw_ring_state; - bool second_timeout = false; - xhci = (struct xhci_hcd *) data; - /* mark this command to be cancelled */ + xhci = container_of(to_delayed_work(work), struct xhci_hcd, cmd_timer); + spin_lock_irqsave(&xhci->lock, flags); - if (xhci->current_cmd) { - if (xhci->current_cmd->status == COMP_CMD_ABORT) - second_timeout = true; - xhci->current_cmd->status = COMP_CMD_ABORT; + + /* + * If timeout work is pending, or current_cmd is NULL, it means we + * raced with command completion. Command is handled so just return. + */ + if (!xhci->current_cmd || delayed_work_pending(&xhci->cmd_timer)) { + spin_unlock_irqrestore(&xhci->lock, flags); + return; } + /* mark this command to be cancelled */ + xhci->current_cmd->status = COMP_CMD_ABORT; /* Make sure command ring is running before aborting it */ hw_ring_state = xhci_read_64(xhci, &xhci->op_regs->cmd_ring); if ((xhci->cmd_ring_state & CMD_RING_STATE_RUNNING) && (hw_ring_state & CMD_RING_RUNNING)) { - spin_unlock_irqrestore(&xhci->lock, flags); + /* Prevent new doorbell, and start command abort */ + xhci->cmd_ring_state = CMD_RING_STATE_ABORTED; xhci_dbg(xhci, "Command timeout\n"); - ret = xhci_abort_cmd_ring(xhci); + ret = xhci_abort_cmd_ring(xhci, flags); if (unlikely(ret == -ESHUTDOWN)) { xhci_err(xhci, "Abort command ring failed\n"); xhci_cleanup_command_queue(xhci); + spin_unlock_irqrestore(&xhci->lock, flags); usb_hc_died(xhci_to_hcd(xhci)->primary_hcd); xhci_dbg(xhci, "xHCI host controller is dead.\n"); + + return; } - return; + + goto time_out_completed; } - /* command ring failed to restart, or host removed. Bail out */ - if (second_timeout || xhci->xhc_state & XHCI_STATE_REMOVING) { - spin_unlock_irqrestore(&xhci->lock, flags); - xhci_dbg(xhci, "command timed out twice, ring start fail?\n"); + /* host removed. Bail out */ + if (xhci->xhc_state & XHCI_STATE_REMOVING) { + xhci_dbg(xhci, "host removed, ring start fail?\n"); xhci_cleanup_command_queue(xhci); - return; + + goto time_out_completed; } /* command timeout on stopped ring, ring can't be aborted */ xhci_dbg(xhci, "Command timeout on stopped ring\n"); xhci_handle_stopped_cmd_ring(xhci, xhci->current_cmd); + +time_out_completed: spin_unlock_irqrestore(&xhci->lock, flags); return; } @@ -1338,7 +1355,7 @@ static void handle_cmd_completion(struct xhci_hcd *xhci, cmd = list_entry(xhci->cmd_list.next, struct xhci_command, cmd_list); - del_timer(&xhci->cmd_timer); + cancel_delayed_work(&xhci->cmd_timer); trace_xhci_cmd_completion(cmd_trb, (struct xhci_generic_trb *) event); @@ -1346,7 +1363,7 @@ static void handle_cmd_completion(struct xhci_hcd *xhci, /* If CMD ring stopped we own the trbs between enqueue and dequeue */ if (cmd_comp_code == COMP_CMD_STOP) { - xhci_handle_stopped_cmd_ring(xhci, cmd); + complete_all(&xhci->cmd_ring_stop_completion); return; } @@ -1364,8 +1381,11 @@ static void handle_cmd_completion(struct xhci_hcd *xhci, */ if (cmd_comp_code == COMP_CMD_ABORT) { xhci->cmd_ring_state = CMD_RING_STATE_STOPPED; - if (cmd->status == COMP_CMD_ABORT) + if (cmd->status == COMP_CMD_ABORT) { + if (xhci->current_cmd == cmd) + xhci->current_cmd = NULL; goto event_handled; + } } cmd_type = TRB_FIELD_TO_TYPE(le32_to_cpu(cmd_trb->generic.field[3])); @@ -1426,7 +1446,9 @@ static void handle_cmd_completion(struct xhci_hcd *xhci, if (cmd->cmd_list.next != &xhci->cmd_list) { xhci->current_cmd = list_entry(cmd->cmd_list.next, struct xhci_command, cmd_list); - mod_timer(&xhci->cmd_timer, jiffies + XHCI_CMD_DEFAULT_TIMEOUT); + xhci_mod_cmd_timer(xhci, XHCI_CMD_DEFAULT_TIMEOUT); + } else if (xhci->current_cmd == cmd) { + xhci->current_cmd = NULL; } event_handled: @@ -3920,9 +3942,9 @@ static int queue_command(struct xhci_hcd *xhci, struct xhci_command *cmd, /* if there are no other commands queued we start the timeout timer */ if (xhci->cmd_list.next == &cmd->cmd_list && - !timer_pending(&xhci->cmd_timer)) { + !delayed_work_pending(&xhci->cmd_timer)) { xhci->current_cmd = cmd; - mod_timer(&xhci->cmd_timer, jiffies + XHCI_CMD_DEFAULT_TIMEOUT); + xhci_mod_cmd_timer(xhci, XHCI_CMD_DEFAULT_TIMEOUT); } queue_trb(xhci, xhci->cmd_ring, false, field1, field2, field3, diff --git a/drivers/usb/host/xhci.c b/drivers/usb/host/xhci.c index 1a4ca02729c274..34e23c7d7797b9 100644 --- a/drivers/usb/host/xhci.c +++ b/drivers/usb/host/xhci.c @@ -1529,19 +1529,6 @@ int xhci_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status) xhci_urb_free_priv(urb_priv); return ret; } - if ((xhci->xhc_state & XHCI_STATE_DYING) || - (xhci->xhc_state & XHCI_STATE_HALTED)) { - xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, - "Ep 0x%x: URB %p to be canceled on " - "non-responsive xHCI host.", - urb->ep->desc.bEndpointAddress, urb); - /* Let the stop endpoint command watchdog timer (which set this - * state) finish cleaning up the endpoint TD lists. We must - * have caught it in the middle of dropping a lock and giving - * back an URB. - */ - goto done; - } ep_index = xhci_get_endpoint_index(&urb->ep->desc); ep = &xhci->devs[urb->dev->slot_id]->eps[ep_index]; @@ -3783,8 +3770,10 @@ static int xhci_setup_device(struct usb_hcd *hcd, struct usb_device *udev, mutex_lock(&xhci->mutex); - if (xhci->xhc_state) /* dying, removing or halted */ + if (xhci->xhc_state) { /* dying, removing or halted */ + ret = -ESHUTDOWN; goto out; + } if (!udev->slot_id) { xhci_dbg_trace(xhci, trace_xhci_dbg_address, diff --git a/drivers/usb/host/xhci.h b/drivers/usb/host/xhci.h index f945380035d07e..c525722aa934ca 100644 --- a/drivers/usb/host/xhci.h +++ b/drivers/usb/host/xhci.h @@ -1571,7 +1571,8 @@ struct xhci_hcd { #define CMD_RING_STATE_STOPPED (1 << 2) struct list_head cmd_list; unsigned int cmd_ring_reserved_trbs; - struct timer_list cmd_timer; + struct delayed_work cmd_timer; + struct completion cmd_ring_stop_completion; struct xhci_command *current_cmd; struct xhci_ring *event_ring; struct xhci_erst erst; @@ -1941,7 +1942,7 @@ void xhci_queue_config_ep_quirk(struct xhci_hcd *xhci, unsigned int slot_id, unsigned int ep_index, struct xhci_dequeue_state *deq_state); void xhci_stop_endpoint_command_watchdog(unsigned long arg); -void xhci_handle_command_timeout(unsigned long data); +void xhci_handle_command_timeout(struct work_struct *work); void xhci_ring_ep_doorbell(struct xhci_hcd *xhci, unsigned int slot_id, unsigned int ep_index, unsigned int stream_id); diff --git a/drivers/usb/musb/blackfin.c b/drivers/usb/musb/blackfin.c index 310238c6b5cd69..8967980718170d 100644 --- a/drivers/usb/musb/blackfin.c +++ b/drivers/usb/musb/blackfin.c @@ -469,6 +469,7 @@ static const struct musb_platform_ops bfin_ops = { .init = bfin_musb_init, .exit = bfin_musb_exit, + .fifo_offset = bfin_fifo_offset, .readb = bfin_readb, .writeb = bfin_writeb, .readw = bfin_readw, diff --git a/drivers/usb/musb/musb_core.c b/drivers/usb/musb/musb_core.c index c3e172e15ec3d9..338575fb2d2781 100644 --- a/drivers/usb/musb/musb_core.c +++ b/drivers/usb/musb/musb_core.c @@ -578,11 +578,11 @@ static irqreturn_t musb_stage0_irq(struct musb *musb, u8 int_usb, | MUSB_PORT_STAT_RESUME; musb->rh_timer = jiffies + msecs_to_jiffies(USB_RESUME_TIMEOUT); - musb->need_finish_resume = 1; - musb->xceiv->otg->state = OTG_STATE_A_HOST; musb->is_active = 1; musb_host_resume_root_hub(musb); + schedule_delayed_work(&musb->finish_resume_work, + msecs_to_jiffies(USB_RESUME_TIMEOUT)); break; case OTG_STATE_B_WAIT_ACON: musb->xceiv->otg->state = OTG_STATE_B_PERIPHERAL; @@ -2691,11 +2691,6 @@ static int musb_resume(struct device *dev) mask = MUSB_DEVCTL_BDEVICE | MUSB_DEVCTL_FSDEV | MUSB_DEVCTL_LSDEV; if ((devctl & mask) != (musb->context.devctl & mask)) musb->port1_status = 0; - if (musb->need_finish_resume) { - musb->need_finish_resume = 0; - schedule_delayed_work(&musb->finish_resume_work, - msecs_to_jiffies(USB_RESUME_TIMEOUT)); - } /* * The USB HUB code expects the device to be in RPM_ACTIVE once it came @@ -2747,12 +2742,6 @@ static int musb_runtime_resume(struct device *dev) musb_restore_context(musb); - if (musb->need_finish_resume) { - musb->need_finish_resume = 0; - schedule_delayed_work(&musb->finish_resume_work, - msecs_to_jiffies(USB_RESUME_TIMEOUT)); - } - spin_lock_irqsave(&musb->lock, flags); error = musb_run_resume_work(musb); if (error) diff --git a/drivers/usb/musb/musb_core.h b/drivers/usb/musb/musb_core.h index 91817d77d59c8e..854fbf7b6b2324 100644 --- a/drivers/usb/musb/musb_core.h +++ b/drivers/usb/musb/musb_core.h @@ -216,6 +216,7 @@ struct musb_platform_ops { void (*pre_root_reset_end)(struct musb *musb); void (*post_root_reset_end)(struct musb *musb); int (*phy_callback)(enum musb_vbus_id_status status); + void (*clear_ep_rxintr)(struct musb *musb, int epnum); }; /* @@ -409,7 +410,6 @@ struct musb { /* is_suspended means USB B_PERIPHERAL suspend */ unsigned is_suspended:1; - unsigned need_finish_resume :1; /* may_wakeup means remote wakeup is enabled */ unsigned may_wakeup:1; @@ -626,4 +626,10 @@ static inline void musb_platform_post_root_reset_end(struct musb *musb) musb->ops->post_root_reset_end(musb); } +static inline void musb_platform_clear_ep_rxintr(struct musb *musb, int epnum) +{ + if (musb->ops->clear_ep_rxintr) + musb->ops->clear_ep_rxintr(musb, epnum); +} + #endif /* __MUSB_CORE_H__ */ diff --git a/drivers/usb/musb/musb_debugfs.c b/drivers/usb/musb/musb_debugfs.c index 9b22d946c08969..534a3f6fa89c1e 100644 --- a/drivers/usb/musb/musb_debugfs.c +++ b/drivers/usb/musb/musb_debugfs.c @@ -114,6 +114,7 @@ static int musb_regdump_show(struct seq_file *s, void *unused) unsigned i; seq_printf(s, "MUSB (M)HDRC Register Dump\n"); + pm_runtime_get_sync(musb->controller); for (i = 0; i < ARRAY_SIZE(musb_regmap); i++) { switch (musb_regmap[i].size) { @@ -132,6 +133,8 @@ static int musb_regdump_show(struct seq_file *s, void *unused) } } + pm_runtime_mark_last_busy(musb->controller); + pm_runtime_put_autosuspend(musb->controller); return 0; } @@ -145,7 +148,10 @@ static int musb_test_mode_show(struct seq_file *s, void *unused) struct musb *musb = s->private; unsigned test; + pm_runtime_get_sync(musb->controller); test = musb_readb(musb->mregs, MUSB_TESTMODE); + pm_runtime_mark_last_busy(musb->controller); + pm_runtime_put_autosuspend(musb->controller); if (test & MUSB_TEST_FORCE_HOST) seq_printf(s, "force host\n"); @@ -194,11 +200,12 @@ static ssize_t musb_test_mode_write(struct file *file, u8 test; char buf[18]; + pm_runtime_get_sync(musb->controller); test = musb_readb(musb->mregs, MUSB_TESTMODE); if (test) { dev_err(musb->controller, "Error: test mode is already set. " "Please do USB Bus Reset to start a new test.\n"); - return count; + goto ret; } memset(buf, 0x00, sizeof(buf)); @@ -234,6 +241,9 @@ static ssize_t musb_test_mode_write(struct file *file, musb_writeb(musb->mregs, MUSB_TESTMODE, test); +ret: + pm_runtime_mark_last_busy(musb->controller); + pm_runtime_put_autosuspend(musb->controller); return count; } @@ -254,8 +264,13 @@ static int musb_softconnect_show(struct seq_file *s, void *unused) switch (musb->xceiv->otg->state) { case OTG_STATE_A_HOST: case OTG_STATE_A_WAIT_BCON: + pm_runtime_get_sync(musb->controller); + reg = musb_readb(musb->mregs, MUSB_DEVCTL); connect = reg & MUSB_DEVCTL_SESSION ? 1 : 0; + + pm_runtime_mark_last_busy(musb->controller); + pm_runtime_put_autosuspend(musb->controller); break; default: connect = -1; @@ -284,6 +299,7 @@ static ssize_t musb_softconnect_write(struct file *file, if (copy_from_user(&buf, ubuf, min_t(size_t, sizeof(buf) - 1, count))) return -EFAULT; + pm_runtime_get_sync(musb->controller); if (!strncmp(buf, "0", 1)) { switch (musb->xceiv->otg->state) { case OTG_STATE_A_HOST: @@ -314,6 +330,8 @@ static ssize_t musb_softconnect_write(struct file *file, } } + pm_runtime_mark_last_busy(musb->controller); + pm_runtime_put_autosuspend(musb->controller); return count; } diff --git a/drivers/usb/musb/musb_dsps.c b/drivers/usb/musb/musb_dsps.c index feae1561b9abb6..9f125e179acd44 100644 --- a/drivers/usb/musb/musb_dsps.c +++ b/drivers/usb/musb/musb_dsps.c @@ -267,6 +267,17 @@ static void otg_timer(unsigned long _musb) pm_runtime_put_autosuspend(dev); } +void dsps_musb_clear_ep_rxintr(struct musb *musb, int epnum) +{ + u32 epintr; + struct dsps_glue *glue = dev_get_drvdata(musb->controller->parent); + const struct dsps_musb_wrapper *wrp = glue->wrp; + + /* musb->lock might already been held */ + epintr = (1 << epnum) << wrp->rxep_shift; + musb_writel(musb->ctrl_base, wrp->epintr_status, epintr); +} + static irqreturn_t dsps_interrupt(int irq, void *hci) { struct musb *musb = hci; @@ -622,6 +633,7 @@ static struct musb_platform_ops dsps_ops = { .set_mode = dsps_musb_set_mode, .recover = dsps_musb_recover, + .clear_ep_rxintr = dsps_musb_clear_ep_rxintr, }; static u64 musb_dmamask = DMA_BIT_MASK(32); diff --git a/drivers/usb/musb/musb_host.c b/drivers/usb/musb/musb_host.c index 53bc4ceefe89ad..806451418cfe24 100644 --- a/drivers/usb/musb/musb_host.c +++ b/drivers/usb/musb/musb_host.c @@ -2374,12 +2374,11 @@ static int musb_cleanup_urb(struct urb *urb, struct musb_qh *qh) int is_in = usb_pipein(urb->pipe); int status = 0; u16 csr; + struct dma_channel *dma = NULL; musb_ep_select(regs, hw_end); if (is_dma_capable()) { - struct dma_channel *dma; - dma = is_in ? ep->rx_channel : ep->tx_channel; if (dma) { status = ep->musb->dma_controller->channel_abort(dma); @@ -2395,10 +2394,9 @@ static int musb_cleanup_urb(struct urb *urb, struct musb_qh *qh) /* giveback saves bulk toggle */ csr = musb_h_flush_rxfifo(ep, 0); - /* REVISIT we still get an irq; should likely clear the - * endpoint's irq status here to avoid bogus irqs. - * clearing that status is platform-specific... - */ + /* clear the endpoint's irq status here to avoid bogus irqs */ + if (is_dma_capable() && dma) + musb_platform_clear_ep_rxintr(musb, ep->epnum); } else if (ep->epnum) { musb_h_tx_flush_fifo(ep); csr = musb_readw(epio, MUSB_TXCSR); diff --git a/drivers/usb/musb/musbhsdma.h b/drivers/usb/musb/musbhsdma.h index f7b13fd252574f..a3dcbd55e43609 100644 --- a/drivers/usb/musb/musbhsdma.h +++ b/drivers/usb/musb/musbhsdma.h @@ -157,5 +157,5 @@ struct musb_dma_controller { void __iomem *base; u8 channel_count; u8 used_channels; - u8 irq; + int irq; }; diff --git a/drivers/usb/phy/Kconfig b/drivers/usb/phy/Kconfig index b9c409a18faaea..6801de9cdac9a9 100644 --- a/drivers/usb/phy/Kconfig +++ b/drivers/usb/phy/Kconfig @@ -6,6 +6,14 @@ menu "USB Physical Layer drivers" config USB_PHY def_bool n +config USB_OTG_WAKELOCK + bool "Hold a wakelock when USB connected" + depends on PM_WAKELOCKS + select USB_OTG_UTILS + help + Select this to automatically hold a wakelock when USB is + connected, preventing suspend. + # # USB Transceiver Drivers # @@ -208,4 +216,13 @@ config USB_ULPI_VIEWPORT Provides read/write operations to the ULPI phy register set for controllers with a viewport register (e.g. Chipidea/ARC controllers). +config DUAL_ROLE_USB_INTF + bool "Generic DUAL ROLE sysfs interface" + depends on SYSFS && USB_PHY + help + A generic sysfs interface to track and change the state of + dual role usb phys. The usb phy drivers can register to + this interface to expose it capabilities to the userspace + and thereby allowing userspace to change the port mode. + endmenu diff --git a/drivers/usb/phy/Makefile b/drivers/usb/phy/Makefile index b433e5d89be4bd..f65ac3e1fc07a1 100644 --- a/drivers/usb/phy/Makefile +++ b/drivers/usb/phy/Makefile @@ -3,6 +3,8 @@ # obj-$(CONFIG_USB_PHY) += phy.o obj-$(CONFIG_OF) += of.o +obj-$(CONFIG_USB_OTG_WAKELOCK) += otg-wakelock.o +obj-$(CONFIG_DUAL_ROLE_USB_INTF) += class-dual-role.o # transceiver drivers, keep the list sorted diff --git a/drivers/usb/phy/class-dual-role.c b/drivers/usb/phy/class-dual-role.c new file mode 100644 index 00000000000000..51fcb545a9d578 --- /dev/null +++ b/drivers/usb/phy/class-dual-role.c @@ -0,0 +1,529 @@ +/* + * class-dual-role.c + * + * Copyright (C) 2015 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DUAL_ROLE_NOTIFICATION_TIMEOUT 2000 + +static ssize_t dual_role_store_property(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count); +static ssize_t dual_role_show_property(struct device *dev, + struct device_attribute *attr, + char *buf); + +#define DUAL_ROLE_ATTR(_name) \ +{ \ + .attr = { .name = #_name }, \ + .show = dual_role_show_property, \ + .store = dual_role_store_property, \ +} + +static struct device_attribute dual_role_attrs[] = { + DUAL_ROLE_ATTR(supported_modes), + DUAL_ROLE_ATTR(mode), + DUAL_ROLE_ATTR(power_role), + DUAL_ROLE_ATTR(data_role), + DUAL_ROLE_ATTR(powers_vconn), +}; + +struct class *dual_role_class; +EXPORT_SYMBOL_GPL(dual_role_class); + +static struct device_type dual_role_dev_type; + +static char *kstrdupcase(const char *str, gfp_t gfp, bool to_upper) +{ + char *ret, *ustr; + + ustr = ret = kmalloc(strlen(str) + 1, gfp); + + if (!ret) + return NULL; + + while (*str) + *ustr++ = to_upper ? toupper(*str++) : tolower(*str++); + + *ustr = 0; + + return ret; +} + +static void dual_role_changed_work(struct work_struct *work) +{ + struct dual_role_phy_instance *dual_role = + container_of(work, struct dual_role_phy_instance, + changed_work); + + dev_dbg(&dual_role->dev, "%s\n", __func__); + kobject_uevent(&dual_role->dev.kobj, KOBJ_CHANGE); +} + +void dual_role_instance_changed(struct dual_role_phy_instance *dual_role) +{ + dev_dbg(&dual_role->dev, "%s\n", __func__); + pm_wakeup_event(&dual_role->dev, DUAL_ROLE_NOTIFICATION_TIMEOUT); + schedule_work(&dual_role->changed_work); +} +EXPORT_SYMBOL_GPL(dual_role_instance_changed); + +int dual_role_get_property(struct dual_role_phy_instance *dual_role, + enum dual_role_property prop, + unsigned int *val) +{ + return dual_role->desc->get_property(dual_role, prop, val); +} +EXPORT_SYMBOL_GPL(dual_role_get_property); + +int dual_role_set_property(struct dual_role_phy_instance *dual_role, + enum dual_role_property prop, + const unsigned int *val) +{ + if (!dual_role->desc->set_property) + return -ENODEV; + + return dual_role->desc->set_property(dual_role, prop, val); +} +EXPORT_SYMBOL_GPL(dual_role_set_property); + +int dual_role_property_is_writeable(struct dual_role_phy_instance *dual_role, + enum dual_role_property prop) +{ + if (!dual_role->desc->property_is_writeable) + return -ENODEV; + + return dual_role->desc->property_is_writeable(dual_role, prop); +} +EXPORT_SYMBOL_GPL(dual_role_property_is_writeable); + +static void dual_role_dev_release(struct device *dev) +{ + struct dual_role_phy_instance *dual_role = + container_of(dev, struct dual_role_phy_instance, dev); + pr_debug("device: '%s': %s\n", dev_name(dev), __func__); + kfree(dual_role); +} + +static struct dual_role_phy_instance *__must_check +__dual_role_register(struct device *parent, + const struct dual_role_phy_desc *desc) +{ + struct device *dev; + struct dual_role_phy_instance *dual_role; + int rc; + + dual_role = kzalloc(sizeof(*dual_role), GFP_KERNEL); + if (!dual_role) + return ERR_PTR(-ENOMEM); + + dev = &dual_role->dev; + + device_initialize(dev); + + dev->class = dual_role_class; + dev->type = &dual_role_dev_type; + dev->parent = parent; + dev->release = dual_role_dev_release; + dev_set_drvdata(dev, dual_role); + dual_role->desc = desc; + + rc = dev_set_name(dev, "%s", desc->name); + if (rc) + goto dev_set_name_failed; + + INIT_WORK(&dual_role->changed_work, dual_role_changed_work); + + rc = device_init_wakeup(dev, true); + if (rc) + goto wakeup_init_failed; + + rc = device_add(dev); + if (rc) + goto device_add_failed; + + dual_role_instance_changed(dual_role); + + return dual_role; + +device_add_failed: + device_init_wakeup(dev, false); +wakeup_init_failed: +dev_set_name_failed: + put_device(dev); + kfree(dual_role); + + return ERR_PTR(rc); +} + +static void dual_role_instance_unregister(struct dual_role_phy_instance + *dual_role) +{ + cancel_work_sync(&dual_role->changed_work); + device_init_wakeup(&dual_role->dev, false); + device_unregister(&dual_role->dev); +} + +static void devm_dual_role_release(struct device *dev, void *res) +{ + struct dual_role_phy_instance **dual_role = res; + + dual_role_instance_unregister(*dual_role); +} + +struct dual_role_phy_instance *__must_check +devm_dual_role_instance_register(struct device *parent, + const struct dual_role_phy_desc *desc) +{ + struct dual_role_phy_instance **ptr, *dual_role; + + ptr = devres_alloc(devm_dual_role_release, sizeof(*ptr), GFP_KERNEL); + + if (!ptr) + return ERR_PTR(-ENOMEM); + dual_role = __dual_role_register(parent, desc); + if (IS_ERR(dual_role)) { + devres_free(ptr); + } else { + *ptr = dual_role; + devres_add(parent, ptr); + } + return dual_role; +} +EXPORT_SYMBOL_GPL(devm_dual_role_instance_register); + +static int devm_dual_role_match(struct device *dev, void *res, void *data) +{ + struct dual_role_phy_instance **r = res; + + if (WARN_ON(!r || !*r)) + return 0; + + return *r == data; +} + +void devm_dual_role_instance_unregister(struct device *dev, + struct dual_role_phy_instance + *dual_role) +{ + int rc; + + rc = devres_release(dev, devm_dual_role_release, + devm_dual_role_match, dual_role); + WARN_ON(rc); +} +EXPORT_SYMBOL_GPL(devm_dual_role_instance_unregister); + +void *dual_role_get_drvdata(struct dual_role_phy_instance *dual_role) +{ + return dual_role->drv_data; +} +EXPORT_SYMBOL_GPL(dual_role_get_drvdata); + +/***************** Device attribute functions **************************/ + +/* port type */ +static char *supported_modes_text[] = { + "ufp dfp", "dfp", "ufp" +}; + +/* current mode */ +static char *mode_text[] = { + "ufp", "dfp", "none" +}; + +/* Power role */ +static char *pr_text[] = { + "source", "sink", "none" +}; + +/* Data role */ +static char *dr_text[] = { + "host", "device", "none" +}; + +/* Vconn supply */ +static char *vconn_supply_text[] = { + "n", "y" +}; + +static ssize_t dual_role_show_property(struct device *dev, + struct device_attribute *attr, char *buf) +{ + ssize_t ret = 0; + struct dual_role_phy_instance *dual_role = dev_get_drvdata(dev); + const ptrdiff_t off = attr - dual_role_attrs; + unsigned int value; + + if (off == DUAL_ROLE_PROP_SUPPORTED_MODES) { + value = dual_role->desc->supported_modes; + } else { + ret = dual_role_get_property(dual_role, off, &value); + + if (ret < 0) { + if (ret == -ENODATA) + dev_dbg(dev, + "driver has no data for `%s' property\n", + attr->attr.name); + else if (ret != -ENODEV) + dev_err(dev, + "driver failed to report `%s' property: %zd\n", + attr->attr.name, ret); + return ret; + } + } + + if (off == DUAL_ROLE_PROP_SUPPORTED_MODES) { + BUILD_BUG_ON(DUAL_ROLE_PROP_SUPPORTED_MODES_TOTAL != + ARRAY_SIZE(supported_modes_text)); + if (value < DUAL_ROLE_PROP_SUPPORTED_MODES_TOTAL) + return snprintf(buf, PAGE_SIZE, "%s\n", + supported_modes_text[value]); + else + return -EIO; + } else if (off == DUAL_ROLE_PROP_MODE) { + BUILD_BUG_ON(DUAL_ROLE_PROP_MODE_TOTAL != + ARRAY_SIZE(mode_text)); + if (value < DUAL_ROLE_PROP_MODE_TOTAL) + return snprintf(buf, PAGE_SIZE, "%s\n", + mode_text[value]); + else + return -EIO; + } else if (off == DUAL_ROLE_PROP_PR) { + BUILD_BUG_ON(DUAL_ROLE_PROP_PR_TOTAL != ARRAY_SIZE(pr_text)); + if (value < DUAL_ROLE_PROP_PR_TOTAL) + return snprintf(buf, PAGE_SIZE, "%s\n", + pr_text[value]); + else + return -EIO; + } else if (off == DUAL_ROLE_PROP_DR) { + BUILD_BUG_ON(DUAL_ROLE_PROP_DR_TOTAL != ARRAY_SIZE(dr_text)); + if (value < DUAL_ROLE_PROP_DR_TOTAL) + return snprintf(buf, PAGE_SIZE, "%s\n", + dr_text[value]); + else + return -EIO; + } else if (off == DUAL_ROLE_PROP_VCONN_SUPPLY) { + BUILD_BUG_ON(DUAL_ROLE_PROP_VCONN_SUPPLY_TOTAL != + ARRAY_SIZE(vconn_supply_text)); + if (value < DUAL_ROLE_PROP_VCONN_SUPPLY_TOTAL) + return snprintf(buf, PAGE_SIZE, "%s\n", + vconn_supply_text[value]); + else + return -EIO; + } else + return -EIO; +} + +static ssize_t dual_role_store_property(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + ssize_t ret; + struct dual_role_phy_instance *dual_role = dev_get_drvdata(dev); + const ptrdiff_t off = attr - dual_role_attrs; + unsigned int value; + int total, i; + char *dup_buf, **text_array; + bool result = false; + + dup_buf = kstrdupcase(buf, GFP_KERNEL, false); + switch (off) { + case DUAL_ROLE_PROP_MODE: + total = DUAL_ROLE_PROP_MODE_TOTAL; + text_array = mode_text; + break; + case DUAL_ROLE_PROP_PR: + total = DUAL_ROLE_PROP_PR_TOTAL; + text_array = pr_text; + break; + case DUAL_ROLE_PROP_DR: + total = DUAL_ROLE_PROP_DR_TOTAL; + text_array = dr_text; + break; + case DUAL_ROLE_PROP_VCONN_SUPPLY: + ret = strtobool(dup_buf, &result); + value = result; + if (!ret) + goto setprop; + default: + ret = -EINVAL; + goto error; + } + + for (i = 0; i <= total; i++) { + if (i == total) { + ret = -ENOTSUPP; + goto error; + } + if (!strncmp(*(text_array + i), dup_buf, + strlen(*(text_array + i)))) { + value = i; + break; + } + } + +setprop: + ret = dual_role->desc->set_property(dual_role, off, &value); + +error: + kfree(dup_buf); + + if (ret < 0) + return ret; + + return count; +} + +static umode_t dual_role_attr_is_visible(struct kobject *kobj, + struct attribute *attr, int attrno) +{ + struct device *dev = container_of(kobj, struct device, kobj); + struct dual_role_phy_instance *dual_role = dev_get_drvdata(dev); + umode_t mode = S_IRUSR | S_IRGRP | S_IROTH; + int i; + + if (attrno == DUAL_ROLE_PROP_SUPPORTED_MODES) + return mode; + + for (i = 0; i < dual_role->desc->num_properties; i++) { + int property = dual_role->desc->properties[i]; + + if (property == attrno) { + if (dual_role->desc->property_is_writeable && + dual_role_property_is_writeable(dual_role, property) + > 0) + mode |= S_IWUSR; + + return mode; + } + } + + return 0; +} + +static struct attribute *__dual_role_attrs[ARRAY_SIZE(dual_role_attrs) + 1]; + +static struct attribute_group dual_role_attr_group = { + .attrs = __dual_role_attrs, + .is_visible = dual_role_attr_is_visible, +}; + +static const struct attribute_group *dual_role_attr_groups[] = { + &dual_role_attr_group, + NULL, +}; + +void dual_role_init_attrs(struct device_type *dev_type) +{ + int i; + + dev_type->groups = dual_role_attr_groups; + + for (i = 0; i < ARRAY_SIZE(dual_role_attrs); i++) + __dual_role_attrs[i] = &dual_role_attrs[i].attr; +} + +int dual_role_uevent(struct device *dev, struct kobj_uevent_env *env) +{ + struct dual_role_phy_instance *dual_role = dev_get_drvdata(dev); + int ret = 0, j; + char *prop_buf; + char *attrname; + + dev_dbg(dev, "uevent\n"); + + if (!dual_role || !dual_role->desc) { + dev_dbg(dev, "No dual_role phy yet\n"); + return ret; + } + + dev_dbg(dev, "DUAL_ROLE_NAME=%s\n", dual_role->desc->name); + + ret = add_uevent_var(env, "DUAL_ROLE_NAME=%s", dual_role->desc->name); + if (ret) + return ret; + + prop_buf = (char *)get_zeroed_page(GFP_KERNEL); + if (!prop_buf) + return -ENOMEM; + + for (j = 0; j < dual_role->desc->num_properties; j++) { + struct device_attribute *attr; + char *line; + + attr = &dual_role_attrs[dual_role->desc->properties[j]]; + + ret = dual_role_show_property(dev, attr, prop_buf); + if (ret == -ENODEV || ret == -ENODATA) { + ret = 0; + continue; + } + + if (ret < 0) + goto out; + line = strnchr(prop_buf, PAGE_SIZE, '\n'); + if (line) + *line = 0; + + attrname = kstrdupcase(attr->attr.name, GFP_KERNEL, true); + if (!attrname) + ret = -ENOMEM; + + dev_dbg(dev, "prop %s=%s\n", attrname, prop_buf); + + ret = add_uevent_var(env, "DUAL_ROLE_%s=%s", attrname, + prop_buf); + kfree(attrname); + if (ret) + goto out; + } + +out: + free_page((unsigned long)prop_buf); + + return ret; +} + +/******************* Module Init ***********************************/ + +static int __init dual_role_class_init(void) +{ + dual_role_class = class_create(THIS_MODULE, "dual_role_usb"); + + if (IS_ERR(dual_role_class)) + return PTR_ERR(dual_role_class); + + dual_role_class->dev_uevent = dual_role_uevent; + dual_role_init_attrs(&dual_role_dev_type); + + return 0; +} + +static void __exit dual_role_class_exit(void) +{ + class_destroy(dual_role_class); +} + +subsys_initcall(dual_role_class_init); +module_exit(dual_role_class_exit); diff --git a/drivers/usb/phy/otg-wakelock.c b/drivers/usb/phy/otg-wakelock.c new file mode 100644 index 00000000000000..ecd741027f53d9 --- /dev/null +++ b/drivers/usb/phy/otg-wakelock.c @@ -0,0 +1,170 @@ +/* + * otg-wakelock.c + * + * Copyright (C) 2011 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include +#include +#include +#include +#include +#include + +#define TEMPORARY_HOLD_TIME 2000 + +static bool enabled = true; +static struct usb_phy *otgwl_xceiv; +static struct notifier_block otgwl_nb; + +/* + * otgwl_spinlock is held while the VBUS lock is grabbed or dropped and the + * held field is updated to match. + */ + +static DEFINE_SPINLOCK(otgwl_spinlock); + +/* + * Only one lock, but since these 3 fields are associated with each other... + */ + +struct otgwl_lock { + char name[40]; + struct wakeup_source wakesrc; + bool held; +}; + +/* + * VBUS present lock. Also used as a timed lock on charger + * connect/disconnect and USB host disconnect, to allow the system + * to react to the change in power. + */ + +static struct otgwl_lock vbus_lock; + +static void otgwl_hold(struct otgwl_lock *lock) +{ + if (!lock->held) { + __pm_stay_awake(&lock->wakesrc); + lock->held = true; + } +} + +static void otgwl_temporary_hold(struct otgwl_lock *lock) +{ + __pm_wakeup_event(&lock->wakesrc, TEMPORARY_HOLD_TIME); + lock->held = false; +} + +static void otgwl_drop(struct otgwl_lock *lock) +{ + if (lock->held) { + __pm_relax(&lock->wakesrc); + lock->held = false; + } +} + +static void otgwl_handle_event(unsigned long event) +{ + unsigned long irqflags; + + spin_lock_irqsave(&otgwl_spinlock, irqflags); + + if (!enabled) { + otgwl_drop(&vbus_lock); + spin_unlock_irqrestore(&otgwl_spinlock, irqflags); + return; + } + + switch (event) { + case USB_EVENT_VBUS: + case USB_EVENT_ENUMERATED: + otgwl_hold(&vbus_lock); + break; + + case USB_EVENT_NONE: + case USB_EVENT_ID: + case USB_EVENT_CHARGER: + otgwl_temporary_hold(&vbus_lock); + break; + + default: + break; + } + + spin_unlock_irqrestore(&otgwl_spinlock, irqflags); +} + +static int otgwl_otg_notifications(struct notifier_block *nb, + unsigned long event, void *unused) +{ + otgwl_handle_event(event); + return NOTIFY_OK; +} + +static int set_enabled(const char *val, const struct kernel_param *kp) +{ + int rv = param_set_bool(val, kp); + + if (rv) + return rv; + + if (otgwl_xceiv) + otgwl_handle_event(otgwl_xceiv->last_event); + + return 0; +} + +static struct kernel_param_ops enabled_param_ops = { + .set = set_enabled, + .get = param_get_bool, +}; + +module_param_cb(enabled, &enabled_param_ops, &enabled, 0644); +MODULE_PARM_DESC(enabled, "enable wakelock when VBUS present"); + +static int __init otg_wakelock_init(void) +{ + int ret; + struct usb_phy *phy; + + phy = usb_get_phy(USB_PHY_TYPE_USB2); + + if (IS_ERR(phy)) { + pr_err("%s: No USB transceiver found\n", __func__); + return PTR_ERR(phy); + } + otgwl_xceiv = phy; + + snprintf(vbus_lock.name, sizeof(vbus_lock.name), "vbus-%s", + dev_name(otgwl_xceiv->dev)); + wakeup_source_init(&vbus_lock.wakesrc, vbus_lock.name); + + otgwl_nb.notifier_call = otgwl_otg_notifications; + ret = usb_register_notifier(otgwl_xceiv, &otgwl_nb); + + if (ret) { + pr_err("%s: usb_register_notifier on transceiver %s" + " failed\n", __func__, + dev_name(otgwl_xceiv->dev)); + otgwl_xceiv = NULL; + wakeup_source_trash(&vbus_lock.wakesrc); + return ret; + } + + otgwl_handle_event(otgwl_xceiv->last_event); + return ret; +} + +late_initcall(otg_wakelock_init); diff --git a/drivers/usb/phy/phy-am335x-control.c b/drivers/usb/phy/phy-am335x-control.c index 42a1afe36a9052..5f5f19813fde15 100644 --- a/drivers/usb/phy/phy-am335x-control.c +++ b/drivers/usb/phy/phy-am335x-control.c @@ -134,10 +134,12 @@ struct phy_control *am335x_get_phy_control(struct device *dev) return NULL; dev = bus_find_device(&platform_bus_type, NULL, node, match); + of_node_put(node); if (!dev) return NULL; ctrl_usb = dev_get_drvdata(dev); + put_device(dev); if (!ctrl_usb) return NULL; return &ctrl_usb->phy_ctrl; diff --git a/drivers/usb/serial/ch341.c b/drivers/usb/serial/ch341.c index f139488d0816c0..e98590aab633cd 100644 --- a/drivers/usb/serial/ch341.c +++ b/drivers/usb/serial/ch341.c @@ -99,6 +99,8 @@ static int ch341_control_out(struct usb_device *dev, u8 request, r = usb_control_msg(dev, usb_sndctrlpipe(dev, 0), request, USB_TYPE_VENDOR | USB_RECIP_DEVICE | USB_DIR_OUT, value, index, NULL, 0, DEFAULT_TIMEOUT); + if (r < 0) + dev_err(&dev->dev, "failed to send control message: %d\n", r); return r; } @@ -116,7 +118,20 @@ static int ch341_control_in(struct usb_device *dev, r = usb_control_msg(dev, usb_rcvctrlpipe(dev, 0), request, USB_TYPE_VENDOR | USB_RECIP_DEVICE | USB_DIR_IN, value, index, buf, bufsize, DEFAULT_TIMEOUT); - return r; + if (r < bufsize) { + if (r >= 0) { + dev_err(&dev->dev, + "short control message received (%d < %u)\n", + r, bufsize); + r = -EIO; + } + + dev_err(&dev->dev, "failed to receive control message: %d\n", + r); + return r; + } + + return 0; } static int ch341_set_baudrate(struct usb_device *dev, @@ -158,9 +173,9 @@ static int ch341_set_handshake(struct usb_device *dev, u8 control) static int ch341_get_status(struct usb_device *dev, struct ch341_private *priv) { + const unsigned int size = 2; char *buffer; int r; - const unsigned size = 8; unsigned long flags; buffer = kmalloc(size, GFP_KERNEL); @@ -171,14 +186,9 @@ static int ch341_get_status(struct usb_device *dev, struct ch341_private *priv) if (r < 0) goto out; - /* setup the private status if available */ - if (r == 2) { - r = 0; - spin_lock_irqsave(&priv->lock, flags); - priv->line_status = (~(*buffer)) & CH341_BITS_MODEM_STAT; - spin_unlock_irqrestore(&priv->lock, flags); - } else - r = -EPROTO; + spin_lock_irqsave(&priv->lock, flags); + priv->line_status = (~(*buffer)) & CH341_BITS_MODEM_STAT; + spin_unlock_irqrestore(&priv->lock, flags); out: kfree(buffer); return r; @@ -188,9 +198,9 @@ out: kfree(buffer); static int ch341_configure(struct usb_device *dev, struct ch341_private *priv) { + const unsigned int size = 2; char *buffer; int r; - const unsigned size = 8; buffer = kmalloc(size, GFP_KERNEL); if (!buffer) @@ -253,7 +263,6 @@ static int ch341_port_probe(struct usb_serial_port *port) spin_lock_init(&priv->lock); priv->baud_rate = DEFAULT_BAUD_RATE; - priv->line_control = CH341_BIT_RTS | CH341_BIT_DTR; r = ch341_configure(port->serial->dev, priv); if (r < 0) @@ -315,7 +324,7 @@ static int ch341_open(struct tty_struct *tty, struct usb_serial_port *port) r = ch341_configure(serial->dev, priv); if (r) - goto out; + return r; if (tty) ch341_set_termios(tty, port, NULL); @@ -325,12 +334,19 @@ static int ch341_open(struct tty_struct *tty, struct usb_serial_port *port) if (r) { dev_err(&port->dev, "%s - failed to submit interrupt urb: %d\n", __func__, r); - goto out; + return r; } r = usb_serial_generic_open(tty, port); + if (r) + goto err_kill_interrupt_urb; -out: return r; + return 0; + +err_kill_interrupt_urb: + usb_kill_urb(port->interrupt_in_urb); + + return r; } /* Old_termios contains the original termios settings and @@ -345,26 +361,25 @@ static void ch341_set_termios(struct tty_struct *tty, baud_rate = tty_get_baud_rate(tty); - priv->baud_rate = baud_rate; - if (baud_rate) { - spin_lock_irqsave(&priv->lock, flags); - priv->line_control |= (CH341_BIT_DTR | CH341_BIT_RTS); - spin_unlock_irqrestore(&priv->lock, flags); + priv->baud_rate = baud_rate; ch341_set_baudrate(port->serial->dev, priv); - } else { - spin_lock_irqsave(&priv->lock, flags); - priv->line_control &= ~(CH341_BIT_DTR | CH341_BIT_RTS); - spin_unlock_irqrestore(&priv->lock, flags); } - ch341_set_handshake(port->serial->dev, priv->line_control); - /* Unimplemented: * (cflag & CSIZE) : data bits [5, 8] * (cflag & PARENB) : parity {NONE, EVEN, ODD} * (cflag & CSTOPB) : stop bits [1, 2] */ + + spin_lock_irqsave(&priv->lock, flags); + if (C_BAUD(tty) == B0) + priv->line_control &= ~(CH341_BIT_DTR | CH341_BIT_RTS); + else if (old_termios && (old_termios->c_cflag & CBAUD) == B0) + priv->line_control |= (CH341_BIT_DTR | CH341_BIT_RTS); + spin_unlock_irqrestore(&priv->lock, flags); + + ch341_set_handshake(port->serial->dev, priv->line_control); } static void ch341_break_ctl(struct tty_struct *tty, int break_state) @@ -539,14 +554,23 @@ static int ch341_tiocmget(struct tty_struct *tty) static int ch341_reset_resume(struct usb_serial *serial) { - struct ch341_private *priv; - - priv = usb_get_serial_port_data(serial->port[0]); + struct usb_serial_port *port = serial->port[0]; + struct ch341_private *priv = usb_get_serial_port_data(port); + int ret; /* reconfigure ch341 serial port after bus-reset */ ch341_configure(serial->dev, priv); - return 0; + if (tty_port_initialized(&port->port)) { + ret = usb_submit_urb(port->interrupt_in_urb, GFP_NOIO); + if (ret) { + dev_err(&port->dev, "failed to submit interrupt urb: %d\n", + ret); + return ret; + } + } + + return usb_serial_generic_resume(serial); } static struct usb_serial_driver ch341_device = { diff --git a/drivers/usb/serial/cyberjack.c b/drivers/usb/serial/cyberjack.c index 5f17a3b9916d79..80260b08398b2e 100644 --- a/drivers/usb/serial/cyberjack.c +++ b/drivers/usb/serial/cyberjack.c @@ -50,6 +50,7 @@ #define CYBERJACK_PRODUCT_ID 0x0100 /* Function prototypes */ +static int cyberjack_attach(struct usb_serial *serial); static int cyberjack_port_probe(struct usb_serial_port *port); static int cyberjack_port_remove(struct usb_serial_port *port); static int cyberjack_open(struct tty_struct *tty, @@ -77,6 +78,7 @@ static struct usb_serial_driver cyberjack_device = { .description = "Reiner SCT Cyberjack USB card reader", .id_table = id_table, .num_ports = 1, + .attach = cyberjack_attach, .port_probe = cyberjack_port_probe, .port_remove = cyberjack_port_remove, .open = cyberjack_open, @@ -100,6 +102,14 @@ struct cyberjack_private { short wrsent; /* Data already sent */ }; +static int cyberjack_attach(struct usb_serial *serial) +{ + if (serial->num_bulk_out < serial->num_ports) + return -ENODEV; + + return 0; +} + static int cyberjack_port_probe(struct usb_serial_port *port) { struct cyberjack_private *priv; diff --git a/drivers/usb/serial/garmin_gps.c b/drivers/usb/serial/garmin_gps.c index 97cabf803c2fa9..b2f2e87aed945d 100644 --- a/drivers/usb/serial/garmin_gps.c +++ b/drivers/usb/serial/garmin_gps.c @@ -1043,6 +1043,7 @@ static int garmin_write_bulk(struct usb_serial_port *port, "%s - usb_submit_urb(write bulk) failed with status = %d\n", __func__, status); count = status; + kfree(buffer); } /* we are done with this urb, so let the host driver diff --git a/drivers/usb/serial/io_edgeport.c b/drivers/usb/serial/io_edgeport.c index 11c05ce2f35f8c..36dfe9972b173b 100644 --- a/drivers/usb/serial/io_edgeport.c +++ b/drivers/usb/serial/io_edgeport.c @@ -2754,6 +2754,11 @@ static int edge_startup(struct usb_serial *serial) EDGE_COMPATIBILITY_MASK1, EDGE_COMPATIBILITY_MASK2 }; + if (serial->num_bulk_in < 1 || serial->num_interrupt_in < 1) { + dev_err(&serial->interface->dev, "missing endpoints\n"); + return -ENODEV; + } + dev = serial->dev; /* create our private serial structure */ diff --git a/drivers/usb/serial/io_ti.c b/drivers/usb/serial/io_ti.c index fce82fd79f77bf..c02808a304364f 100644 --- a/drivers/usb/serial/io_ti.c +++ b/drivers/usb/serial/io_ti.c @@ -1499,8 +1499,7 @@ static int do_boot_mode(struct edgeport_serial *serial, dev_dbg(dev, "%s - Download successful -- Device rebooting...\n", __func__); - /* return an error on purpose */ - return -ENODEV; + return 1; } stayinbootmode: @@ -1508,7 +1507,7 @@ static int do_boot_mode(struct edgeport_serial *serial, dev_dbg(dev, "%s - STAYING IN BOOT MODE\n", __func__); serial->product_info.TiMode = TI_MODE_BOOT; - return 0; + return 1; } static int ti_do_config(struct edgeport_port *port, int feature, int on) @@ -2549,6 +2548,13 @@ static int edge_startup(struct usb_serial *serial) int status; u16 product_id; + /* Make sure we have the required endpoints when in download mode. */ + if (serial->interface->cur_altsetting->desc.bNumEndpoints > 1) { + if (serial->num_bulk_in < serial->num_ports || + serial->num_bulk_out < serial->num_ports) + return -ENODEV; + } + /* create our private serial structure */ edge_serial = kzalloc(sizeof(struct edgeport_serial), GFP_KERNEL); if (!edge_serial) @@ -2556,14 +2562,18 @@ static int edge_startup(struct usb_serial *serial) mutex_init(&edge_serial->es_lock); edge_serial->serial = serial; + INIT_DELAYED_WORK(&edge_serial->heartbeat_work, edge_heartbeat_work); usb_set_serial_data(serial, edge_serial); status = download_fw(edge_serial); - if (status) { + if (status < 0) { kfree(edge_serial); return status; } + if (status > 0) + return 1; /* bind but do not register any ports */ + product_id = le16_to_cpu( edge_serial->serial->dev->descriptor.idProduct); @@ -2575,7 +2585,6 @@ static int edge_startup(struct usb_serial *serial) } } - INIT_DELAYED_WORK(&edge_serial->heartbeat_work, edge_heartbeat_work); edge_heartbeat_schedule(edge_serial); return 0; @@ -2583,6 +2592,9 @@ static int edge_startup(struct usb_serial *serial) static void edge_disconnect(struct usb_serial *serial) { + struct edgeport_serial *edge_serial = usb_get_serial_data(serial); + + cancel_delayed_work_sync(&edge_serial->heartbeat_work); } static void edge_release(struct usb_serial *serial) diff --git a/drivers/usb/serial/iuu_phoenix.c b/drivers/usb/serial/iuu_phoenix.c index 344b4eea4bd59c..d57fb51992182a 100644 --- a/drivers/usb/serial/iuu_phoenix.c +++ b/drivers/usb/serial/iuu_phoenix.c @@ -68,6 +68,16 @@ struct iuu_private { u32 clk; }; +static int iuu_attach(struct usb_serial *serial) +{ + unsigned char num_ports = serial->num_ports; + + if (serial->num_bulk_in < num_ports || serial->num_bulk_out < num_ports) + return -ENODEV; + + return 0; +} + static int iuu_port_probe(struct usb_serial_port *port) { struct iuu_private *priv; @@ -1196,6 +1206,7 @@ static struct usb_serial_driver iuu_device = { .tiocmset = iuu_tiocmset, .set_termios = iuu_set_termios, .init_termios = iuu_init_termios, + .attach = iuu_attach, .port_probe = iuu_port_probe, .port_remove = iuu_port_remove, }; diff --git a/drivers/usb/serial/keyspan_pda.c b/drivers/usb/serial/keyspan_pda.c index e49ad0c63ad8b5..83523fcf6fb9d6 100644 --- a/drivers/usb/serial/keyspan_pda.c +++ b/drivers/usb/serial/keyspan_pda.c @@ -699,6 +699,19 @@ MODULE_FIRMWARE("keyspan_pda/keyspan_pda.fw"); MODULE_FIRMWARE("keyspan_pda/xircom_pgs.fw"); #endif +static int keyspan_pda_attach(struct usb_serial *serial) +{ + unsigned char num_ports = serial->num_ports; + + if (serial->num_bulk_out < num_ports || + serial->num_interrupt_in < num_ports) { + dev_err(&serial->interface->dev, "missing endpoints\n"); + return -ENODEV; + } + + return 0; +} + static int keyspan_pda_port_probe(struct usb_serial_port *port) { @@ -776,6 +789,7 @@ static struct usb_serial_driver keyspan_pda_device = { .break_ctl = keyspan_pda_break_ctl, .tiocmget = keyspan_pda_tiocmget, .tiocmset = keyspan_pda_tiocmset, + .attach = keyspan_pda_attach, .port_probe = keyspan_pda_port_probe, .port_remove = keyspan_pda_port_remove, }; diff --git a/drivers/usb/serial/kl5kusb105.c b/drivers/usb/serial/kl5kusb105.c index fc5d3a791e08c6..6cb45757818fae 100644 --- a/drivers/usb/serial/kl5kusb105.c +++ b/drivers/usb/serial/kl5kusb105.c @@ -192,10 +192,11 @@ static int klsi_105_get_line_state(struct usb_serial_port *port, status_buf, KLSI_STATUSBUF_LEN, 10000 ); - if (rc < 0) - dev_err(&port->dev, "Reading line status failed (error = %d)\n", - rc); - else { + if (rc != KLSI_STATUSBUF_LEN) { + dev_err(&port->dev, "reading line status failed: %d\n", rc); + if (rc >= 0) + rc = -EIO; + } else { status = get_unaligned_le16(status_buf); dev_info(&port->serial->dev->dev, "read status %x %x\n", @@ -296,7 +297,7 @@ static int klsi_105_open(struct tty_struct *tty, struct usb_serial_port *port) rc = usb_serial_generic_open(tty, port); if (rc) { retval = rc; - goto exit; + goto err_free_cfg; } rc = usb_control_msg(port->serial->dev, @@ -311,21 +312,38 @@ static int klsi_105_open(struct tty_struct *tty, struct usb_serial_port *port) if (rc < 0) { dev_err(&port->dev, "Enabling read failed (error = %d)\n", rc); retval = rc; + goto err_generic_close; } else dev_dbg(&port->dev, "%s - enabled reading\n", __func__); rc = klsi_105_get_line_state(port, &line_state); - if (rc >= 0) { - spin_lock_irqsave(&priv->lock, flags); - priv->line_state = line_state; - spin_unlock_irqrestore(&priv->lock, flags); - dev_dbg(&port->dev, "%s - read line state 0x%lx\n", __func__, line_state); - retval = 0; - } else + if (rc < 0) { retval = rc; + goto err_disable_read; + } + + spin_lock_irqsave(&priv->lock, flags); + priv->line_state = line_state; + spin_unlock_irqrestore(&priv->lock, flags); + dev_dbg(&port->dev, "%s - read line state 0x%lx\n", __func__, + line_state); + + return 0; -exit: +err_disable_read: + usb_control_msg(port->serial->dev, + usb_sndctrlpipe(port->serial->dev, 0), + KL5KUSB105A_SIO_CONFIGURE, + USB_TYPE_VENDOR | USB_DIR_OUT, + KL5KUSB105A_SIO_CONFIGURE_READ_OFF, + 0, /* index */ + NULL, 0, + KLSI_TIMEOUT); +err_generic_close: + usb_serial_generic_close(port); +err_free_cfg: kfree(cfg); + return retval; } diff --git a/drivers/usb/serial/kobil_sct.c b/drivers/usb/serial/kobil_sct.c index 2363654cafc9b7..813035f51fe73a 100644 --- a/drivers/usb/serial/kobil_sct.c +++ b/drivers/usb/serial/kobil_sct.c @@ -51,6 +51,7 @@ /* Function prototypes */ +static int kobil_attach(struct usb_serial *serial); static int kobil_port_probe(struct usb_serial_port *probe); static int kobil_port_remove(struct usb_serial_port *probe); static int kobil_open(struct tty_struct *tty, struct usb_serial_port *port); @@ -86,6 +87,7 @@ static struct usb_serial_driver kobil_device = { .description = "KOBIL USB smart card terminal", .id_table = id_table, .num_ports = 1, + .attach = kobil_attach, .port_probe = kobil_port_probe, .port_remove = kobil_port_remove, .ioctl = kobil_ioctl, @@ -113,6 +115,16 @@ struct kobil_private { }; +static int kobil_attach(struct usb_serial *serial) +{ + if (serial->num_interrupt_out < serial->num_ports) { + dev_err(&serial->interface->dev, "missing interrupt-out endpoint\n"); + return -ENODEV; + } + + return 0; +} + static int kobil_port_probe(struct usb_serial_port *port) { struct usb_serial *serial = port->serial; diff --git a/drivers/usb/serial/mos7720.c b/drivers/usb/serial/mos7720.c index de9992b492b08b..136ff5e1b7c1f7 100644 --- a/drivers/usb/serial/mos7720.c +++ b/drivers/usb/serial/mos7720.c @@ -65,8 +65,6 @@ struct moschip_port { struct urb *write_urb_pool[NUM_URBS]; }; -static struct usb_serial_driver moschip7720_2port_driver; - #define USB_VENDOR_ID_MOSCHIP 0x9710 #define MOSCHIP_DEVICE_ID_7720 0x7720 #define MOSCHIP_DEVICE_ID_7715 0x7715 @@ -970,25 +968,6 @@ static void mos7720_bulk_out_data_callback(struct urb *urb) tty_port_tty_wakeup(&mos7720_port->port->port); } -/* - * mos77xx_probe - * this function installs the appropriate read interrupt endpoint callback - * depending on whether the device is a 7720 or 7715, thus avoiding costly - * run-time checks in the high-frequency callback routine itself. - */ -static int mos77xx_probe(struct usb_serial *serial, - const struct usb_device_id *id) -{ - if (id->idProduct == MOSCHIP_DEVICE_ID_7715) - moschip7720_2port_driver.read_int_callback = - mos7715_interrupt_callback; - else - moschip7720_2port_driver.read_int_callback = - mos7720_interrupt_callback; - - return 0; -} - static int mos77xx_calc_num_ports(struct usb_serial *serial) { u16 product = le16_to_cpu(serial->dev->descriptor.idProduct); @@ -1920,6 +1899,11 @@ static int mos7720_startup(struct usb_serial *serial) u16 product; int ret_val; + if (serial->num_bulk_in < 2 || serial->num_bulk_out < 2) { + dev_err(&serial->interface->dev, "missing bulk endpoints\n"); + return -ENODEV; + } + product = le16_to_cpu(serial->dev->descriptor.idProduct); dev = serial->dev; @@ -1944,19 +1928,18 @@ static int mos7720_startup(struct usb_serial *serial) tmp->interrupt_in_endpointAddress; serial->port[1]->interrupt_in_urb = NULL; serial->port[1]->interrupt_in_buffer = NULL; + + if (serial->port[0]->interrupt_in_urb) { + struct urb *urb = serial->port[0]->interrupt_in_urb; + + urb->complete = mos7715_interrupt_callback; + } } /* setting configuration feature to one */ usb_control_msg(serial->dev, usb_sndctrlpipe(serial->dev, 0), (__u8)0x03, 0x00, 0x01, 0x00, NULL, 0x00, 5000); - /* start the interrupt urb */ - ret_val = usb_submit_urb(serial->port[0]->interrupt_in_urb, GFP_KERNEL); - if (ret_val) - dev_err(&dev->dev, - "%s - Error %d submitting control urb\n", - __func__, ret_val); - #ifdef CONFIG_USB_SERIAL_MOS7715_PARPORT if (product == MOSCHIP_DEVICE_ID_7715) { ret_val = mos7715_parport_init(serial); @@ -1964,6 +1947,13 @@ static int mos7720_startup(struct usb_serial *serial) return ret_val; } #endif + /* start the interrupt urb */ + ret_val = usb_submit_urb(serial->port[0]->interrupt_in_urb, GFP_KERNEL); + if (ret_val) { + dev_err(&dev->dev, "failed to submit interrupt urb: %d\n", + ret_val); + } + /* LSR For Port 1 */ read_mos_reg(serial, 0, MOS7720_LSR, &data); dev_dbg(&dev->dev, "LSR:%x\n", data); @@ -1973,6 +1963,8 @@ static int mos7720_startup(struct usb_serial *serial) static void mos7720_release(struct usb_serial *serial) { + usb_kill_urb(serial->port[0]->interrupt_in_urb); + #ifdef CONFIG_USB_SERIAL_MOS7715_PARPORT /* close the parallel port */ @@ -2056,7 +2048,6 @@ static struct usb_serial_driver moschip7720_2port_driver = { .close = mos7720_close, .throttle = mos7720_throttle, .unthrottle = mos7720_unthrottle, - .probe = mos77xx_probe, .attach = mos7720_startup, .release = mos7720_release, .port_probe = mos7720_port_probe, @@ -2070,7 +2061,7 @@ static struct usb_serial_driver moschip7720_2port_driver = { .chars_in_buffer = mos7720_chars_in_buffer, .break_ctl = mos7720_break, .read_bulk_callback = mos7720_bulk_in_callback, - .read_int_callback = NULL /* dynamically assigned in probe() */ + .read_int_callback = mos7720_interrupt_callback, }; static struct usb_serial_driver * const serial_drivers[] = { diff --git a/drivers/usb/serial/mos7840.c b/drivers/usb/serial/mos7840.c index 57426d703a098d..4f9af47e6a29ad 100644 --- a/drivers/usb/serial/mos7840.c +++ b/drivers/usb/serial/mos7840.c @@ -2116,6 +2116,17 @@ static int mos7840_calc_num_ports(struct usb_serial *serial) return mos7840_num_ports; } +static int mos7840_attach(struct usb_serial *serial) +{ + if (serial->num_bulk_in < serial->num_ports || + serial->num_bulk_out < serial->num_ports) { + dev_err(&serial->interface->dev, "missing endpoints\n"); + return -ENODEV; + } + + return 0; +} + static int mos7840_port_probe(struct usb_serial_port *port) { struct usb_serial *serial = port->serial; @@ -2391,6 +2402,7 @@ static struct usb_serial_driver moschip7840_4port_device = { .tiocmset = mos7840_tiocmset, .tiocmiwait = usb_serial_generic_tiocmiwait, .get_icount = usb_serial_generic_get_icount, + .attach = mos7840_attach, .port_probe = mos7840_port_probe, .port_remove = mos7840_port_remove, .read_bulk_callback = mos7840_bulk_in_callback, diff --git a/drivers/usb/serial/omninet.c b/drivers/usb/serial/omninet.c index f6c6900bccf01c..a180b17d24323b 100644 --- a/drivers/usb/serial/omninet.c +++ b/drivers/usb/serial/omninet.c @@ -38,6 +38,7 @@ static int omninet_write(struct tty_struct *tty, struct usb_serial_port *port, const unsigned char *buf, int count); static int omninet_write_room(struct tty_struct *tty); static void omninet_disconnect(struct usb_serial *serial); +static int omninet_attach(struct usb_serial *serial); static int omninet_port_probe(struct usb_serial_port *port); static int omninet_port_remove(struct usb_serial_port *port); @@ -56,6 +57,7 @@ static struct usb_serial_driver zyxel_omninet_device = { .description = "ZyXEL - omni.net lcd plus usb", .id_table = id_table, .num_ports = 1, + .attach = omninet_attach, .port_probe = omninet_port_probe, .port_remove = omninet_port_remove, .open = omninet_open, @@ -104,6 +106,17 @@ struct omninet_data { __u8 od_outseq; /* Sequence number for bulk_out URBs */ }; +static int omninet_attach(struct usb_serial *serial) +{ + /* The second bulk-out endpoint is used for writing. */ + if (serial->num_bulk_out < 2) { + dev_err(&serial->interface->dev, "missing endpoints\n"); + return -ENODEV; + } + + return 0; +} + static int omninet_port_probe(struct usb_serial_port *port) { struct omninet_data *od; diff --git a/drivers/usb/serial/option.c b/drivers/usb/serial/option.c index 9894e341c6ac93..42cc72e54c051b 100644 --- a/drivers/usb/serial/option.c +++ b/drivers/usb/serial/option.c @@ -268,6 +268,8 @@ static void option_instat_callback(struct urb *urb); #define TELIT_PRODUCT_CC864_SINGLE 0x1006 #define TELIT_PRODUCT_DE910_DUAL 0x1010 #define TELIT_PRODUCT_UE910_V2 0x1012 +#define TELIT_PRODUCT_LE922_USBCFG1 0x1040 +#define TELIT_PRODUCT_LE922_USBCFG2 0x1041 #define TELIT_PRODUCT_LE922_USBCFG0 0x1042 #define TELIT_PRODUCT_LE922_USBCFG3 0x1043 #define TELIT_PRODUCT_LE922_USBCFG5 0x1045 @@ -1210,6 +1212,10 @@ static const struct usb_device_id option_ids[] = { { USB_DEVICE(TELIT_VENDOR_ID, TELIT_PRODUCT_UE910_V2) }, { USB_DEVICE(TELIT_VENDOR_ID, TELIT_PRODUCT_LE922_USBCFG0), .driver_info = (kernel_ulong_t)&telit_le922_blacklist_usbcfg0 }, + { USB_DEVICE(TELIT_VENDOR_ID, TELIT_PRODUCT_LE922_USBCFG1), + .driver_info = (kernel_ulong_t)&telit_le910_blacklist }, + { USB_DEVICE(TELIT_VENDOR_ID, TELIT_PRODUCT_LE922_USBCFG2), + .driver_info = (kernel_ulong_t)&telit_le922_blacklist_usbcfg3 }, { USB_DEVICE(TELIT_VENDOR_ID, TELIT_PRODUCT_LE922_USBCFG3), .driver_info = (kernel_ulong_t)&telit_le922_blacklist_usbcfg3 }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, TELIT_PRODUCT_LE922_USBCFG5, 0xff), @@ -1989,6 +1995,7 @@ static const struct usb_device_id option_ids[] = { { USB_DEVICE_AND_INTERFACE_INFO(0x2001, 0x7d02, 0xff, 0x00, 0x00) }, { USB_DEVICE_AND_INTERFACE_INFO(0x2001, 0x7d03, 0xff, 0x02, 0x01) }, { USB_DEVICE_AND_INTERFACE_INFO(0x2001, 0x7d03, 0xff, 0x00, 0x00) }, + { USB_DEVICE_INTERFACE_CLASS(0x2001, 0x7d04, 0xff) }, /* D-Link DWM-158 */ { USB_DEVICE_INTERFACE_CLASS(0x2001, 0x7e19, 0xff), /* D-Link DWM-221 B1 */ .driver_info = (kernel_ulong_t)&net_intf4_blacklist }, { USB_DEVICE_AND_INTERFACE_INFO(0x07d1, 0x3e01, 0xff, 0xff, 0xff) }, /* D-Link DWM-152/C1 */ @@ -2000,6 +2007,7 @@ static const struct usb_device_id option_ids[] = { { USB_DEVICE_AND_INTERFACE_INFO(WETELECOM_VENDOR_ID, WETELECOM_PRODUCT_WMD200, 0xff, 0xff, 0xff) }, { USB_DEVICE_AND_INTERFACE_INFO(WETELECOM_VENDOR_ID, WETELECOM_PRODUCT_6802, 0xff, 0xff, 0xff) }, { USB_DEVICE_AND_INTERFACE_INFO(WETELECOM_VENDOR_ID, WETELECOM_PRODUCT_WMD300, 0xff, 0xff, 0xff) }, + { USB_DEVICE_AND_INTERFACE_INFO(0x03f0, 0x421d, 0xff, 0xff, 0xff) }, /* HP lt2523 (Novatel E371) */ { } /* Terminating entry */ }; MODULE_DEVICE_TABLE(usb, option_ids); diff --git a/drivers/usb/serial/oti6858.c b/drivers/usb/serial/oti6858.c index a4b88bc038b6d0..b8bf52bf7a944d 100644 --- a/drivers/usb/serial/oti6858.c +++ b/drivers/usb/serial/oti6858.c @@ -134,6 +134,7 @@ static int oti6858_chars_in_buffer(struct tty_struct *tty); static int oti6858_tiocmget(struct tty_struct *tty); static int oti6858_tiocmset(struct tty_struct *tty, unsigned int set, unsigned int clear); +static int oti6858_attach(struct usb_serial *serial); static int oti6858_port_probe(struct usb_serial_port *port); static int oti6858_port_remove(struct usb_serial_port *port); @@ -158,6 +159,7 @@ static struct usb_serial_driver oti6858_device = { .write_bulk_callback = oti6858_write_bulk_callback, .write_room = oti6858_write_room, .chars_in_buffer = oti6858_chars_in_buffer, + .attach = oti6858_attach, .port_probe = oti6858_port_probe, .port_remove = oti6858_port_remove, }; @@ -324,6 +326,20 @@ static void send_data(struct work_struct *work) usb_serial_port_softint(port); } +static int oti6858_attach(struct usb_serial *serial) +{ + unsigned char num_ports = serial->num_ports; + + if (serial->num_bulk_in < num_ports || + serial->num_bulk_out < num_ports || + serial->num_interrupt_in < num_ports) { + dev_err(&serial->interface->dev, "missing endpoints\n"); + return -ENODEV; + } + + return 0; +} + static int oti6858_port_probe(struct usb_serial_port *port) { struct oti6858_private *priv; diff --git a/drivers/usb/serial/pl2303.c b/drivers/usb/serial/pl2303.c index ae682e4eeaef57..1db4b61bdf7bd7 100644 --- a/drivers/usb/serial/pl2303.c +++ b/drivers/usb/serial/pl2303.c @@ -49,6 +49,7 @@ static const struct usb_device_id id_table[] = { { USB_DEVICE(IODATA_VENDOR_ID, IODATA_PRODUCT_ID) }, { USB_DEVICE(IODATA_VENDOR_ID, IODATA_PRODUCT_ID_RSAQ5) }, { USB_DEVICE(ATEN_VENDOR_ID, ATEN_PRODUCT_ID) }, + { USB_DEVICE(ATEN_VENDOR_ID, ATEN_PRODUCT_ID2) }, { USB_DEVICE(ATEN_VENDOR_ID2, ATEN_PRODUCT_ID) }, { USB_DEVICE(ELCOM_VENDOR_ID, ELCOM_PRODUCT_ID) }, { USB_DEVICE(ELCOM_VENDOR_ID, ELCOM_PRODUCT_ID_UCSGT) }, @@ -220,9 +221,17 @@ static int pl2303_probe(struct usb_serial *serial, static int pl2303_startup(struct usb_serial *serial) { struct pl2303_serial_private *spriv; + unsigned char num_ports = serial->num_ports; enum pl2303_type type = TYPE_01; unsigned char *buf; + if (serial->num_bulk_in < num_ports || + serial->num_bulk_out < num_ports || + serial->num_interrupt_in < num_ports) { + dev_err(&serial->interface->dev, "missing endpoints\n"); + return -ENODEV; + } + spriv = kzalloc(sizeof(*spriv), GFP_KERNEL); if (!spriv) return -ENOMEM; diff --git a/drivers/usb/serial/pl2303.h b/drivers/usb/serial/pl2303.h index e3b7af8adfb73c..09d9be88209e1c 100644 --- a/drivers/usb/serial/pl2303.h +++ b/drivers/usb/serial/pl2303.h @@ -27,6 +27,7 @@ #define ATEN_VENDOR_ID 0x0557 #define ATEN_VENDOR_ID2 0x0547 #define ATEN_PRODUCT_ID 0x2008 +#define ATEN_PRODUCT_ID2 0x2118 #define IODATA_VENDOR_ID 0x04bb #define IODATA_PRODUCT_ID 0x0a03 diff --git a/drivers/usb/serial/qcserial.c b/drivers/usb/serial/qcserial.c index 1bc6089b90083a..696458db7e3c45 100644 --- a/drivers/usb/serial/qcserial.c +++ b/drivers/usb/serial/qcserial.c @@ -124,6 +124,7 @@ static const struct usb_device_id id_table[] = { {USB_DEVICE(0x1410, 0xa021)}, /* Novatel Gobi 3000 Composite */ {USB_DEVICE(0x413c, 0x8193)}, /* Dell Gobi 3000 QDL */ {USB_DEVICE(0x413c, 0x8194)}, /* Dell Gobi 3000 Composite */ + {USB_DEVICE(0x413c, 0x81a6)}, /* Dell DW5570 QDL (MC8805) */ {USB_DEVICE(0x1199, 0x68a4)}, /* Sierra Wireless QDL */ {USB_DEVICE(0x1199, 0x68a5)}, /* Sierra Wireless Modem */ {USB_DEVICE(0x1199, 0x68a8)}, /* Sierra Wireless QDL */ diff --git a/drivers/usb/serial/quatech2.c b/drivers/usb/serial/quatech2.c index 85acb50a7ee2f4..bd1a1307e0f002 100644 --- a/drivers/usb/serial/quatech2.c +++ b/drivers/usb/serial/quatech2.c @@ -408,16 +408,12 @@ static void qt2_close(struct usb_serial_port *port) { struct usb_serial *serial; struct qt2_port_private *port_priv; - unsigned long flags; int i; serial = port->serial; port_priv = usb_get_serial_port_data(port); - spin_lock_irqsave(&port_priv->urb_lock, flags); usb_kill_urb(port_priv->write_urb); - port_priv->urb_in_use = false; - spin_unlock_irqrestore(&port_priv->urb_lock, flags); /* flush the port transmit buffer */ i = usb_control_msg(serial->dev, diff --git a/drivers/usb/serial/spcp8x5.c b/drivers/usb/serial/spcp8x5.c index ef0dbf0703c574..475e6c31b266b0 100644 --- a/drivers/usb/serial/spcp8x5.c +++ b/drivers/usb/serial/spcp8x5.c @@ -154,6 +154,19 @@ static int spcp8x5_probe(struct usb_serial *serial, return 0; } +static int spcp8x5_attach(struct usb_serial *serial) +{ + unsigned char num_ports = serial->num_ports; + + if (serial->num_bulk_in < num_ports || + serial->num_bulk_out < num_ports) { + dev_err(&serial->interface->dev, "missing endpoints\n"); + return -ENODEV; + } + + return 0; +} + static int spcp8x5_port_probe(struct usb_serial_port *port) { const struct usb_device_id *id = usb_get_serial_data(port->serial); @@ -477,6 +490,7 @@ static struct usb_serial_driver spcp8x5_device = { .tiocmget = spcp8x5_tiocmget, .tiocmset = spcp8x5_tiocmset, .probe = spcp8x5_probe, + .attach = spcp8x5_attach, .port_probe = spcp8x5_port_probe, .port_remove = spcp8x5_port_remove, }; diff --git a/drivers/usb/serial/ti_usb_3410_5052.c b/drivers/usb/serial/ti_usb_3410_5052.c index a8b9bdba314fdc..bdbddbc8bd4d0c 100644 --- a/drivers/usb/serial/ti_usb_3410_5052.c +++ b/drivers/usb/serial/ti_usb_3410_5052.c @@ -579,6 +579,13 @@ static int ti_startup(struct usb_serial *serial) goto free_tdev; } + if (serial->num_bulk_in < serial->num_ports || + serial->num_bulk_out < serial->num_ports) { + dev_err(&serial->interface->dev, "missing endpoints\n"); + status = -ENODEV; + goto free_tdev; + } + return 0; free_tdev: diff --git a/drivers/usb/storage/unusual_devs.h b/drivers/usb/storage/unusual_devs.h index af3c7eecff91d4..16cc18369111d0 100644 --- a/drivers/usb/storage/unusual_devs.h +++ b/drivers/usb/storage/unusual_devs.h @@ -2109,6 +2109,13 @@ UNUSUAL_DEV( 0x152d, 0x2566, 0x0114, 0x0114, USB_SC_DEVICE, USB_PR_DEVICE, NULL, US_FL_BROKEN_FUA ), +/* Reported-by George Cherian */ +UNUSUAL_DEV(0x152d, 0x9561, 0x0000, 0x9999, + "JMicron", + "JMS56x", + USB_SC_DEVICE, USB_PR_DEVICE, NULL, + US_FL_NO_REPORT_OPCODES), + /* * Entrega Technologies U1-SC25 (later Xircom PortGear PGSCSI) * and Mac USB Dock USB-SCSI */ diff --git a/drivers/usb/usbip/vudc_transfer.c b/drivers/usb/usbip/vudc_transfer.c index aba6bd47804511..bc0296d937d0f6 100644 --- a/drivers/usb/usbip/vudc_transfer.c +++ b/drivers/usb/usbip/vudc_transfer.c @@ -339,6 +339,8 @@ static void v_timer(unsigned long _vudc) total = timer->frame_limit; } + /* We have to clear ep0 flags separately as it's not on the list */ + udc->ep[0].already_seen = 0; list_for_each_entry(_ep, &udc->gadget.ep_list, ep_list) { ep = to_vep(_ep); ep->already_seen = 0; diff --git a/drivers/usb/wusbcore/crypto.c b/drivers/usb/wusbcore/crypto.c index 79451f7ef1b763..062c205f00469f 100644 --- a/drivers/usb/wusbcore/crypto.c +++ b/drivers/usb/wusbcore/crypto.c @@ -216,7 +216,6 @@ static int wusb_ccm_mac(struct crypto_skcipher *tfm_cbc, struct scatterlist sg[4], sg_dst; void *dst_buf; size_t dst_size; - const u8 bzero[16] = { 0 }; u8 iv[crypto_skcipher_ivsize(tfm_cbc)]; size_t zero_padding; @@ -261,7 +260,7 @@ static int wusb_ccm_mac(struct crypto_skcipher *tfm_cbc, sg_set_buf(&sg[1], &scratch->b1, sizeof(scratch->b1)); sg_set_buf(&sg[2], b, blen); /* 0 if well behaved :) */ - sg_set_buf(&sg[3], bzero, zero_padding); + sg_set_page(&sg[3], ZERO_PAGE(0), zero_padding, 0); sg_init_one(&sg_dst, dst_buf, dst_size); skcipher_request_set_tfm(req, tfm_cbc); diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index c6f2d89c0e97cd..64613fbf5cf8a5 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -130,14 +130,14 @@ static long vhost_get_vring_endian(struct vhost_virtqueue *vq, u32 idx, static void vhost_init_is_le(struct vhost_virtqueue *vq) { - if (vhost_has_feature(vq, VIRTIO_F_VERSION_1)) - vq->is_le = true; + vq->is_le = vhost_has_feature(vq, VIRTIO_F_VERSION_1) + || virtio_legacy_is_little_endian(); } #endif /* CONFIG_VHOST_CROSS_ENDIAN_LEGACY */ static void vhost_reset_is_le(struct vhost_virtqueue *vq) { - vq->is_le = virtio_legacy_is_little_endian(); + vhost_init_is_le(vq); } struct vhost_flush_struct { @@ -1713,10 +1713,8 @@ int vhost_vq_init_access(struct vhost_virtqueue *vq) int r; bool is_le = vq->is_le; - if (!vq->private_data) { - vhost_reset_is_le(vq); + if (!vq->private_data) return 0; - } vhost_init_is_le(vq); diff --git a/drivers/video/Kconfig b/drivers/video/Kconfig index 3c20af99989357..1e31cf0550b504 100644 --- a/drivers/video/Kconfig +++ b/drivers/video/Kconfig @@ -18,6 +18,7 @@ source "drivers/gpu/vga/Kconfig" source "drivers/gpu/host1x/Kconfig" source "drivers/gpu/ipu-v3/Kconfig" +source "drivers/gpu/arm/Kconfig" source "drivers/gpu/drm/Kconfig" menu "Frame buffer Devices" diff --git a/drivers/video/fbdev/core/fbcmap.c b/drivers/video/fbdev/core/fbcmap.c index f89245b8ba8e9a..68a113594808f2 100644 --- a/drivers/video/fbdev/core/fbcmap.c +++ b/drivers/video/fbdev/core/fbcmap.c @@ -163,17 +163,18 @@ void fb_dealloc_cmap(struct fb_cmap *cmap) int fb_copy_cmap(const struct fb_cmap *from, struct fb_cmap *to) { - int tooff = 0, fromoff = 0; - int size; + unsigned int tooff = 0, fromoff = 0; + size_t size; if (to->start > from->start) fromoff = to->start - from->start; else tooff = from->start - to->start; - size = to->len - tooff; - if (size > (int) (from->len - fromoff)) - size = from->len - fromoff; - if (size <= 0) + if (fromoff >= from->len || tooff >= to->len) + return -EINVAL; + + size = min_t(size_t, to->len - tooff, from->len - fromoff); + if (size == 0) return -EINVAL; size *= sizeof(u16); @@ -187,17 +188,18 @@ int fb_copy_cmap(const struct fb_cmap *from, struct fb_cmap *to) int fb_cmap_to_user(const struct fb_cmap *from, struct fb_cmap_user *to) { - int tooff = 0, fromoff = 0; - int size; + unsigned int tooff = 0, fromoff = 0; + size_t size; if (to->start > from->start) fromoff = to->start - from->start; else tooff = from->start - to->start; - size = to->len - tooff; - if (size > (int) (from->len - fromoff)) - size = from->len - fromoff; - if (size <= 0) + if (fromoff >= from->len || tooff >= to->len) + return -EINVAL; + + size = min_t(size_t, to->len - tooff, from->len - fromoff); + if (size == 0) return -EINVAL; size *= sizeof(u16); diff --git a/drivers/video/fbdev/goldfishfb.c b/drivers/video/fbdev/goldfishfb.c index 7f6c9e6cfc6c99..1e56b50e408234 100644 --- a/drivers/video/fbdev/goldfishfb.c +++ b/drivers/video/fbdev/goldfishfb.c @@ -26,6 +26,7 @@ #include #include #include +#include enum { FB_GET_WIDTH = 0x00, @@ -234,7 +235,7 @@ static int goldfish_fb_probe(struct platform_device *pdev) fb->fb.var.activate = FB_ACTIVATE_NOW; fb->fb.var.height = readl(fb->reg_base + FB_GET_PHYS_HEIGHT); fb->fb.var.width = readl(fb->reg_base + FB_GET_PHYS_WIDTH); - fb->fb.var.pixclock = 10000; + fb->fb.var.pixclock = 0; fb->fb.var.red.offset = 11; fb->fb.var.red.length = 5; @@ -304,12 +305,25 @@ static int goldfish_fb_remove(struct platform_device *pdev) return 0; } +static const struct of_device_id goldfish_fb_of_match[] = { + { .compatible = "google,goldfish-fb", }, + {}, +}; +MODULE_DEVICE_TABLE(of, goldfish_fb_of_match); + +static const struct acpi_device_id goldfish_fb_acpi_match[] = { + { "GFSH0004", 0 }, + { }, +}; +MODULE_DEVICE_TABLE(acpi, goldfish_fb_acpi_match); static struct platform_driver goldfish_fb_driver = { .probe = goldfish_fb_probe, .remove = goldfish_fb_remove, .driver = { - .name = "goldfish_fb" + .name = "goldfish_fb", + .of_match_table = goldfish_fb_of_match, + .acpi_match_table = ACPI_PTR(goldfish_fb_acpi_match), } }; diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c index 48bfea91dbcac9..50840984fbfac9 100644 --- a/drivers/virtio/virtio_mmio.c +++ b/drivers/virtio/virtio_mmio.c @@ -59,6 +59,7 @@ #define pr_fmt(fmt) "virtio-mmio: " fmt #include +#include #include #include #include @@ -497,6 +498,7 @@ static int virtio_mmio_probe(struct platform_device *pdev) struct virtio_mmio_device *vm_dev; struct resource *mem; unsigned long magic; + int rc; mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (!mem) @@ -545,9 +547,25 @@ static int virtio_mmio_probe(struct platform_device *pdev) } vm_dev->vdev.id.vendor = readl(vm_dev->base + VIRTIO_MMIO_VENDOR_ID); - if (vm_dev->version == 1) + if (vm_dev->version == 1) { writel(PAGE_SIZE, vm_dev->base + VIRTIO_MMIO_GUEST_PAGE_SIZE); + rc = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)); + /* + * In the legacy case, ensure our coherently-allocated virtio + * ring will be at an address expressable as a 32-bit PFN. + */ + if (!rc) + dma_set_coherent_mask(&pdev->dev, + DMA_BIT_MASK(32 + PAGE_SHIFT)); + } else { + rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + } + if (rc) + rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (rc) + dev_warn(&pdev->dev, "Failed to enable 64-bit or 32-bit DMA. Trying to continue, but this might not work.\n"); + platform_set_drvdata(pdev, vm_dev); return register_virtio_device(&vm_dev->vdev); diff --git a/drivers/vme/bridges/vme_ca91cx42.c b/drivers/vme/bridges/vme_ca91cx42.c index 6b5ee896af6318..7cc51223db1cbe 100644 --- a/drivers/vme/bridges/vme_ca91cx42.c +++ b/drivers/vme/bridges/vme_ca91cx42.c @@ -464,7 +464,7 @@ static int ca91cx42_slave_get(struct vme_slave_resource *image, int *enabled, vme_bound = ioread32(bridge->base + CA91CX42_VSI_BD[i]); pci_offset = ioread32(bridge->base + CA91CX42_VSI_TO[i]); - *pci_base = (dma_addr_t)vme_base + pci_offset; + *pci_base = (dma_addr_t)*vme_base + pci_offset; *size = (unsigned long long)((vme_bound - *vme_base) + granularity); *enabled = 0; diff --git a/drivers/w1/masters/ds2482.c b/drivers/w1/masters/ds2482.c index 2e30db1b1a43a4..fa13fa8c81af56 100644 --- a/drivers/w1/masters/ds2482.c +++ b/drivers/w1/masters/ds2482.c @@ -18,6 +18,8 @@ #include #include #include +#include +#include #include #include "../w1.h" @@ -97,7 +99,8 @@ static const u8 ds2482_chan_rd[8] = static int ds2482_probe(struct i2c_client *client, const struct i2c_device_id *id); static int ds2482_remove(struct i2c_client *client); - +static int ds2482_suspend(struct device *dev); +static int ds2482_resume(struct device *dev); /** * Driver data (common to all clients) @@ -108,9 +111,15 @@ static const struct i2c_device_id ds2482_id[] = { }; MODULE_DEVICE_TABLE(i2c, ds2482_id); +static const struct dev_pm_ops ds2482_pm_ops = { + .suspend = ds2482_suspend, + .resume = ds2482_resume, +}; + static struct i2c_driver ds2482_driver = { .driver = { .name = "ds2482", + .pm = &ds2482_pm_ops, }, .probe = ds2482_probe, .remove = ds2482_remove, @@ -132,6 +141,7 @@ struct ds2482_w1_chan { struct ds2482_data { struct i2c_client *client; struct mutex access_lock; + int slpz_gpio; /* 1-wire interface(s) */ int w1_count; /* 1 or 8 */ @@ -460,11 +470,31 @@ static u8 ds2482_w1_set_pullup(void *data, int delay) return retval; } +static int ds2482_suspend(struct device *dev) +{ + struct i2c_client *client = to_i2c_client(dev); + struct ds2482_data *data = i2c_get_clientdata(client); + + if (data->slpz_gpio >= 0) + gpio_set_value(data->slpz_gpio, 0); + return 0; +} + +static int ds2482_resume(struct device *dev) +{ + struct i2c_client *client = to_i2c_client(dev); + struct ds2482_data *data = i2c_get_clientdata(client); + + if (data->slpz_gpio >= 0) + gpio_set_value(data->slpz_gpio, 1); + return 0; +} static int ds2482_probe(struct i2c_client *client, const struct i2c_device_id *id) { struct ds2482_data *data; + struct ds2482_platform_data *pdata; int err = -ENODEV; int temp1; int idx; @@ -531,6 +561,16 @@ static int ds2482_probe(struct i2c_client *client, } } + pdata = client->dev.platform_data; + data->slpz_gpio = pdata ? pdata->slpz_gpio : -1; + + if (data->slpz_gpio >= 0) { + err = gpio_request_one(data->slpz_gpio, GPIOF_OUT_INIT_HIGH, + "ds2482.slpz"); + if (err < 0) + goto exit_w1_remove; + } + return 0; exit_w1_remove: @@ -555,6 +595,11 @@ static int ds2482_remove(struct i2c_client *client) w1_remove_master_device(&data->w1_ch[idx].w1_bm); } + if (data->slpz_gpio >= 0) { + gpio_set_value(data->slpz_gpio, 0); + gpio_free(data->slpz_gpio); + } + /* Free the memory */ kfree(data); return 0; diff --git a/drivers/watchdog/mei_wdt.c b/drivers/watchdog/mei_wdt.c index 630bd189f16788..2a9d5cdedea2cf 100644 --- a/drivers/watchdog/mei_wdt.c +++ b/drivers/watchdog/mei_wdt.c @@ -389,6 +389,8 @@ static int mei_wdt_register(struct mei_wdt *wdt) wdt->wdd.max_timeout = MEI_WDT_MAX_TIMEOUT; watchdog_set_drvdata(&wdt->wdd, wdt); + watchdog_stop_on_reboot(&wdt->wdd); + ret = watchdog_register_device(&wdt->wdd); if (ret) { dev_err(dev, "unable to register watchdog device = %d.\n", ret); diff --git a/drivers/watchdog/qcom-wdt.c b/drivers/watchdog/qcom-wdt.c index 5796b5d1b3f2cb..4f47b5e9095662 100644 --- a/drivers/watchdog/qcom-wdt.c +++ b/drivers/watchdog/qcom-wdt.c @@ -209,7 +209,7 @@ static int qcom_wdt_probe(struct platform_device *pdev) wdt->wdd.parent = &pdev->dev; wdt->layout = regs; - if (readl(wdt->base + WDT_STS) & 1) + if (readl(wdt_addr(wdt, WDT_STS)) & 1) wdt->wdd.bootstatus = WDIOF_CARDRESET; /* diff --git a/drivers/xen/gntdev.c b/drivers/xen/gntdev.c index bb952121ea944b..2ef2b61b69dfe0 100644 --- a/drivers/xen/gntdev.c +++ b/drivers/xen/gntdev.c @@ -1007,7 +1007,7 @@ static int gntdev_mmap(struct file *flip, struct vm_area_struct *vma) vma->vm_ops = &gntdev_vmops; - vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP | VM_IO; + vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP | VM_MIXEDMAP; if (use_ptemod) vma->vm_flags |= VM_DONTCOPY; diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c index 87e6035c9e81b2..8e7a3d646531cc 100644 --- a/drivers/xen/swiotlb-xen.c +++ b/drivers/xen/swiotlb-xen.c @@ -392,7 +392,7 @@ dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page, if (dma_capable(dev, dev_addr, size) && !range_straddles_page_boundary(phys, size) && !xen_arch_need_swiotlb(dev, phys, dev_addr) && - !swiotlb_force) { + (swiotlb_force != SWIOTLB_FORCE)) { /* we are not interested in the dma_addr returned by * xen_dma_map_page, only in the potential cache flushes executed * by the function. */ @@ -549,7 +549,7 @@ xen_swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, phys_addr_t paddr = sg_phys(sg); dma_addr_t dev_addr = xen_phys_to_bus(paddr); - if (swiotlb_force || + if (swiotlb_force == SWIOTLB_FORCE || xen_arch_need_swiotlb(hwdev, paddr, dev_addr) || !dma_capable(hwdev, dev_addr, sg->length) || range_straddles_page_boundary(paddr, sg->length)) { diff --git a/fs/9p/acl.c b/fs/9p/acl.c index b3c2cc79c20d25..082d227fa56b37 100644 --- a/fs/9p/acl.c +++ b/fs/9p/acl.c @@ -277,6 +277,7 @@ static int v9fs_xattr_set_acl(const struct xattr_handler *handler, case ACL_TYPE_ACCESS: if (acl) { struct iattr iattr; + struct posix_acl *old_acl = acl; retval = posix_acl_update_mode(inode, &iattr.ia_mode, &acl); if (retval) @@ -287,6 +288,7 @@ static int v9fs_xattr_set_acl(const struct xattr_handler *handler, * by the mode bits. So don't * update ACL. */ + posix_acl_release(old_acl); value = NULL; size = 0; } diff --git a/fs/Kconfig b/fs/Kconfig index 4bd03a2b051817..20a8d95b8b3df7 100644 --- a/fs/Kconfig +++ b/fs/Kconfig @@ -227,6 +227,7 @@ source "fs/orangefs/Kconfig" source "fs/adfs/Kconfig" source "fs/affs/Kconfig" source "fs/ecryptfs/Kconfig" +source "fs/sdcardfs/Kconfig" source "fs/hfs/Kconfig" source "fs/hfsplus/Kconfig" source "fs/befs/Kconfig" diff --git a/fs/Makefile b/fs/Makefile index ed2b63257ba99b..f207d43030520d 100644 --- a/fs/Makefile +++ b/fs/Makefile @@ -3,7 +3,7 @@ # # 14 Sep 2000, Christoph Hellwig # Rewritten to use lists instead of if-statements. -# +# obj-y := open.o read_write.o file_table.o super.o \ char_dev.o stat.o exec.o pipe.o namei.o fcntl.o \ @@ -61,7 +61,7 @@ obj-y += devpts/ obj-$(CONFIG_PROFILING) += dcookies.o obj-$(CONFIG_DLM) += dlm/ - + # Do not add any filesystems before this line obj-$(CONFIG_FSCACHE) += fscache/ obj-$(CONFIG_REISERFS_FS) += reiserfs/ @@ -83,6 +83,7 @@ obj-$(CONFIG_ISO9660_FS) += isofs/ obj-$(CONFIG_HFSPLUS_FS) += hfsplus/ # Before hfs to find wrapped HFS+ obj-$(CONFIG_HFS_FS) += hfs/ obj-$(CONFIG_ECRYPT_FS) += ecryptfs/ +obj-$(CONFIG_SDCARD_FS) += sdcardfs/ obj-$(CONFIG_VXFS_FS) += freevxfs/ obj-$(CONFIG_NFS_FS) += nfs/ obj-$(CONFIG_EXPORTFS) += exportfs/ diff --git a/fs/attr.c b/fs/attr.c index c902b3d5350800..c4093c5196be65 100644 --- a/fs/attr.c +++ b/fs/attr.c @@ -200,7 +200,7 @@ EXPORT_SYMBOL(setattr_copy); * the file open for write, as there can be no conflicting delegation in * that case. */ -int notify_change(struct dentry * dentry, struct iattr * attr, struct inode **delegated_inode) +int notify_change2(struct vfsmount *mnt, struct dentry * dentry, struct iattr * attr, struct inode **delegated_inode) { struct inode *inode = dentry->d_inode; umode_t mode = inode->i_mode; @@ -224,7 +224,7 @@ int notify_change(struct dentry * dentry, struct iattr * attr, struct inode **de return -EPERM; if (!inode_owner_or_capable(inode)) { - error = inode_permission(inode, MAY_WRITE); + error = inode_permission2(mnt, inode, MAY_WRITE); if (error) return error; } @@ -307,7 +307,9 @@ int notify_change(struct dentry * dentry, struct iattr * attr, struct inode **de if (error) return error; - if (inode->i_op->setattr) + if (mnt && inode->i_op->setattr2) + error = inode->i_op->setattr2(mnt, dentry, attr); + else if (inode->i_op->setattr) error = inode->i_op->setattr(dentry, attr); else error = simple_setattr(dentry, attr); @@ -320,4 +322,10 @@ int notify_change(struct dentry * dentry, struct iattr * attr, struct inode **de return error; } +EXPORT_SYMBOL(notify_change2); + +int notify_change(struct dentry * dentry, struct iattr * attr, struct inode **delegated_inode) +{ + return notify_change2(NULL, dentry, attr, delegated_inode); +} EXPORT_SYMBOL(notify_change); diff --git a/fs/bad_inode.c b/fs/bad_inode.c index 8712062275b83f..5f685c81929818 100644 --- a/fs/bad_inode.c +++ b/fs/bad_inode.c @@ -106,6 +106,50 @@ static ssize_t bad_inode_listxattr(struct dentry *dentry, char *buffer, return -EIO; } +static const char *bad_inode_get_link(struct dentry *dentry, + struct inode *inode, + struct delayed_call *done) +{ + return ERR_PTR(-EIO); +} + +static struct posix_acl *bad_inode_get_acl(struct inode *inode, int type) +{ + return ERR_PTR(-EIO); +} + +static int bad_inode_fiemap(struct inode *inode, + struct fiemap_extent_info *fieinfo, u64 start, + u64 len) +{ + return -EIO; +} + +static int bad_inode_update_time(struct inode *inode, struct timespec *time, + int flags) +{ + return -EIO; +} + +static int bad_inode_atomic_open(struct inode *inode, struct dentry *dentry, + struct file *file, unsigned int open_flag, + umode_t create_mode, int *opened) +{ + return -EIO; +} + +static int bad_inode_tmpfile(struct inode *inode, struct dentry *dentry, + umode_t mode) +{ + return -EIO; +} + +static int bad_inode_set_acl(struct inode *inode, struct posix_acl *acl, + int type) +{ + return -EIO; +} + static const struct inode_operations bad_inode_ops = { .create = bad_inode_create, @@ -118,14 +162,17 @@ static const struct inode_operations bad_inode_ops = .mknod = bad_inode_mknod, .rename = bad_inode_rename2, .readlink = bad_inode_readlink, - /* follow_link must be no-op, otherwise unmounting this inode - won't work */ - /* put_link returns void */ - /* truncate returns void */ .permission = bad_inode_permission, .getattr = bad_inode_getattr, .setattr = bad_inode_setattr, .listxattr = bad_inode_listxattr, + .get_link = bad_inode_get_link, + .get_acl = bad_inode_get_acl, + .fiemap = bad_inode_fiemap, + .update_time = bad_inode_update_time, + .atomic_open = bad_inode_atomic_open, + .tmpfile = bad_inode_tmpfile, + .set_acl = bad_inode_set_acl, }; diff --git a/fs/block_dev.c b/fs/block_dev.c index 05b553368bb4f1..092a2eed162895 100644 --- a/fs/block_dev.c +++ b/fs/block_dev.c @@ -832,7 +832,7 @@ static bool bd_may_claim(struct block_device *bdev, struct block_device *whole, return true; /* already a holder */ else if (bdev->bd_holder != NULL) return false; /* held by someone else */ - else if (bdev->bd_contains == bdev) + else if (whole == bdev) return true; /* is a whole device which isn't held */ else if (whole->bd_holder == bd_may_claim) @@ -1950,6 +1950,7 @@ void iterate_bdevs(void (*func)(struct block_device *, void *), void *arg) spin_lock(&blockdev_superblock->s_inode_list_lock); list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list) { struct address_space *mapping = inode->i_mapping; + struct block_device *bdev; spin_lock(&inode->i_lock); if (inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW) || @@ -1970,8 +1971,12 @@ void iterate_bdevs(void (*func)(struct block_device *, void *), void *arg) */ iput(old_inode); old_inode = inode; + bdev = I_BDEV(inode); - func(I_BDEV(inode), arg); + mutex_lock(&bdev->bd_mutex); + if (bdev->bd_openers) + func(bdev, arg); + mutex_unlock(&bdev->bd_mutex); spin_lock(&blockdev_superblock->s_inode_list_lock); } diff --git a/fs/btrfs/async-thread.c b/fs/btrfs/async-thread.c index e0f071f6b5a761..ff0b0be92d6122 100644 --- a/fs/btrfs/async-thread.c +++ b/fs/btrfs/async-thread.c @@ -86,6 +86,20 @@ btrfs_work_owner(struct btrfs_work *work) return work->wq->fs_info; } +bool btrfs_workqueue_normal_congested(struct btrfs_workqueue *wq) +{ + /* + * We could compare wq->normal->pending with num_online_cpus() + * to support "thresh == NO_THRESHOLD" case, but it requires + * moving up atomic_inc/dec in thresh_queue/exec_hook. Let's + * postpone it until someone needs the support of that case. + */ + if (wq->normal->thresh == NO_THRESHOLD) + return false; + + return atomic_read(&wq->normal->pending) > wq->normal->thresh * 2; +} + BTRFS_WORK_HELPER(worker_helper); BTRFS_WORK_HELPER(delalloc_helper); BTRFS_WORK_HELPER(flush_delalloc_helper); @@ -259,6 +273,8 @@ static void run_ordered_work(struct __btrfs_workqueue *wq) unsigned long flags; while (1) { + void *wtag; + spin_lock_irqsave(lock, flags); if (list_empty(list)) break; @@ -285,11 +301,13 @@ static void run_ordered_work(struct __btrfs_workqueue *wq) spin_unlock_irqrestore(lock, flags); /* - * we don't want to call the ordered free functions - * with the lock held though + * We don't want to call the ordered free functions with the + * lock held though. Save the work as tag for the trace event, + * because the callback could free the structure. */ + wtag = work; work->ordered_free(work); - trace_btrfs_all_work_done(work); + trace_btrfs_all_work_done(wq->fs_info, wtag); } spin_unlock_irqrestore(lock, flags); } @@ -297,6 +315,7 @@ static void run_ordered_work(struct __btrfs_workqueue *wq) static void normal_work_helper(struct btrfs_work *work) { struct __btrfs_workqueue *wq; + void *wtag; int need_order = 0; /* @@ -310,6 +329,8 @@ static void normal_work_helper(struct btrfs_work *work) if (work->ordered_func) need_order = 1; wq = work->wq; + /* Safe for tracepoints in case work gets freed by the callback */ + wtag = work; trace_btrfs_work_sched(work); thresh_exec_hook(wq); @@ -319,7 +340,7 @@ static void normal_work_helper(struct btrfs_work *work) run_ordered_work(wq); } if (!need_order) - trace_btrfs_all_work_done(work); + trace_btrfs_all_work_done(wq->fs_info, wtag); } void btrfs_init_work(struct btrfs_work *work, btrfs_work_func_t uniq_func, diff --git a/fs/btrfs/async-thread.h b/fs/btrfs/async-thread.h index 8e52484cd46155..1f9597355c9d97 100644 --- a/fs/btrfs/async-thread.h +++ b/fs/btrfs/async-thread.h @@ -84,4 +84,5 @@ void btrfs_workqueue_set_max(struct btrfs_workqueue *wq, int max); void btrfs_set_work_high_priority(struct btrfs_work *work); struct btrfs_fs_info *btrfs_work_owner(struct btrfs_work *work); struct btrfs_fs_info *btrfs_workqueue_owner(struct __btrfs_workqueue *wq); +bool btrfs_workqueue_normal_congested(struct btrfs_workqueue *wq); #endif diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index 0b8ce2b9f7d0c8..86245b884fce29 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -2210,6 +2210,8 @@ btrfs_disk_balance_args_to_cpu(struct btrfs_balance_args *cpu, cpu->target = le64_to_cpu(disk->target); cpu->flags = le64_to_cpu(disk->flags); cpu->limit = le64_to_cpu(disk->limit); + cpu->stripes_min = le32_to_cpu(disk->stripes_min); + cpu->stripes_max = le32_to_cpu(disk->stripes_max); } static inline void @@ -2228,6 +2230,8 @@ btrfs_cpu_balance_args_to_disk(struct btrfs_disk_balance_args *disk, disk->target = cpu_to_le64(cpu->target); disk->flags = cpu_to_le64(cpu->flags); disk->limit = cpu_to_le64(cpu->limit); + disk->stripes_min = cpu_to_le32(cpu->stripes_min); + disk->stripes_max = cpu_to_le32(cpu->stripes_max); } /* struct btrfs_super_block */ diff --git a/fs/btrfs/delayed-inode.c b/fs/btrfs/delayed-inode.c index 0fcf5f25d524ab..4d8f8a8c9c908d 100644 --- a/fs/btrfs/delayed-inode.c +++ b/fs/btrfs/delayed-inode.c @@ -1353,7 +1353,8 @@ static void btrfs_async_run_delayed_root(struct btrfs_work *work) total_done++; btrfs_release_prepared_delayed_node(delayed_node); - if (async_work->nr == 0 || total_done < async_work->nr) + if ((async_work->nr == 0 && total_done < BTRFS_DELAYED_WRITEBACK) || + total_done < async_work->nr) goto again; free_path: @@ -1369,7 +1370,8 @@ static int btrfs_wq_run_delayed_node(struct btrfs_delayed_root *delayed_root, { struct btrfs_async_delayed_work *async_work; - if (atomic_read(&delayed_root->items) < BTRFS_DELAYED_BACKGROUND) + if (atomic_read(&delayed_root->items) < BTRFS_DELAYED_BACKGROUND || + btrfs_workqueue_normal_congested(fs_info->delayed_workers)) return 0; async_work = kmalloc(sizeof(*async_work), GFP_NOFS); diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 3a57f99d96aa7a..1cd325765aaa8c 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -559,7 +559,15 @@ static noinline int check_leaf(struct btrfs_root *root, u32 nritems = btrfs_header_nritems(leaf); int slot; - if (nritems == 0) { + /* + * Extent buffers from a relocation tree have a owner field that + * corresponds to the subvolume tree they are based on. So just from an + * extent buffer alone we can not find out what is the id of the + * corresponding subvolume tree, so we can not figure out if the extent + * buffer corresponds to the root of the relocation tree or not. So skip + * this check for relocation trees. + */ + if (nritems == 0 && !btrfs_header_flag(leaf, BTRFS_HEADER_FLAG_RELOC)) { struct btrfs_root *check_root; key.objectid = btrfs_header_owner(leaf); @@ -572,17 +580,24 @@ static noinline int check_leaf(struct btrfs_root *root, * open_ctree() some roots has not yet been set up. */ if (!IS_ERR_OR_NULL(check_root)) { + struct extent_buffer *eb; + + eb = btrfs_root_node(check_root); /* if leaf is the root, then it's fine */ - if (leaf->start != - btrfs_root_bytenr(&check_root->root_item)) { + if (leaf != eb) { CORRUPT("non-root leaf's nritems is 0", - leaf, root, 0); + leaf, check_root, 0); + free_extent_buffer(eb); return -EIO; } + free_extent_buffer(eb); } return 0; } + if (nritems == 0) + return 0; + /* Check the 0 item */ if (btrfs_item_offset_nr(leaf, 0) + btrfs_item_size_nr(leaf, 0) != BTRFS_LEAF_DATA_SIZE(root)) { diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index 4607af38c72e10..5909ae8c673102 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -2537,11 +2537,11 @@ static noinline int __btrfs_run_delayed_refs(struct btrfs_trans_handle *trans, if (ref && ref->seq && btrfs_check_delayed_seq(fs_info, delayed_refs, ref->seq)) { spin_unlock(&locked_ref->lock); - btrfs_delayed_ref_unlock(locked_ref); spin_lock(&delayed_refs->lock); locked_ref->processing = 0; delayed_refs->num_heads_ready++; spin_unlock(&delayed_refs->lock); + btrfs_delayed_ref_unlock(locked_ref); locked_ref = NULL; cond_resched(); count++; @@ -2587,7 +2587,10 @@ static noinline int __btrfs_run_delayed_refs(struct btrfs_trans_handle *trans, */ if (must_insert_reserved) locked_ref->must_insert_reserved = 1; + spin_lock(&delayed_refs->lock); locked_ref->processing = 0; + delayed_refs->num_heads_ready++; + spin_unlock(&delayed_refs->lock); btrfs_debug(fs_info, "run_delayed_extent_op returned %d", ret); diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 8e3a5a266917c0..be4da91d880f64 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -3819,10 +3819,7 @@ static int btrfs_read_locked_inode(struct inode *inode) break; case S_IFDIR: inode->i_fop = &btrfs_dir_file_operations; - if (root == root->fs_info->tree_root) - inode->i_op = &btrfs_dir_ro_inode_operations; - else - inode->i_op = &btrfs_dir_inode_operations; + inode->i_op = &btrfs_dir_inode_operations; break; case S_IFLNK: inode->i_op = &btrfs_symlink_inode_operations; @@ -5682,6 +5679,7 @@ static struct inode *new_simple_dir(struct super_block *s, inode->i_ino = BTRFS_EMPTY_SUBVOL_DIR_OBJECTID; inode->i_op = &btrfs_dir_ro_inode_operations; + inode->i_opflags &= ~IOP_XATTR; inode->i_fop = &simple_dir_operations; inode->i_mode = S_IFDIR | S_IRUGO | S_IWUSR | S_IXUGO; inode->i_mtime = current_time(inode); @@ -10587,8 +10585,6 @@ static const struct inode_operations btrfs_dir_inode_operations = { static const struct inode_operations btrfs_dir_ro_inode_operations = { .lookup = btrfs_lookup, .permission = btrfs_permission, - .get_acl = btrfs_get_acl, - .set_acl = btrfs_set_acl, .update_time = btrfs_update_time, }; diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index 7acbd2cf6192ee..1782804f6c2631 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -5648,6 +5648,10 @@ long btrfs_ioctl(struct file *file, unsigned int #ifdef CONFIG_COMPAT long btrfs_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { + /* + * These all access 32-bit values anyway so no further + * handling is necessary. + */ switch (cmd) { case FS_IOC32_GETFLAGS: cmd = FS_IOC_GETFLAGS; @@ -5658,8 +5662,6 @@ long btrfs_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg) case FS_IOC32_GETVERSION: cmd = FS_IOC_GETVERSION; break; - default: - return -ENOIOCTLCMD; } return btrfs_ioctl(file, cmd, (unsigned long) compat_ptr(arg)); diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c index 11f4fffe503e23..dfd99867ff4d7e 100644 --- a/fs/btrfs/qgroup.c +++ b/fs/btrfs/qgroup.c @@ -2335,10 +2335,6 @@ static void btrfs_qgroup_rescan_worker(struct btrfs_work *work) int err = -ENOMEM; int ret = 0; - mutex_lock(&fs_info->qgroup_rescan_lock); - fs_info->qgroup_rescan_running = true; - mutex_unlock(&fs_info->qgroup_rescan_lock); - path = btrfs_alloc_path(); if (!path) goto out; @@ -2449,6 +2445,7 @@ qgroup_rescan_init(struct btrfs_fs_info *fs_info, u64 progress_objectid, sizeof(fs_info->qgroup_rescan_progress)); fs_info->qgroup_rescan_progress.objectid = progress_objectid; init_completion(&fs_info->qgroup_rescan_completion); + fs_info->qgroup_rescan_running = true; spin_unlock(&fs_info->qgroup_lock); mutex_unlock(&fs_info->qgroup_rescan_lock); diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index c4af0cdb783d0e..2cf5e142675e18 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -1395,14 +1395,23 @@ static struct btrfs_root *create_reloc_root(struct btrfs_trans_handle *trans, root_key.offset = objectid; if (root->root_key.objectid == objectid) { + u64 commit_root_gen; + /* called by btrfs_init_reloc_root */ ret = btrfs_copy_root(trans, root, root->commit_root, &eb, BTRFS_TREE_RELOC_OBJECTID); BUG_ON(ret); - last_snap = btrfs_root_last_snapshot(&root->root_item); - btrfs_set_root_last_snapshot(&root->root_item, - trans->transid - 1); + /* + * Set the last_snapshot field to the generation of the commit + * root - like this ctree.c:btrfs_block_can_be_shared() behaves + * correctly (returns true) when the relocation root is created + * either inside the critical section of a transaction commit + * (through transaction.c:qgroup_account_snapshot()) and when + * it's created before the transaction commit is started. + */ + commit_root_gen = btrfs_header_generation(root->commit_root); + btrfs_set_root_last_snapshot(&root->root_item, commit_root_gen); } else { /* * called by btrfs_reloc_post_snapshot_hook. diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index 3d33c4e41e5f9a..b89004513c09a9 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -1940,12 +1940,11 @@ static noinline int find_dir_range(struct btrfs_root *root, next: /* check the next slot in the tree to see if it is a valid item */ nritems = btrfs_header_nritems(path->nodes[0]); + path->slots[0]++; if (path->slots[0] >= nritems) { ret = btrfs_next_leaf(root, path); if (ret) goto out; - } else { - path->slots[0]++; } btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); @@ -5205,6 +5204,7 @@ static int log_new_dir_dentries(struct btrfs_trans_handle *trans, if (di_key.type == BTRFS_ROOT_ITEM_KEY) continue; + btrfs_release_path(path); di_inode = btrfs_iget(root->fs_info->sb, &di_key, root, NULL); if (IS_ERR(di_inode)) { @@ -5214,13 +5214,12 @@ static int log_new_dir_dentries(struct btrfs_trans_handle *trans, if (btrfs_inode_in_log(di_inode, trans->transid)) { iput(di_inode); - continue; + break; } ctx->log_new_dentries = false; if (type == BTRFS_FT_DIR || type == BTRFS_FT_SYMLINK) log_mode = LOG_INODE_ALL; - btrfs_release_path(path); ret = btrfs_log_inode(trans, root, di_inode, log_mode, 0, LLONG_MAX, ctx); if (!ret && diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c index 16e6ded0b7f281..f3f21105b86044 100644 --- a/fs/ceph/caps.c +++ b/fs/ceph/caps.c @@ -2507,9 +2507,20 @@ int ceph_get_caps(struct ceph_inode_info *ci, int need, int want, if (err < 0) ret = err; } else { - ret = wait_event_interruptible(ci->i_cap_wq, - try_get_cap_refs(ci, need, want, endoff, - true, &_got, &err)); + DEFINE_WAIT_FUNC(wait, woken_wake_function); + add_wait_queue(&ci->i_cap_wq, &wait); + + while (!try_get_cap_refs(ci, need, want, endoff, + true, &_got, &err)) { + if (signal_pending(current)) { + ret = -ERESTARTSYS; + break; + } + wait_woken(&wait, TASK_INTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT); + } + + remove_wait_queue(&ci->i_cap_wq, &wait); + if (err == -EAGAIN) continue; if (err < 0) diff --git a/fs/ceph/dir.c b/fs/ceph/dir.c index a594c7879cc245..1afa111910007f 100644 --- a/fs/ceph/dir.c +++ b/fs/ceph/dir.c @@ -1255,7 +1255,8 @@ static int ceph_d_revalidate(struct dentry *dentry, unsigned int flags) struct ceph_mds_client *mdsc = ceph_sb_to_client(dir->i_sb)->mdsc; struct ceph_mds_request *req; - int op, mask, err; + int op, err; + u32 mask; if (flags & LOOKUP_RCU) return -ECHILD; @@ -1270,7 +1271,7 @@ static int ceph_d_revalidate(struct dentry *dentry, unsigned int flags) mask = CEPH_STAT_CAP_INODE | CEPH_CAP_AUTH_SHARED; if (ceph_security_xattr_wanted(dir)) mask |= CEPH_CAP_XATTR_SHARED; - req->r_args.getattr.mask = mask; + req->r_args.getattr.mask = cpu_to_le32(mask); err = ceph_mdsc_do_request(mdsc, NULL, req); switch (err) { diff --git a/fs/ceph/inode.c b/fs/ceph/inode.c index ef4d0464732560..12f2252f6c9803 100644 --- a/fs/ceph/inode.c +++ b/fs/ceph/inode.c @@ -305,7 +305,8 @@ static int frag_tree_split_cmp(const void *l, const void *r) { struct ceph_frag_tree_split *ls = (struct ceph_frag_tree_split*)l; struct ceph_frag_tree_split *rs = (struct ceph_frag_tree_split*)r; - return ceph_frag_compare(ls->frag, rs->frag); + return ceph_frag_compare(le32_to_cpu(ls->frag), + le32_to_cpu(rs->frag)); } static bool is_frag_child(u32 f, struct ceph_inode_frag *frag) diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c index 815acd1a56d461..6a26c7bd1286d8 100644 --- a/fs/ceph/mds_client.c +++ b/fs/ceph/mds_client.c @@ -288,12 +288,13 @@ static int parse_reply_info_extra(void **p, void *end, struct ceph_mds_reply_info_parsed *info, u64 features) { - if (info->head->op == CEPH_MDS_OP_GETFILELOCK) + u32 op = le32_to_cpu(info->head->op); + + if (op == CEPH_MDS_OP_GETFILELOCK) return parse_reply_info_filelock(p, end, info, features); - else if (info->head->op == CEPH_MDS_OP_READDIR || - info->head->op == CEPH_MDS_OP_LSSNAP) + else if (op == CEPH_MDS_OP_READDIR || op == CEPH_MDS_OP_LSSNAP) return parse_reply_info_dir(p, end, info, features); - else if (info->head->op == CEPH_MDS_OP_CREATE) + else if (op == CEPH_MDS_OP_CREATE) return parse_reply_info_create(p, end, info, features); else return -EIO; diff --git a/fs/cifs/cifsglob.h b/fs/cifs/cifsglob.h index 1f17f6bd7a601c..203287f8652545 100644 --- a/fs/cifs/cifsglob.h +++ b/fs/cifs/cifsglob.h @@ -646,6 +646,8 @@ struct TCP_Server_Info { unsigned int max_read; unsigned int max_write; __u8 preauth_hash[512]; + struct delayed_work reconnect; /* reconnect workqueue job */ + struct mutex reconnect_mutex; /* prevent simultaneous reconnects */ #endif /* CONFIG_CIFS_SMB2 */ unsigned long echo_interval; }; @@ -849,6 +851,7 @@ cap_unix(struct cifs_ses *ses) struct cifs_tcon { struct list_head tcon_list; int tc_count; + struct list_head rlist; /* reconnect list */ struct list_head openFileList; spinlock_t open_file_lock; /* protects list above */ struct cifs_ses *ses; /* pointer to session associated with */ @@ -922,6 +925,7 @@ struct cifs_tcon { bool broken_posix_open; /* e.g. Samba server versions < 3.3.2, 3.2.9 */ bool broken_sparse_sup; /* if server or share does not support sparse */ bool need_reconnect:1; /* connection reset, tid now invalid */ + bool need_reopen_files:1; /* need to reopen tcon file handles */ bool use_resilient:1; /* use resilient instead of durable handles */ bool use_persistent:1; /* use persistent instead of durable handles */ #ifdef CONFIG_CIFS_SMB2 diff --git a/fs/cifs/cifsproto.h b/fs/cifs/cifsproto.h index ced0e42ce46096..cd8025a249bba9 100644 --- a/fs/cifs/cifsproto.h +++ b/fs/cifs/cifsproto.h @@ -206,6 +206,9 @@ extern void cifs_add_pending_open_locked(struct cifs_fid *fid, struct tcon_link *tlink, struct cifs_pending_open *open); extern void cifs_del_pending_open(struct cifs_pending_open *open); +extern void cifs_put_tcp_session(struct TCP_Server_Info *server, + int from_reconnect); +extern void cifs_put_tcon(struct cifs_tcon *tcon); #if IS_ENABLED(CONFIG_CIFS_DFS_UPCALL) extern void cifs_dfs_release_automount_timer(void); diff --git a/fs/cifs/connect.c b/fs/cifs/connect.c index 4547aeddd12b19..893be072264303 100644 --- a/fs/cifs/connect.c +++ b/fs/cifs/connect.c @@ -52,6 +52,9 @@ #include "nterr.h" #include "rfc1002pdu.h" #include "fscache.h" +#ifdef CONFIG_CIFS_SMB2 +#include "smb2proto.h" +#endif #define CIFS_PORT 445 #define RFC1001_PORT 139 @@ -2100,8 +2103,8 @@ cifs_find_tcp_session(struct smb_vol *vol) return NULL; } -static void -cifs_put_tcp_session(struct TCP_Server_Info *server) +void +cifs_put_tcp_session(struct TCP_Server_Info *server, int from_reconnect) { struct task_struct *task; @@ -2118,6 +2121,19 @@ cifs_put_tcp_session(struct TCP_Server_Info *server) cancel_delayed_work_sync(&server->echo); +#ifdef CONFIG_CIFS_SMB2 + if (from_reconnect) + /* + * Avoid deadlock here: reconnect work calls + * cifs_put_tcp_session() at its end. Need to be sure + * that reconnect work does nothing with server pointer after + * that step. + */ + cancel_delayed_work(&server->reconnect); + else + cancel_delayed_work_sync(&server->reconnect); +#endif + spin_lock(&GlobalMid_Lock); server->tcpStatus = CifsExiting; spin_unlock(&GlobalMid_Lock); @@ -2182,6 +2198,10 @@ cifs_get_tcp_session(struct smb_vol *volume_info) INIT_LIST_HEAD(&tcp_ses->tcp_ses_list); INIT_LIST_HEAD(&tcp_ses->smb_ses_list); INIT_DELAYED_WORK(&tcp_ses->echo, cifs_echo_request); +#ifdef CONFIG_CIFS_SMB2 + INIT_DELAYED_WORK(&tcp_ses->reconnect, smb2_reconnect_server); + mutex_init(&tcp_ses->reconnect_mutex); +#endif memcpy(&tcp_ses->srcaddr, &volume_info->srcaddr, sizeof(tcp_ses->srcaddr)); memcpy(&tcp_ses->dstaddr, &volume_info->dstaddr, @@ -2340,7 +2360,7 @@ cifs_put_smb_ses(struct cifs_ses *ses) spin_unlock(&cifs_tcp_ses_lock); sesInfoFree(ses); - cifs_put_tcp_session(server); + cifs_put_tcp_session(server, 0); } #ifdef CONFIG_KEYS @@ -2514,7 +2534,7 @@ cifs_get_smb_ses(struct TCP_Server_Info *server, struct smb_vol *volume_info) mutex_unlock(&ses->session_mutex); /* existing SMB ses has a server reference already */ - cifs_put_tcp_session(server); + cifs_put_tcp_session(server, 0); free_xid(xid); return ses; } @@ -2604,7 +2624,7 @@ cifs_find_tcon(struct cifs_ses *ses, const char *unc) return NULL; } -static void +void cifs_put_tcon(struct cifs_tcon *tcon) { unsigned int xid; @@ -3792,7 +3812,7 @@ cifs_mount(struct cifs_sb_info *cifs_sb, struct smb_vol *volume_info) else if (ses) cifs_put_smb_ses(ses); else - cifs_put_tcp_session(server); + cifs_put_tcp_session(server, 0); bdi_destroy(&cifs_sb->bdi); } @@ -4103,7 +4123,7 @@ cifs_construct_tcon(struct cifs_sb_info *cifs_sb, kuid_t fsuid) ses = cifs_get_smb_ses(master_tcon->ses->server, vol_info); if (IS_ERR(ses)) { tcon = (struct cifs_tcon *)ses; - cifs_put_tcp_session(master_tcon->ses->server); + cifs_put_tcp_session(master_tcon->ses->server, 0); goto out; } diff --git a/fs/cifs/file.c b/fs/cifs/file.c index 7f5f6176c6f15c..18a1e1d6671fc0 100644 --- a/fs/cifs/file.c +++ b/fs/cifs/file.c @@ -777,6 +777,11 @@ cifs_reopen_persistent_handles(struct cifs_tcon *tcon) struct list_head *tmp1; struct list_head tmp_list; + if (!tcon->use_persistent || !tcon->need_reopen_files) + return; + + tcon->need_reopen_files = false; + cifs_dbg(FYI, "Reopen persistent handles"); INIT_LIST_HEAD(&tmp_list); @@ -793,7 +798,8 @@ cifs_reopen_persistent_handles(struct cifs_tcon *tcon) list_for_each_safe(tmp, tmp1, &tmp_list) { open_file = list_entry(tmp, struct cifsFileInfo, rlist); - cifs_reopen_file(open_file, false /* do not flush */); + if (cifs_reopen_file(open_file, false /* do not flush */)) + tcon->need_reopen_files = true; list_del_init(&open_file->rlist); cifsFileInfo_put(open_file); } diff --git a/fs/cifs/ioctl.c b/fs/cifs/ioctl.c index 9f51b81119f2b2..001528781b6b0f 100644 --- a/fs/cifs/ioctl.c +++ b/fs/cifs/ioctl.c @@ -189,7 +189,7 @@ long cifs_ioctl(struct file *filep, unsigned int command, unsigned long arg) xid = get_xid(); cifs_sb = CIFS_SB(inode->i_sb); - cifs_dbg(VFS, "cifs ioctl 0x%x\n", command); + cifs_dbg(FYI, "cifs ioctl 0x%x\n", command); switch (command) { case FS_IOC_GETFLAGS: if (pSMBFile == NULL) diff --git a/fs/cifs/readdir.c b/fs/cifs/readdir.c index 8f6a2a5863b9d9..a27fc8791551cc 100644 --- a/fs/cifs/readdir.c +++ b/fs/cifs/readdir.c @@ -285,6 +285,7 @@ initiate_cifs_search(const unsigned int xid, struct file *file) rc = -ENOMEM; goto error_exit; } + spin_lock_init(&cifsFile->file_info_lock); file->private_data = cifsFile; cifsFile->tlink = cifs_get_tlink(tlink); tcon = tlink_tcon(tlink); diff --git a/fs/cifs/smb2file.c b/fs/cifs/smb2file.c index f9e766f464beec..b2aff0c6f22c52 100644 --- a/fs/cifs/smb2file.c +++ b/fs/cifs/smb2file.c @@ -260,7 +260,7 @@ smb2_push_mandatory_locks(struct cifsFileInfo *cfile) * and check it for zero before using. */ max_buf = tlink_tcon(cfile->tlink)->ses->server->maxBuf; - if (!max_buf) { + if (max_buf < sizeof(struct smb2_lock_element)) { free_xid(xid); return -EINVAL; } diff --git a/fs/cifs/smb2pdu.c b/fs/cifs/smb2pdu.c index 5ca5ea4668a148..87457227812c39 100644 --- a/fs/cifs/smb2pdu.c +++ b/fs/cifs/smb2pdu.c @@ -250,16 +250,19 @@ smb2_reconnect(__le16 smb2_command, struct cifs_tcon *tcon) } cifs_mark_open_files_invalid(tcon); + if (tcon->use_persistent) + tcon->need_reopen_files = true; rc = SMB2_tcon(0, tcon->ses, tcon->treeName, tcon, nls_codepage); mutex_unlock(&tcon->ses->session_mutex); - if (tcon->use_persistent) - cifs_reopen_persistent_handles(tcon); - cifs_dbg(FYI, "reconnect tcon rc = %d\n", rc); if (rc) goto out; + + if (smb2_command != SMB2_INTERNAL_CMD) + queue_delayed_work(cifsiod_wq, &server->reconnect, 0); + atomic_inc(&tconInfoReconnectCount); out: /* @@ -280,7 +283,7 @@ smb2_reconnect(__le16 smb2_command, struct cifs_tcon *tcon) case SMB2_CHANGE_NOTIFY: case SMB2_QUERY_INFO: case SMB2_SET_INFO: - return -EAGAIN; + rc = -EAGAIN; } unload_nls(nls_codepage); return rc; @@ -1972,6 +1975,55 @@ smb2_echo_callback(struct mid_q_entry *mid) add_credits(server, credits_received, CIFS_ECHO_OP); } +void smb2_reconnect_server(struct work_struct *work) +{ + struct TCP_Server_Info *server = container_of(work, + struct TCP_Server_Info, reconnect.work); + struct cifs_ses *ses; + struct cifs_tcon *tcon, *tcon2; + struct list_head tmp_list; + int tcon_exist = false; + + /* Prevent simultaneous reconnects that can corrupt tcon->rlist list */ + mutex_lock(&server->reconnect_mutex); + + INIT_LIST_HEAD(&tmp_list); + cifs_dbg(FYI, "Need negotiate, reconnecting tcons\n"); + + spin_lock(&cifs_tcp_ses_lock); + list_for_each_entry(ses, &server->smb_ses_list, smb_ses_list) { + list_for_each_entry(tcon, &ses->tcon_list, tcon_list) { + if (tcon->need_reconnect || tcon->need_reopen_files) { + tcon->tc_count++; + list_add_tail(&tcon->rlist, &tmp_list); + tcon_exist = true; + } + } + } + /* + * Get the reference to server struct to be sure that the last call of + * cifs_put_tcon() in the loop below won't release the server pointer. + */ + if (tcon_exist) + server->srv_count++; + + spin_unlock(&cifs_tcp_ses_lock); + + list_for_each_entry_safe(tcon, tcon2, &tmp_list, rlist) { + if (!smb2_reconnect(SMB2_INTERNAL_CMD, tcon)) + cifs_reopen_persistent_handles(tcon); + list_del_init(&tcon->rlist); + cifs_put_tcon(tcon); + } + + cifs_dbg(FYI, "Reconnecting tcons finished\n"); + mutex_unlock(&server->reconnect_mutex); + + /* now we can safely release srv struct */ + if (tcon_exist) + cifs_put_tcp_session(server, 1); +} + int SMB2_echo(struct TCP_Server_Info *server) { @@ -1984,32 +2036,11 @@ SMB2_echo(struct TCP_Server_Info *server) cifs_dbg(FYI, "In echo request\n"); if (server->tcpStatus == CifsNeedNegotiate) { - struct list_head *tmp, *tmp2; - struct cifs_ses *ses; - struct cifs_tcon *tcon; - - cifs_dbg(FYI, "Need negotiate, reconnecting tcons\n"); - spin_lock(&cifs_tcp_ses_lock); - list_for_each(tmp, &server->smb_ses_list) { - ses = list_entry(tmp, struct cifs_ses, smb_ses_list); - list_for_each(tmp2, &ses->tcon_list) { - tcon = list_entry(tmp2, struct cifs_tcon, - tcon_list); - /* add check for persistent handle reconnect */ - if (tcon && tcon->need_reconnect) { - spin_unlock(&cifs_tcp_ses_lock); - rc = smb2_reconnect(SMB2_ECHO, tcon); - spin_lock(&cifs_tcp_ses_lock); - } - } - } - spin_unlock(&cifs_tcp_ses_lock); + /* No need to send echo on newly established connections */ + queue_delayed_work(cifsiod_wq, &server->reconnect, 0); + return rc; } - /* if no session, renegotiate failed above */ - if (server->tcpStatus == CifsNeedNegotiate) - return -EIO; - rc = small_smb2_init(SMB2_ECHO, NULL, (void **)&req); if (rc) return rc; diff --git a/fs/cifs/smb2pdu.h b/fs/cifs/smb2pdu.h index fd3709e8de33ed..dc0d141f33e257 100644 --- a/fs/cifs/smb2pdu.h +++ b/fs/cifs/smb2pdu.h @@ -80,6 +80,8 @@ #define SMB2_SET_INFO cpu_to_le16(SMB2_SET_INFO_HE) #define SMB2_OPLOCK_BREAK cpu_to_le16(SMB2_OPLOCK_BREAK_HE) +#define SMB2_INTERNAL_CMD cpu_to_le16(0xFFFF) + #define NUMBER_OF_SMB2_COMMANDS 0x0013 /* BB FIXME - analyze following length BB */ diff --git a/fs/cifs/smb2proto.h b/fs/cifs/smb2proto.h index eb2cde2f64ba7c..f2d511a6971b88 100644 --- a/fs/cifs/smb2proto.h +++ b/fs/cifs/smb2proto.h @@ -96,6 +96,7 @@ extern int smb2_open_file(const unsigned int xid, extern int smb2_unlock_range(struct cifsFileInfo *cfile, struct file_lock *flock, const unsigned int xid); extern int smb2_push_mandatory_locks(struct cifsFileInfo *cfile); +extern void smb2_reconnect_server(struct work_struct *work); /* * SMB2 Worker functions - most of protocol specific implementation details diff --git a/fs/cifs/smbencrypt.c b/fs/cifs/smbencrypt.c index 699b7868108f65..c12bffefa3c9b5 100644 --- a/fs/cifs/smbencrypt.c +++ b/fs/cifs/smbencrypt.c @@ -23,7 +23,7 @@ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ -#include +#include #include #include #include @@ -69,46 +69,22 @@ str_to_key(unsigned char *str, unsigned char *key) static int smbhash(unsigned char *out, const unsigned char *in, unsigned char *key) { - int rc; unsigned char key2[8]; - struct crypto_skcipher *tfm_des; - struct scatterlist sgin, sgout; - struct skcipher_request *req; + struct crypto_cipher *tfm_des; str_to_key(key, key2); - tfm_des = crypto_alloc_skcipher("ecb(des)", 0, CRYPTO_ALG_ASYNC); + tfm_des = crypto_alloc_cipher("des", 0, 0); if (IS_ERR(tfm_des)) { - rc = PTR_ERR(tfm_des); - cifs_dbg(VFS, "could not allocate des crypto API\n"); - goto smbhash_err; - } - - req = skcipher_request_alloc(tfm_des, GFP_KERNEL); - if (!req) { - rc = -ENOMEM; cifs_dbg(VFS, "could not allocate des crypto API\n"); - goto smbhash_free_skcipher; + return PTR_ERR(tfm_des); } - crypto_skcipher_setkey(tfm_des, key2, 8); - - sg_init_one(&sgin, in, 8); - sg_init_one(&sgout, out, 8); + crypto_cipher_setkey(tfm_des, key2, 8); + crypto_cipher_encrypt_one(tfm_des, out, in); + crypto_free_cipher(tfm_des); - skcipher_request_set_callback(req, 0, NULL, NULL); - skcipher_request_set_crypt(req, &sgin, &sgout, 8, NULL); - - rc = crypto_skcipher_encrypt(req); - if (rc) - cifs_dbg(VFS, "could not encrypt crypt key rc: %d\n", rc); - - skcipher_request_free(req); - -smbhash_free_skcipher: - crypto_free_skcipher(tfm_des); -smbhash_err: - return rc; + return 0; } static int diff --git a/fs/coredump.c b/fs/coredump.c index eb9c92c9b20f5d..8bdda8e660d807 100644 --- a/fs/coredump.c +++ b/fs/coredump.c @@ -744,7 +744,7 @@ void do_coredump(const siginfo_t *siginfo) goto close_fail; if (!(cprm.file->f_mode & FMODE_CAN_WRITE)) goto close_fail; - if (do_truncate(cprm.file->f_path.dentry, 0, 0, cprm.file)) + if (do_truncate2(cprm.file->f_path.mnt, cprm.file->f_path.dentry, 0, 0, cprm.file)) goto close_fail; } diff --git a/fs/crypto/policy.c b/fs/crypto/policy.c index 6865663aac6969..abc18847b98d77 100644 --- a/fs/crypto/policy.c +++ b/fs/crypto/policy.c @@ -171,6 +171,11 @@ int fscrypt_has_permitted_context(struct inode *parent, struct inode *child) BUG_ON(1); } + /* No restrictions on file types which are never encrypted */ + if (!S_ISREG(child->i_mode) && !S_ISDIR(child->i_mode) && + !S_ISLNK(child->i_mode)) + return 1; + /* no restrictions if the parent directory is not encrypted */ if (!parent->i_sb->s_cop->is_encrypted(parent)) return 1; diff --git a/fs/dax.c b/fs/dax.c index 014defd2e744e0..bf6218da79287e 100644 --- a/fs/dax.c +++ b/fs/dax.c @@ -1270,6 +1270,11 @@ iomap_dax_actor(struct inode *inode, loff_t pos, loff_t length, void *data, struct blk_dax_ctl dax = { 0 }; ssize_t map_len; + if (fatal_signal_pending(current)) { + ret = -EINTR; + break; + } + dax.sector = iomap->blkno + (((pos & PAGE_MASK) - iomap->offset) >> 9); dax.size = (length + offset + PAGE_SIZE - 1) & PAGE_MASK; diff --git a/fs/dcache.c b/fs/dcache.c index 5c7cc953ac8191..362396ae5650ea 100644 --- a/fs/dcache.c +++ b/fs/dcache.c @@ -1330,8 +1330,11 @@ int d_set_mounted(struct dentry *dentry) } spin_lock(&dentry->d_lock); if (!d_unlinked(dentry)) { - dentry->d_flags |= DCACHE_MOUNTED; - ret = 0; + ret = -EBUSY; + if (!d_mountpoint(dentry)) { + dentry->d_flags |= DCACHE_MOUNTED; + ret = 0; + } } spin_unlock(&dentry->d_lock); out: @@ -3184,6 +3187,7 @@ char *d_absolute_path(const struct path *path, return ERR_PTR(error); return res; } +EXPORT_SYMBOL(d_absolute_path); /* * same as __d_path but appends "(deleted)" for unlinked files. diff --git a/fs/eventpoll.c b/fs/eventpoll.c index 10db9121893380..dc4a34f9c859c6 100644 --- a/fs/eventpoll.c +++ b/fs/eventpoll.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -1663,7 +1664,8 @@ static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events, } spin_unlock_irqrestore(&ep->lock, flags); - if (!schedule_hrtimeout_range(to, slack, HRTIMER_MODE_ABS)) + if (!freezable_schedule_hrtimeout_range(to, slack, + HRTIMER_MODE_ABS)) timed_out = 1; spin_lock_irqsave(&ep->lock, flags); diff --git a/fs/exec.c b/fs/exec.c index 4e497b9ee71ee9..c8ca0640a10ca6 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -19,7 +19,7 @@ * current->executable is only used by the procfs. This allows a dispatch * table to check for several different types of binary formats. We keep * trying until we recognize the file or we run out of supported binary - * formats. + * formats. */ #include @@ -1266,6 +1266,13 @@ int flush_old_exec(struct linux_binprm * bprm) flush_thread(); current->personality &= ~bprm->per_clear; + /* + * We have to apply CLOEXEC before we change whether the process is + * dumpable (in setup_new_exec) to avoid a race with a process in userspace + * trying to access the should-be-closed file descriptors of a process + * undergoing exec(2). + */ + do_close_on_exec(current->files); return 0; out: @@ -1275,8 +1282,22 @@ EXPORT_SYMBOL(flush_old_exec); void would_dump(struct linux_binprm *bprm, struct file *file) { - if (inode_permission(file_inode(file), MAY_READ) < 0) + struct inode *inode = file_inode(file); + if (inode_permission2(file->f_path.mnt, inode, MAY_READ) < 0) { + struct user_namespace *old, *user_ns; bprm->interp_flags |= BINPRM_FLAGS_ENFORCE_NONDUMP; + + /* Ensure mm->user_ns contains the executable */ + user_ns = old = bprm->mm->user_ns; + while ((user_ns != &init_user_ns) && + !privileged_wrt_inode_uidgid(user_ns, inode)) + user_ns = user_ns->parent; + + if (old != user_ns) { + bprm->mm->user_ns = get_user_ns(user_ns); + put_user_ns(old); + } + } } EXPORT_SYMBOL(would_dump); @@ -1306,7 +1327,6 @@ void setup_new_exec(struct linux_binprm * bprm) !gid_eq(bprm->cred->gid, current_egid())) { current->pdeath_signal = 0; } else { - would_dump(bprm, bprm->file); if (bprm->interp_flags & BINPRM_FLAGS_ENFORCE_NONDUMP) set_dumpable(current->mm, suid_dumpable); } @@ -1315,7 +1335,6 @@ void setup_new_exec(struct linux_binprm * bprm) group */ current->self_exec_id++; flush_signal_handlers(current, 0); - do_close_on_exec(current->files); } EXPORT_SYMBOL(setup_new_exec); @@ -1406,7 +1425,7 @@ static void check_unsafe_exec(struct linux_binprm *bprm) unsigned n_fs; if (p->ptrace) { - if (p->ptrace & PT_PTRACE_CAP) + if (ptracer_capable(p, current_user_ns())) bprm->unsafe |= LSM_UNSAFE_PTRACE_CAP; else bprm->unsafe |= LSM_UNSAFE_PTRACE; @@ -1741,6 +1760,8 @@ static int do_execveat_common(int fd, struct filename *filename, if (retval < 0) goto out; + would_dump(bprm, bprm->file); + retval = exec_binprm(bprm); if (retval < 0) goto out; diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h index a8a750f5962170..20ee0e4a5829d6 100644 --- a/fs/ext4/ext4.h +++ b/fs/ext4/ext4.h @@ -2450,7 +2450,8 @@ extern int ext4_mb_add_groupinfo(struct super_block *sb, ext4_group_t i, struct ext4_group_desc *desc); extern int ext4_group_add_blocks(handle_t *handle, struct super_block *sb, ext4_fsblk_t block, unsigned long count); -extern int ext4_trim_fs(struct super_block *, struct fstrim_range *); +extern int ext4_trim_fs(struct super_block *, struct fstrim_range *, + unsigned long blkdev_flags); /* inode.c */ int ext4_inode_is_fast_symlink(struct inode *inode); diff --git a/fs/ext4/ext4_jbd2.h b/fs/ext4/ext4_jbd2.h index b1d52c14098e13..f976111710231d 100644 --- a/fs/ext4/ext4_jbd2.h +++ b/fs/ext4/ext4_jbd2.h @@ -414,17 +414,19 @@ static inline int ext4_inode_journal_mode(struct inode *inode) return EXT4_INODE_WRITEBACK_DATA_MODE; /* writeback */ /* We do not support data journalling with delayed allocation */ if (!S_ISREG(inode->i_mode) || - test_opt(inode->i_sb, DATA_FLAGS) == EXT4_MOUNT_JOURNAL_DATA) - return EXT4_INODE_JOURNAL_DATA_MODE; /* journal data */ - if (ext4_test_inode_flag(inode, EXT4_INODE_JOURNAL_DATA) && - !test_opt(inode->i_sb, DELALLOC)) + test_opt(inode->i_sb, DATA_FLAGS) == EXT4_MOUNT_JOURNAL_DATA || + (ext4_test_inode_flag(inode, EXT4_INODE_JOURNAL_DATA) && + !test_opt(inode->i_sb, DELALLOC))) { + /* We do not support data journalling for encrypted data */ + if (S_ISREG(inode->i_mode) && ext4_encrypted_inode(inode)) + return EXT4_INODE_ORDERED_DATA_MODE; /* ordered */ return EXT4_INODE_JOURNAL_DATA_MODE; /* journal data */ + } if (test_opt(inode->i_sb, DATA_FLAGS) == EXT4_MOUNT_ORDERED_DATA) return EXT4_INODE_ORDERED_DATA_MODE; /* ordered */ if (test_opt(inode->i_sb, DATA_FLAGS) == EXT4_MOUNT_WRITEBACK_DATA) return EXT4_INODE_WRITEBACK_DATA_MODE; /* writeback */ - else - BUG(); + BUG(); } static inline int ext4_should_journal_data(struct inode *inode) diff --git a/fs/ext4/inline.c b/fs/ext4/inline.c index f74d5ee2cdec09..dbef345ab17f76 100644 --- a/fs/ext4/inline.c +++ b/fs/ext4/inline.c @@ -18,6 +18,7 @@ #include "ext4.h" #include "xattr.h" #include "truncate.h" +#include #define EXT4_XATTR_SYSTEM_DATA "data" #define EXT4_MIN_INLINE_DATA_SIZE ((sizeof(__le32) * EXT4_N_BLOCKS)) @@ -336,8 +337,10 @@ static int ext4_update_inline_data(handle_t *handle, struct inode *inode, len -= EXT4_MIN_INLINE_DATA_SIZE; value = kzalloc(len, GFP_NOFS); - if (!value) + if (!value) { + error = -ENOMEM; goto out; + } error = ext4_xattr_ibody_get(inode, i.name_index, i.name, value, len); @@ -500,6 +503,17 @@ int ext4_readpage_inline(struct inode *inode, struct page *page) return -EAGAIN; } + if (trace_android_fs_dataread_start_enabled()) { + char *path, pathbuf[MAX_TRACE_PATHBUF_LEN]; + + path = android_fstrace_get_pathname(pathbuf, + MAX_TRACE_PATHBUF_LEN, + inode); + trace_android_fs_dataread_start(inode, page_offset(page), + PAGE_SIZE, current->pid, + path, current->comm); + } + /* * Current inline data can only exist in the 1st page, * So for all the other pages, just set them uptodate. @@ -511,6 +525,8 @@ int ext4_readpage_inline(struct inode *inode, struct page *page) SetPageUptodate(page); } + trace_android_fs_dataread_end(inode, page_offset(page), PAGE_SIZE); + up_read(&EXT4_I(inode)->xattr_sem); unlock_page(page); diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index 9c064727ed6297..46912da95a3b83 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -44,6 +44,7 @@ #include "truncate.h" #include +#include #define MPAGE_DA_EXTENT_TAIL 0x01 @@ -1183,6 +1184,16 @@ static int ext4_write_begin(struct file *file, struct address_space *mapping, pgoff_t index; unsigned from, to; + if (trace_android_fs_datawrite_start_enabled()) { + char *path, pathbuf[MAX_TRACE_PATHBUF_LEN]; + + path = android_fstrace_get_pathname(pathbuf, + MAX_TRACE_PATHBUF_LEN, + inode); + trace_android_fs_datawrite_start(inode, pos, len, + current->pid, path, + current->comm); + } trace_ext4_write_begin(inode, pos, len, flags); /* * Reserve one block more for addition to orphan list in case @@ -1320,6 +1331,7 @@ static int ext4_write_end(struct file *file, int ret = 0, ret2; int i_size_changed = 0; + trace_android_fs_datawrite_end(inode, pos, len); trace_ext4_write_end(inode, pos, len, copied); if (ext4_has_inline_data(inode)) { ret = ext4_write_inline_data_end(inode, pos, len, @@ -1419,6 +1431,7 @@ static int ext4_journalled_write_end(struct file *file, unsigned from, to; int size_changed = 0; + trace_android_fs_datawrite_end(inode, pos, len); trace_ext4_journalled_write_end(inode, pos, len, copied); from = pos & (PAGE_SIZE - 1); to = from + len; @@ -2897,6 +2910,16 @@ static int ext4_da_write_begin(struct file *file, struct address_space *mapping, len, flags, pagep, fsdata); } *fsdata = (void *)0; + if (trace_android_fs_datawrite_start_enabled()) { + char *path, pathbuf[MAX_TRACE_PATHBUF_LEN]; + + path = android_fstrace_get_pathname(pathbuf, + MAX_TRACE_PATHBUF_LEN, + inode); + trace_android_fs_datawrite_start(inode, pos, len, + current->pid, + path, current->comm); + } trace_ext4_da_write_begin(inode, pos, len, flags); if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) { @@ -3015,6 +3038,7 @@ static int ext4_da_write_end(struct file *file, return ext4_write_end(file, mapping, pos, len, copied, page, fsdata); + trace_android_fs_datawrite_end(inode, pos, len); trace_ext4_da_write_end(inode, pos, len, copied); start = pos & (PAGE_SIZE - 1); end = start + copied - 1; @@ -3571,6 +3595,7 @@ static ssize_t ext4_direct_IO(struct kiocb *iocb, struct iov_iter *iter) size_t count = iov_iter_count(iter); loff_t offset = iocb->ki_pos; ssize_t ret; + int rw = iov_iter_rw(iter); #ifdef CONFIG_EXT4_FS_ENCRYPTION if (ext4_encrypted_inode(inode) && S_ISREG(inode->i_mode)) @@ -3587,12 +3612,42 @@ static ssize_t ext4_direct_IO(struct kiocb *iocb, struct iov_iter *iter) if (ext4_has_inline_data(inode)) return 0; + if (trace_android_fs_dataread_start_enabled() && + (rw == READ)) { + char *path, pathbuf[MAX_TRACE_PATHBUF_LEN]; + + path = android_fstrace_get_pathname(pathbuf, + MAX_TRACE_PATHBUF_LEN, + inode); + trace_android_fs_dataread_start(inode, offset, count, + current->pid, path, + current->comm); + } + if (trace_android_fs_datawrite_start_enabled() && + (rw == WRITE)) { + char *path, pathbuf[MAX_TRACE_PATHBUF_LEN]; + + path = android_fstrace_get_pathname(pathbuf, + MAX_TRACE_PATHBUF_LEN, + inode); + trace_android_fs_datawrite_start(inode, offset, count, + current->pid, path, + current->comm); + } trace_ext4_direct_IO_enter(inode, offset, count, iov_iter_rw(iter)); if (iov_iter_rw(iter) == READ) ret = ext4_direct_IO_read(iocb, iter); else ret = ext4_direct_IO_write(iocb, iter); trace_ext4_direct_IO_exit(inode, offset, count, iov_iter_rw(iter), ret); + + if (trace_android_fs_dataread_start_enabled() && + (rw == READ)) + trace_android_fs_dataread_end(inode, offset, count); + if (trace_android_fs_datawrite_start_enabled() && + (rw == WRITE)) + trace_android_fs_datawrite_end(inode, offset, count); + return ret; } @@ -3811,6 +3866,11 @@ static int ext4_block_truncate_page(handle_t *handle, unsigned blocksize; struct inode *inode = mapping->host; + /* If we are processing an encrypted inode during orphan list + * handling */ + if (ext4_encrypted_inode(inode) && !fscrypt_has_encryption_key(inode)) + return 0; + blocksize = inode->i_sb->s_blocksize; length = blocksize - (offset & (blocksize - 1)); @@ -4434,6 +4494,7 @@ struct inode *ext4_iget(struct super_block *sb, unsigned long ino) struct inode *inode; journal_t *journal = EXT4_SB(sb)->s_journal; long ret; + loff_t size; int block; uid_t i_uid; gid_t i_gid; @@ -4534,6 +4595,11 @@ struct inode *ext4_iget(struct super_block *sb, unsigned long ino) ei->i_file_acl |= ((__u64)le16_to_cpu(raw_inode->i_file_acl_high)) << 32; inode->i_size = ext4_isize(raw_inode); + if ((size = i_size_read(inode)) < 0) { + EXT4_ERROR_INODE(inode, "bad i_size value: %lld", size); + ret = -EFSCORRUPTED; + goto bad_inode; + } ei->i_disksize = inode->i_size; #ifdef CONFIG_QUOTA ei->i_reserved_quota = 0; diff --git a/fs/ext4/ioctl.c b/fs/ext4/ioctl.c index bf5ae8ebbc9788..cec9280950b899 100644 --- a/fs/ext4/ioctl.c +++ b/fs/ext4/ioctl.c @@ -735,11 +735,13 @@ long ext4_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) return err; } + case FIDTRIM: case FITRIM: { struct request_queue *q = bdev_get_queue(sb->s_bdev); struct fstrim_range range; int ret = 0; + int flags = cmd == FIDTRIM ? BLKDEV_DISCARD_SECURE : 0; if (!capable(CAP_SYS_ADMIN)) return -EPERM; @@ -747,13 +749,16 @@ long ext4_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) if (!blk_queue_discard(q)) return -EOPNOTSUPP; + if ((flags & BLKDEV_DISCARD_SECURE) && !blk_queue_secure_erase(q)) + return -EOPNOTSUPP; + if (copy_from_user(&range, (struct fstrim_range __user *)arg, sizeof(range))) return -EFAULT; range.minlen = max((unsigned int)range.minlen, q->limits.discard_granularity); - ret = ext4_trim_fs(sb, &range); + ret = ext4_trim_fs(sb, &range, flags); if (ret < 0) return ret; diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c index f418f55c2bbe1b..168ab9fae22053 100644 --- a/fs/ext4/mballoc.c +++ b/fs/ext4/mballoc.c @@ -669,7 +669,7 @@ static void ext4_mb_mark_free_simple(struct super_block *sb, ext4_grpblk_t min; ext4_grpblk_t max; ext4_grpblk_t chunk; - unsigned short border; + unsigned int border; BUG_ON(len > EXT4_CLUSTERS_PER_GROUP(sb)); @@ -2287,7 +2287,7 @@ static int ext4_mb_seq_groups_show(struct seq_file *seq, void *v) struct ext4_group_info *grinfo; struct sg { struct ext4_group_info info; - ext4_grpblk_t counters[16]; + ext4_grpblk_t counters[EXT4_MAX_BLOCK_LOG_SIZE + 2]; } sg; group--; @@ -2770,7 +2770,8 @@ int ext4_mb_release(struct super_block *sb) } static inline int ext4_issue_discard(struct super_block *sb, - ext4_group_t block_group, ext4_grpblk_t cluster, int count) + ext4_group_t block_group, ext4_grpblk_t cluster, int count, + unsigned long flags) { ext4_fsblk_t discard_block; @@ -2779,7 +2780,7 @@ static inline int ext4_issue_discard(struct super_block *sb, count = EXT4_C2B(EXT4_SB(sb), count); trace_ext4_discard_blocks(sb, (unsigned long long) discard_block, count); - return sb_issue_discard(sb, discard_block, count, GFP_NOFS, 0); + return sb_issue_discard(sb, discard_block, count, GFP_NOFS, flags); } /* @@ -2801,7 +2802,7 @@ static void ext4_free_data_callback(struct super_block *sb, if (test_opt(sb, DISCARD)) { err = ext4_issue_discard(sb, entry->efd_group, entry->efd_start_cluster, - entry->efd_count); + entry->efd_count, 0); if (err && err != -EOPNOTSUPP) ext4_msg(sb, KERN_WARNING, "discard request in" " group:%d block:%d count:%d failed" @@ -4847,7 +4848,8 @@ void ext4_free_blocks(handle_t *handle, struct inode *inode, * them with group lock_held */ if (test_opt(sb, DISCARD)) { - err = ext4_issue_discard(sb, block_group, bit, count); + err = ext4_issue_discard(sb, block_group, bit, count, + 0); if (err && err != -EOPNOTSUPP) ext4_msg(sb, KERN_WARNING, "discard request in" " group:%d block:%d count:%lu failed" @@ -5043,13 +5045,15 @@ int ext4_group_add_blocks(handle_t *handle, struct super_block *sb, * @count: number of blocks to TRIM * @group: alloc. group we are working with * @e4b: ext4 buddy for the group + * @blkdev_flags: flags for the block device * * Trim "count" blocks starting at "start" in the "group". To assure that no * one will allocate those blocks, mark it as used in buddy bitmap. This must * be called with under the group lock. */ static int ext4_trim_extent(struct super_block *sb, int start, int count, - ext4_group_t group, struct ext4_buddy *e4b) + ext4_group_t group, struct ext4_buddy *e4b, + unsigned long blkdev_flags) __releases(bitlock) __acquires(bitlock) { @@ -5070,7 +5074,7 @@ __acquires(bitlock) */ mb_mark_used(e4b, &ex); ext4_unlock_group(sb, group); - ret = ext4_issue_discard(sb, group, start, count); + ret = ext4_issue_discard(sb, group, start, count, blkdev_flags); ext4_lock_group(sb, group); mb_free_blocks(NULL, e4b, start, ex.fe_len); return ret; @@ -5083,6 +5087,7 @@ __acquires(bitlock) * @start: first group block to examine * @max: last group block to examine * @minblocks: minimum extent block count + * @blkdev_flags: flags for the block device * * ext4_trim_all_free walks through group's buddy bitmap searching for free * extents. When the free block is found, ext4_trim_extent is called to TRIM @@ -5097,7 +5102,7 @@ __acquires(bitlock) static ext4_grpblk_t ext4_trim_all_free(struct super_block *sb, ext4_group_t group, ext4_grpblk_t start, ext4_grpblk_t max, - ext4_grpblk_t minblocks) + ext4_grpblk_t minblocks, unsigned long blkdev_flags) { void *bitmap; ext4_grpblk_t next, count = 0, free_count = 0; @@ -5130,7 +5135,8 @@ ext4_trim_all_free(struct super_block *sb, ext4_group_t group, if ((next - start) >= minblocks) { ret = ext4_trim_extent(sb, start, - next - start, group, &e4b); + next - start, group, &e4b, + blkdev_flags); if (ret && ret != -EOPNOTSUPP) break; ret = 0; @@ -5172,6 +5178,7 @@ ext4_trim_all_free(struct super_block *sb, ext4_group_t group, * ext4_trim_fs() -- trim ioctl handle function * @sb: superblock for filesystem * @range: fstrim_range structure + * @blkdev_flags: flags for the block device * * start: First Byte to trim * len: number of Bytes to trim from start @@ -5180,7 +5187,8 @@ ext4_trim_all_free(struct super_block *sb, ext4_group_t group, * start to start+len. For each such a group ext4_trim_all_free function * is invoked to trim all free space. */ -int ext4_trim_fs(struct super_block *sb, struct fstrim_range *range) +int ext4_trim_fs(struct super_block *sb, struct fstrim_range *range, + unsigned long blkdev_flags) { struct ext4_group_info *grp; ext4_group_t group, first_group, last_group; @@ -5236,7 +5244,7 @@ int ext4_trim_fs(struct super_block *sb, struct fstrim_range *range) if (grp->bb_free >= minlen) { cnt = ext4_trim_all_free(sb, group, first_cluster, - end, minlen); + end, minlen, blkdev_flags); if (cnt < 0) { ret = cnt; break; diff --git a/fs/ext4/readpage.c b/fs/ext4/readpage.c index a81b829d56def3..2531cc1df4bdcd 100644 --- a/fs/ext4/readpage.c +++ b/fs/ext4/readpage.c @@ -45,6 +45,7 @@ #include #include "ext4.h" +#include static inline bool ext4_bio_encrypted(struct bio *bio) { @@ -55,6 +56,17 @@ static inline bool ext4_bio_encrypted(struct bio *bio) #endif } +static void +ext4_trace_read_completion(struct bio *bio) +{ + struct page *first_page = bio->bi_io_vec[0].bv_page; + + if (first_page != NULL) + trace_android_fs_dataread_end(first_page->mapping->host, + page_offset(first_page), + bio->bi_iter.bi_size); +} + /* * I/O completion handler for multipage BIOs. * @@ -72,6 +84,9 @@ static void mpage_end_io(struct bio *bio) struct bio_vec *bv; int i; + if (trace_android_fs_dataread_start_enabled()) + ext4_trace_read_completion(bio); + if (ext4_bio_encrypted(bio)) { if (bio->bi_error) { fscrypt_release_ctx(bio->bi_private); @@ -95,6 +110,30 @@ static void mpage_end_io(struct bio *bio) bio_put(bio); } +static void +ext4_submit_bio_read(struct bio *bio) +{ + if (trace_android_fs_dataread_start_enabled()) { + struct page *first_page = bio->bi_io_vec[0].bv_page; + + if (first_page != NULL) { + char *path, pathbuf[MAX_TRACE_PATHBUF_LEN]; + + path = android_fstrace_get_pathname(pathbuf, + MAX_TRACE_PATHBUF_LEN, + first_page->mapping->host); + trace_android_fs_dataread_start( + first_page->mapping->host, + page_offset(first_page), + bio->bi_iter.bi_size, + current->pid, + path, + current->comm); + } + } + submit_bio(bio); +} + int ext4_mpage_readpages(struct address_space *mapping, struct list_head *pages, struct page *page, unsigned nr_pages) @@ -235,7 +274,7 @@ int ext4_mpage_readpages(struct address_space *mapping, */ if (bio && (last_block_in_bio != blocks[0] - 1)) { submit_and_realloc: - submit_bio(bio); + ext4_submit_bio_read(bio); bio = NULL; } if (bio == NULL) { @@ -268,14 +307,14 @@ int ext4_mpage_readpages(struct address_space *mapping, if (((map.m_flags & EXT4_MAP_BOUNDARY) && (relative_block == map.m_len)) || (first_hole != blocks_per_page)) { - submit_bio(bio); + ext4_submit_bio_read(bio); bio = NULL; } else last_block_in_bio = blocks[blocks_per_page - 1]; goto next_page; confused: if (bio) { - submit_bio(bio); + ext4_submit_bio_read(bio); bio = NULL; } if (!PageUptodate(page)) @@ -288,6 +327,6 @@ int ext4_mpage_readpages(struct address_space *mapping, } BUG_ON(pages && !list_empty(pages)); if (bio) - submit_bio(bio); + ext4_submit_bio_read(bio); return 0; } diff --git a/fs/ext4/super.c b/fs/ext4/super.c index 52b0530c5d65a9..bbc316db94954a 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -3193,10 +3193,15 @@ static int count_overhead(struct super_block *sb, ext4_group_t grp, ext4_set_bit(s++, buf); count++; } - for (j = ext4_bg_num_gdb(sb, grp); j > 0; j--) { - ext4_set_bit(EXT4_B2C(sbi, s++), buf); - count++; + j = ext4_bg_num_gdb(sb, grp); + if (s + j > EXT4_BLOCKS_PER_GROUP(sb)) { + ext4_error(sb, "Invalid number of block group " + "descriptor blocks: %d", j); + j = EXT4_BLOCKS_PER_GROUP(sb) - s; } + count += j; + for (; j > 0; j--) + ext4_set_bit(EXT4_B2C(sbi, s++), buf); } if (!count) return 0; @@ -3301,7 +3306,7 @@ static int ext4_fill_super(struct super_block *sb, void *data, int silent) char *orig_data = kstrdup(data, GFP_KERNEL); struct buffer_head *bh; struct ext4_super_block *es = NULL; - struct ext4_sb_info *sbi; + struct ext4_sb_info *sbi = kzalloc(sizeof(*sbi), GFP_KERNEL); ext4_fsblk_t block; ext4_fsblk_t sb_block = get_sb_block(&data); ext4_fsblk_t logical_sb_block; @@ -3320,16 +3325,14 @@ static int ext4_fill_super(struct super_block *sb, void *data, int silent) unsigned int journal_ioprio = DEFAULT_JOURNAL_IOPRIO; ext4_group_t first_not_zeroed; - sbi = kzalloc(sizeof(*sbi), GFP_KERNEL); - if (!sbi) - goto out_free_orig; + if ((data && !orig_data) || !sbi) + goto out_free_base; sbi->s_blockgroup_lock = kzalloc(sizeof(struct blockgroup_lock), GFP_KERNEL); - if (!sbi->s_blockgroup_lock) { - kfree(sbi); - goto out_free_orig; - } + if (!sbi->s_blockgroup_lock) + goto out_free_base; + sb->s_fs_info = sbi; sbi->s_sb = sb; sbi->s_inode_readahead_blks = EXT4_DEF_INODE_READAHEAD_BLKS; @@ -3475,11 +3478,19 @@ static int ext4_fill_super(struct super_block *sb, void *data, int silent) */ sbi->s_li_wait_mult = EXT4_DEF_LI_WAIT_MULT; - if (!parse_options((char *) sbi->s_es->s_mount_opts, sb, - &journal_devnum, &journal_ioprio, 0)) { - ext4_msg(sb, KERN_WARNING, - "failed to parse options in superblock: %s", - sbi->s_es->s_mount_opts); + if (sbi->s_es->s_mount_opts[0]) { + char *s_mount_opts = kstrndup(sbi->s_es->s_mount_opts, + sizeof(sbi->s_es->s_mount_opts), + GFP_KERNEL); + if (!s_mount_opts) + goto failed_mount; + if (!parse_options(s_mount_opts, sb, &journal_devnum, + &journal_ioprio, 0)) { + ext4_msg(sb, KERN_WARNING, + "failed to parse options in superblock: %s", + s_mount_opts); + } + kfree(s_mount_opts); } sbi->s_def_mount_opt = sbi->s_mount_opt; if (!parse_options((char *) data, sb, &journal_devnum, @@ -3505,6 +3516,11 @@ static int ext4_fill_super(struct super_block *sb, void *data, int silent) "both data=journal and dax"); goto failed_mount; } + if (ext4_has_feature_encrypt(sb)) { + ext4_msg(sb, KERN_WARNING, + "encrypted files will use data=ordered " + "instead of data journaling mode"); + } if (test_opt(sb, DELALLOC)) clear_opt(sb, DELALLOC); } else { @@ -3660,12 +3676,16 @@ static int ext4_fill_super(struct super_block *sb, void *data, int silent) sbi->s_blocks_per_group = le32_to_cpu(es->s_blocks_per_group); sbi->s_inodes_per_group = le32_to_cpu(es->s_inodes_per_group); - if (EXT4_INODE_SIZE(sb) == 0 || EXT4_INODES_PER_GROUP(sb) == 0) - goto cantfind_ext4; sbi->s_inodes_per_block = blocksize / EXT4_INODE_SIZE(sb); if (sbi->s_inodes_per_block == 0) goto cantfind_ext4; + if (sbi->s_inodes_per_group < sbi->s_inodes_per_block || + sbi->s_inodes_per_group > blocksize * 8) { + ext4_msg(sb, KERN_ERR, "invalid inodes per group: %lu\n", + sbi->s_blocks_per_group); + goto failed_mount; + } sbi->s_itb_per_group = sbi->s_inodes_per_group / sbi->s_inodes_per_block; sbi->s_desc_per_block = blocksize / EXT4_DESC_SIZE(sb); @@ -3748,13 +3768,6 @@ static int ext4_fill_super(struct super_block *sb, void *data, int silent) } sbi->s_cluster_ratio = clustersize / blocksize; - if (sbi->s_inodes_per_group > blocksize * 8) { - ext4_msg(sb, KERN_ERR, - "#inodes per group too big: %lu", - sbi->s_inodes_per_group); - goto failed_mount; - } - /* Do we have standard group size of clustersize * 8 blocks ? */ if (sbi->s_blocks_per_group == clustersize << 3) set_opt2(sb, STD_GROUP_SIZE); @@ -3814,6 +3827,15 @@ static int ext4_fill_super(struct super_block *sb, void *data, int silent) (EXT4_MAX_BLOCK_FILE_PHYS / EXT4_BLOCKS_PER_GROUP(sb))); db_count = (sbi->s_groups_count + EXT4_DESC_PER_BLOCK(sb) - 1) / EXT4_DESC_PER_BLOCK(sb); + if (ext4_has_feature_meta_bg(sb)) { + if (le32_to_cpu(es->s_first_meta_bg) >= db_count) { + ext4_msg(sb, KERN_WARNING, + "first meta block group too large: %u " + "(group descriptor block count %u)", + le32_to_cpu(es->s_first_meta_bg), db_count); + goto failed_mount; + } + } sbi->s_group_desc = ext4_kvmalloc(db_count * sizeof(struct buffer_head *), GFP_KERNEL); @@ -4160,7 +4182,9 @@ static int ext4_fill_super(struct super_block *sb, void *data, int silent) if (___ratelimit(&ext4_mount_msg_ratelimit, "EXT4-fs mount")) ext4_msg(sb, KERN_INFO, "mounted filesystem with%s. " - "Opts: %s%s%s", descr, sbi->s_es->s_mount_opts, + "Opts: %.*s%s%s", descr, + (int) sizeof(sbi->s_es->s_mount_opts), + sbi->s_es->s_mount_opts, *sbi->s_es->s_mount_opts ? "; " : "", orig_data); if (es->s_error_count) @@ -4239,8 +4263,8 @@ static int ext4_fill_super(struct super_block *sb, void *data, int silent) out_fail: sb->s_fs_info = NULL; kfree(sbi->s_blockgroup_lock); +out_free_base: kfree(sbi); -out_free_orig: kfree(orig_data); return err ? err : ret; } @@ -4550,7 +4574,8 @@ static int ext4_commit_super(struct super_block *sb, int sync) &EXT4_SB(sb)->s_freeinodes_counter)); BUFFER_TRACE(sbh, "marking dirty"); ext4_superblock_csum_set(sb); - lock_buffer(sbh); + if (sync) + lock_buffer(sbh); if (buffer_write_io_error(sbh)) { /* * Oh, dear. A previous attempt to write the @@ -4566,8 +4591,8 @@ static int ext4_commit_super(struct super_block *sb, int sync) set_buffer_uptodate(sbh); } mark_buffer_dirty(sbh); - unlock_buffer(sbh); if (sync) { + unlock_buffer(sbh); error = __sync_dirty_buffer(sbh, test_opt(sb, BARRIER) ? WRITE_FUA : WRITE_SYNC); if (error) diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c index 7e9b504bd8b295..b4dbc2f59656c1 100644 --- a/fs/f2fs/checkpoint.c +++ b/fs/f2fs/checkpoint.c @@ -772,6 +772,11 @@ int get_valid_checkpoint(struct f2fs_sb_info *sbi) if (sanity_check_ckpt(sbi)) goto fail_no_cp; + if (cur_page == cp1) + sbi->cur_cp_pack = 1; + else + sbi->cur_cp_pack = 2; + if (cp_blks <= 1) goto done; @@ -1123,7 +1128,7 @@ static int do_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc) le32_to_cpu(ckpt->checksum_offset))) = cpu_to_le32(crc32); - start_blk = __start_cp_addr(sbi); + start_blk = __start_cp_next_addr(sbi); /* need to wait for end_io results */ wait_on_all_pages_writeback(sbi); @@ -1187,6 +1192,7 @@ static int do_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc) clear_prefree_segments(sbi, cpc); clear_sbi_flag(sbi, SBI_IS_DIRTY); clear_sbi_flag(sbi, SBI_NEED_CP); + __set_cp_next_pack(sbi); /* * redirty superblock if metadata like node page or inode cache is diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 9ae194fd2fdb81..2c5ae0b41ada8d 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -28,6 +28,7 @@ #include "segment.h" #include "trace.h" #include +#include static void f2fs_read_end_io(struct bio *bio) { @@ -716,7 +717,7 @@ int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, } prealloc = 0; - ofs_in_node = dn.ofs_in_node; + last_ofs_in_node = ofs_in_node = dn.ofs_in_node; end_offset = ADDRS_PER_PAGE(dn.node_page, inode); next_block: @@ -1606,6 +1607,16 @@ static int f2fs_write_begin(struct file *file, struct address_space *mapping, block_t blkaddr = NULL_ADDR; int err = 0; + if (trace_android_fs_datawrite_start_enabled()) { + char *path, pathbuf[MAX_TRACE_PATHBUF_LEN]; + + path = android_fstrace_get_pathname(pathbuf, + MAX_TRACE_PATHBUF_LEN, + inode); + trace_android_fs_datawrite_start(inode, pos, len, + current->pid, path, + current->comm); + } trace_f2fs_write_begin(inode, pos, len, flags); /* @@ -1697,6 +1708,7 @@ static int f2fs_write_end(struct file *file, { struct inode *inode = page->mapping->host; + trace_android_fs_datawrite_end(inode, pos, len); trace_f2fs_write_end(inode, pos, len, copied); /* @@ -1758,6 +1770,29 @@ static ssize_t f2fs_direct_IO(struct kiocb *iocb, struct iov_iter *iter) trace_f2fs_direct_IO_enter(inode, offset, count, rw); + if (trace_android_fs_dataread_start_enabled() && + (rw == READ)) { + char *path, pathbuf[MAX_TRACE_PATHBUF_LEN]; + + path = android_fstrace_get_pathname(pathbuf, + MAX_TRACE_PATHBUF_LEN, + inode); + trace_android_fs_dataread_start(inode, offset, + count, current->pid, path, + current->comm); + } + if (trace_android_fs_datawrite_start_enabled() && + (rw == WRITE)) { + char *path, pathbuf[MAX_TRACE_PATHBUF_LEN]; + + path = android_fstrace_get_pathname(pathbuf, + MAX_TRACE_PATHBUF_LEN, + inode); + trace_android_fs_datawrite_start(inode, offset, count, + current->pid, path, + current->comm); + } + down_read(&F2FS_I(inode)->dio_rwsem[rw]); err = blockdev_direct_IO(iocb, inode, iter, get_data_block_dio); up_read(&F2FS_I(inode)->dio_rwsem[rw]); @@ -1769,6 +1804,13 @@ static ssize_t f2fs_direct_IO(struct kiocb *iocb, struct iov_iter *iter) f2fs_write_failed(mapping, offset + count); } + if (trace_android_fs_dataread_start_enabled() && + (rw == READ)) + trace_android_fs_dataread_end(inode, offset, count); + if (trace_android_fs_datawrite_start_enabled() && + (rw == WRITE)) + trace_android_fs_datawrite_end(inode, offset, count); + trace_f2fs_direct_IO_exit(inode, offset, count, rw, err); return err; diff --git a/fs/f2fs/debug.c b/fs/f2fs/debug.c index fb245bd302e4a9..687998e9557c67 100644 --- a/fs/f2fs/debug.c +++ b/fs/f2fs/debug.c @@ -310,17 +310,17 @@ static int stat_show(struct seq_file *s, void *v) seq_printf(s, " - Inner Struct Count: tree: %d(%d), node: %d\n", si->ext_tree, si->zombie_tree, si->ext_node); seq_puts(s, "\nBalancing F2FS Async:\n"); - seq_printf(s, " - inmem: %4lld, wb_bios: %4d\n", + seq_printf(s, " - inmem: %4d, wb_bios: %4d\n", si->inmem_pages, si->wb_bios); - seq_printf(s, " - nodes: %4lld in %4d\n", + seq_printf(s, " - nodes: %4d in %4d\n", si->ndirty_node, si->node_pages); - seq_printf(s, " - dents: %4lld in dirs:%4d (%4d)\n", + seq_printf(s, " - dents: %4d in dirs:%4d (%4d)\n", si->ndirty_dent, si->ndirty_dirs, si->ndirty_all); - seq_printf(s, " - datas: %4lld in files:%4d\n", + seq_printf(s, " - datas: %4d in files:%4d\n", si->ndirty_data, si->ndirty_files); - seq_printf(s, " - meta: %4lld in %4d\n", + seq_printf(s, " - meta: %4d in %4d\n", si->ndirty_meta, si->meta_pages); - seq_printf(s, " - imeta: %4lld\n", + seq_printf(s, " - imeta: %4d\n", si->ndirty_imeta); seq_printf(s, " - NATs: %9d/%9d\n - SITs: %9d/%9d\n", si->dirty_nats, si->nats, si->dirty_sits, si->sits); @@ -373,6 +373,7 @@ static int stat_open(struct inode *inode, struct file *file) } static const struct file_operations stat_fops = { + .owner = THIS_MODULE, .open = stat_open, .read = seq_read, .llseek = seq_lseek, diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index 9e8de18a168af6..506af456412fcc 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -428,7 +428,7 @@ struct f2fs_inode_info { /* Use below internally in f2fs*/ unsigned long flags; /* use to pass per-file flags */ struct rw_semaphore i_sem; /* protect fi info */ - struct percpu_counter dirty_pages; /* # of dirty pages */ + atomic_t dirty_pages; /* # of dirty pages */ f2fs_hash_t chash; /* hash value of given file name */ unsigned int clevel; /* maximum level of given file name */ nid_t i_xattr_nid; /* node id that contains xattrs */ @@ -764,6 +764,7 @@ struct f2fs_sb_info { /* for checkpoint */ struct f2fs_checkpoint *ckpt; /* raw checkpoint pointer */ + int cur_cp_pack; /* remain current cp pack */ spinlock_t cp_lock; /* for flag in ckpt */ struct inode *meta_inode; /* cache meta blocks */ struct mutex cp_mutex; /* checkpoint procedure lock */ @@ -818,7 +819,7 @@ struct f2fs_sb_info { atomic_t nr_wb_bios; /* # of writeback bios */ /* # of pages, see count_type */ - struct percpu_counter nr_pages[NR_COUNT_TYPE]; + atomic_t nr_pages[NR_COUNT_TYPE]; /* # of allocated blocks */ struct percpu_counter alloc_valid_block_count; @@ -1232,7 +1233,7 @@ static inline void dec_valid_block_count(struct f2fs_sb_info *sbi, static inline void inc_page_count(struct f2fs_sb_info *sbi, int count_type) { - percpu_counter_inc(&sbi->nr_pages[count_type]); + atomic_inc(&sbi->nr_pages[count_type]); if (count_type == F2FS_DIRTY_DATA || count_type == F2FS_INMEM_PAGES) return; @@ -1242,14 +1243,14 @@ static inline void inc_page_count(struct f2fs_sb_info *sbi, int count_type) static inline void inode_inc_dirty_pages(struct inode *inode) { - percpu_counter_inc(&F2FS_I(inode)->dirty_pages); + atomic_inc(&F2FS_I(inode)->dirty_pages); inc_page_count(F2FS_I_SB(inode), S_ISDIR(inode->i_mode) ? F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA); } static inline void dec_page_count(struct f2fs_sb_info *sbi, int count_type) { - percpu_counter_dec(&sbi->nr_pages[count_type]); + atomic_dec(&sbi->nr_pages[count_type]); } static inline void inode_dec_dirty_pages(struct inode *inode) @@ -1258,19 +1259,19 @@ static inline void inode_dec_dirty_pages(struct inode *inode) !S_ISLNK(inode->i_mode)) return; - percpu_counter_dec(&F2FS_I(inode)->dirty_pages); + atomic_dec(&F2FS_I(inode)->dirty_pages); dec_page_count(F2FS_I_SB(inode), S_ISDIR(inode->i_mode) ? F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA); } static inline s64 get_pages(struct f2fs_sb_info *sbi, int count_type) { - return percpu_counter_sum_positive(&sbi->nr_pages[count_type]); + return atomic_read(&sbi->nr_pages[count_type]); } -static inline s64 get_dirty_pages(struct inode *inode) +static inline int get_dirty_pages(struct inode *inode) { - return percpu_counter_sum_positive(&F2FS_I(inode)->dirty_pages); + return atomic_read(&F2FS_I(inode)->dirty_pages); } static inline int get_blocktype_secs(struct f2fs_sb_info *sbi, int block_type) @@ -1329,22 +1330,27 @@ static inline void *__bitmap_ptr(struct f2fs_sb_info *sbi, int flag) static inline block_t __start_cp_addr(struct f2fs_sb_info *sbi) { - block_t start_addr; - struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); - unsigned long long ckpt_version = cur_cp_version(ckpt); - - start_addr = le32_to_cpu(F2FS_RAW_SUPER(sbi)->cp_blkaddr); + block_t start_addr = le32_to_cpu(F2FS_RAW_SUPER(sbi)->cp_blkaddr); - /* - * odd numbered checkpoint should at cp segment 0 - * and even segment must be at cp segment 1 - */ - if (!(ckpt_version & 1)) + if (sbi->cur_cp_pack == 2) start_addr += sbi->blocks_per_seg; + return start_addr; +} +static inline block_t __start_cp_next_addr(struct f2fs_sb_info *sbi) +{ + block_t start_addr = le32_to_cpu(F2FS_RAW_SUPER(sbi)->cp_blkaddr); + + if (sbi->cur_cp_pack == 1) + start_addr += sbi->blocks_per_seg; return start_addr; } +static inline void __set_cp_next_pack(struct f2fs_sb_info *sbi) +{ + sbi->cur_cp_pack = (sbi->cur_cp_pack == 1) ? 2 : 1; +} + static inline block_t __start_sum_addr(struct f2fs_sb_info *sbi) { return le32_to_cpu(F2FS_CKPT(sbi)->cp_pack_start_sum); @@ -2181,8 +2187,8 @@ struct f2fs_stat_info { unsigned long long hit_largest, hit_cached, hit_rbtree; unsigned long long hit_total, total_ext; int ext_tree, zombie_tree, ext_node; - s64 ndirty_node, ndirty_dent, ndirty_meta, ndirty_data, ndirty_imeta; - s64 inmem_pages; + int ndirty_node, ndirty_dent, ndirty_meta, ndirty_data, ndirty_imeta; + int inmem_pages; unsigned int ndirty_dirs, ndirty_files, ndirty_all; int nats, dirty_nats, sits, dirty_sits, fnids; int total_count, utilization; diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index c7865073cd26ac..801111e1f8ef94 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -967,7 +967,7 @@ static int __clone_blkaddrs(struct inode *src_inode, struct inode *dst_inode, new_size = (dst + i) << PAGE_SHIFT; if (dst_inode->i_size < new_size) f2fs_i_size_write(dst_inode, new_size); - } while ((do_replace[i] || blkaddr[i] == NULL_ADDR) && --ilen); + } while (--ilen && (do_replace[i] || blkaddr[i] == NULL_ADDR)); f2fs_put_dnode(&dn); } else { @@ -1526,7 +1526,7 @@ static int f2fs_ioc_start_atomic_write(struct file *filp) goto out; f2fs_msg(F2FS_I_SB(inode)->sb, KERN_WARNING, - "Unexpected flush for atomic writes: ino=%lu, npages=%lld", + "Unexpected flush for atomic writes: ino=%lu, npages=%u", inode->i_ino, get_dirty_pages(inode)); ret = filemap_write_and_wait_range(inode->i_mapping, 0, LLONG_MAX); if (ret) diff --git a/fs/f2fs/inline.c b/fs/f2fs/inline.c index 5f1a67f756afc4..1427db9359f8ef 100644 --- a/fs/f2fs/inline.c +++ b/fs/f2fs/inline.c @@ -13,6 +13,7 @@ #include "f2fs.h" #include "node.h" +#include bool f2fs_may_inline_data(struct inode *inode) { @@ -82,14 +83,29 @@ int f2fs_read_inline_data(struct inode *inode, struct page *page) { struct page *ipage; + if (trace_android_fs_dataread_start_enabled()) { + char *path, pathbuf[MAX_TRACE_PATHBUF_LEN]; + + path = android_fstrace_get_pathname(pathbuf, + MAX_TRACE_PATHBUF_LEN, + inode); + trace_android_fs_dataread_start(inode, page_offset(page), + PAGE_SIZE, current->pid, + path, current->comm); + } + ipage = get_node_page(F2FS_I_SB(inode), inode->i_ino); if (IS_ERR(ipage)) { + trace_android_fs_dataread_end(inode, page_offset(page), + PAGE_SIZE); unlock_page(page); return PTR_ERR(ipage); } if (!f2fs_has_inline_data(inode)) { f2fs_put_page(ipage, 1); + trace_android_fs_dataread_end(inode, page_offset(page), + PAGE_SIZE); return -EAGAIN; } @@ -101,6 +117,8 @@ int f2fs_read_inline_data(struct inode *inode, struct page *page) if (!PageUptodate(page)) SetPageUptodate(page); f2fs_put_page(ipage, 1); + trace_android_fs_dataread_end(inode, page_offset(page), + PAGE_SIZE); unlock_page(page); return 0; } diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index 6132b4ce4e4ce3..013c6a541d6b9b 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -558,13 +558,9 @@ static struct inode *f2fs_alloc_inode(struct super_block *sb) init_once((void *) fi); - if (percpu_counter_init(&fi->dirty_pages, 0, GFP_NOFS)) { - kmem_cache_free(f2fs_inode_cachep, fi); - return NULL; - } - /* Initialize f2fs-specific inode info */ fi->vfs_inode.i_version = 1; + atomic_set(&fi->dirty_pages, 0); fi->i_current_depth = 1; fi->i_advise = 0; init_rwsem(&fi->i_sem); @@ -687,16 +683,11 @@ static void f2fs_i_callback(struct rcu_head *head) static void f2fs_destroy_inode(struct inode *inode) { - percpu_counter_destroy(&F2FS_I(inode)->dirty_pages); call_rcu(&inode->i_rcu, f2fs_i_callback); } static void destroy_percpu_info(struct f2fs_sb_info *sbi) { - int i; - - for (i = 0; i < NR_COUNT_TYPE; i++) - percpu_counter_destroy(&sbi->nr_pages[i]); percpu_counter_destroy(&sbi->alloc_valid_block_count); percpu_counter_destroy(&sbi->total_valid_inode_count); } @@ -1447,6 +1438,7 @@ int sanity_check_ckpt(struct f2fs_sb_info *sbi) static void init_sb_info(struct f2fs_sb_info *sbi) { struct f2fs_super_block *raw_super = sbi->raw_super; + int i; sbi->log_sectors_per_block = le32_to_cpu(raw_super->log_sectors_per_block); @@ -1471,6 +1463,9 @@ static void init_sb_info(struct f2fs_sb_info *sbi) sbi->interval_time[REQ_TIME] = DEF_IDLE_INTERVAL; clear_sbi_flag(sbi, SBI_NEED_FSCK); + for (i = 0; i < NR_COUNT_TYPE; i++) + atomic_set(&sbi->nr_pages[i], 0); + INIT_LIST_HEAD(&sbi->s_list); mutex_init(&sbi->umount_mutex); mutex_init(&sbi->wio_mutex[NODE]); @@ -1486,13 +1481,7 @@ static void init_sb_info(struct f2fs_sb_info *sbi) static int init_percpu_info(struct f2fs_sb_info *sbi) { - int i, err; - - for (i = 0; i < NR_COUNT_TYPE; i++) { - err = percpu_counter_init(&sbi->nr_pages[i], 0, GFP_KERNEL); - if (err) - return err; - } + int err; err = percpu_counter_init(&sbi->alloc_valid_block_count, 0, GFP_KERNEL); if (err) diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c index 05713a5da08342..ffec69dfb80d36 100644 --- a/fs/fs-writeback.c +++ b/fs/fs-writeback.c @@ -2104,7 +2104,7 @@ void __mark_inode_dirty(struct inode *inode, int flags) (dirtytime && (inode->i_state & I_DIRTY_INODE))) return; - if (unlikely(block_dump)) + if (unlikely(block_dump > 1)) block_dump___mark_inode_dirty(inode); spin_lock(&inode->i_lock); diff --git a/fs/fs_struct.c b/fs/fs_struct.c index 7dca743b2ce1c8..940c683561dd37 100644 --- a/fs/fs_struct.c +++ b/fs/fs_struct.c @@ -44,6 +44,7 @@ void set_fs_pwd(struct fs_struct *fs, const struct path *path) if (old_pwd.dentry) path_put(&old_pwd); } +EXPORT_SYMBOL(set_fs_pwd); static inline int replace_path(struct path *p, const struct path *old, const struct path *new) { @@ -89,6 +90,7 @@ void free_fs_struct(struct fs_struct *fs) path_put(&fs->pwd); kmem_cache_free(fs_cachep, fs); } +EXPORT_SYMBOL(free_fs_struct); void exit_fs(struct task_struct *tsk) { @@ -127,6 +129,7 @@ struct fs_struct *copy_fs_struct(struct fs_struct *old) } return fs; } +EXPORT_SYMBOL_GPL(copy_fs_struct); int unshare_fs_struct(void) { diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 70ea57c7b6bb2b..3fd1e2183e7623 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -13,12 +13,14 @@ #include #include #include +#include #include #include #include #include #include #include +#include MODULE_ALIAS_MISCDEV(FUSE_MINOR); MODULE_ALIAS("devname:fuse"); @@ -449,7 +451,9 @@ static void request_wait_answer(struct fuse_conn *fc, struct fuse_req *req) * Either request is already in userspace, or it was forced. * Wait it out. */ - wait_event(req->waitq, test_bit(FR_FINISHED, &req->flags)); + while (!test_bit(FR_FINISHED, &req->flags)) + wait_event_freezable(req->waitq, + test_bit(FR_FINISHED, &req->flags)); } static void __fuse_request_send(struct fuse_conn *fc, struct fuse_req *req) @@ -1875,6 +1879,10 @@ static ssize_t fuse_dev_do_write(struct fuse_dev *fud, cs->move_pages = 0; err = copy_out_args(cs, &req->out, nbytes); + if (req->in.h.opcode == FUSE_CANONICAL_PATH) { + req->out.h.error = kern_path((char *)req->out.args[0].value, 0, + req->canonical_path); + } fuse_copy_finish(cs); spin_lock(&fpq->lock); @@ -2025,7 +2033,6 @@ static void end_requests(struct fuse_conn *fc, struct list_head *head) struct fuse_req *req; req = list_entry(head->next, struct fuse_req, list); req->out.h.error = -ECONNABORTED; - clear_bit(FR_PENDING, &req->flags); clear_bit(FR_SENT, &req->flags); list_del_init(&req->list); request_end(fc, req); @@ -2103,6 +2110,8 @@ void fuse_abort_conn(struct fuse_conn *fc) spin_lock(&fiq->waitq.lock); fiq->connected = 0; list_splice_init(&fiq->pending, &to_end2); + list_for_each_entry(req, &to_end2, list) + clear_bit(FR_PENDING, &req->flags); while (forget_pending(fiq)) kfree(dequeue_forget(fiq, 1, NULL)); wake_up_all_locked(&fiq->waitq); diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index 096f79997f75ad..fc8ba62d5c31f8 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -68,7 +68,7 @@ static u64 time_to_jiffies(u64 sec, u32 nsec) if (sec || nsec) { struct timespec64 ts = { sec, - max_t(u32, nsec, NSEC_PER_SEC - 1) + min_t(u32, nsec, NSEC_PER_SEC - 1) }; return get_jiffies_64() + timespec64_to_jiffies(&ts); @@ -262,6 +262,50 @@ static int fuse_dentry_revalidate(struct dentry *entry, unsigned int flags) goto out; } +/* + * Get the canonical path. Since we must translate to a path, this must be done + * in the context of the userspace daemon, however, the userspace daemon cannot + * look up paths on its own. Instead, we handle the lookup as a special case + * inside of the write request. + */ +static void fuse_dentry_canonical_path(const struct path *path, struct path *canonical_path) { + struct inode *inode = path->dentry->d_inode; + struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_req *req; + int err; + char *path_name; + + req = fuse_get_req(fc, 1); + err = PTR_ERR(req); + if (IS_ERR(req)) + goto default_path; + + path_name = (char*)__get_free_page(GFP_KERNEL); + if (!path_name) { + fuse_put_request(fc, req); + goto default_path; + } + + req->in.h.opcode = FUSE_CANONICAL_PATH; + req->in.h.nodeid = get_node_id(inode); + req->in.numargs = 0; + req->out.numargs = 1; + req->out.args[0].size = PATH_MAX; + req->out.args[0].value = path_name; + req->canonical_path = canonical_path; + req->out.argvar = 1; + fuse_request_send(fc, req); + err = req->out.h.error; + fuse_put_request(fc, req); + free_page((unsigned long)path_name); + if (!err) + return; +default_path: + canonical_path->dentry = path->dentry; + canonical_path->mnt = path->mnt; + path_get(canonical_path); +} + static int invalid_nodeid(u64 nodeid) { return !nodeid || nodeid == FUSE_ROOT_ID; @@ -284,11 +328,13 @@ const struct dentry_operations fuse_dentry_operations = { .d_revalidate = fuse_dentry_revalidate, .d_init = fuse_dentry_init, .d_release = fuse_dentry_release, + .d_canonical_path = fuse_dentry_canonical_path, }; const struct dentry_operations fuse_root_dentry_operations = { .d_init = fuse_dentry_init, .d_release = fuse_dentry_release, + .d_canonical_path = fuse_dentry_canonical_path, }; int fuse_valid_type(int m) diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 91307940c8ac5e..6b30a12f898f1e 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -368,6 +368,9 @@ struct fuse_req { /** Inode used in the request or NULL */ struct inode *inode; + /** Path used for completing d_canonical_path */ + struct path *canonical_path; + /** AIO control block */ struct fuse_io_priv *io; diff --git a/fs/inode.c b/fs/inode.c index 88110fd0b282e4..0aaebd1de454e9 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -1780,7 +1780,7 @@ int dentry_needs_remove_privs(struct dentry *dentry) return mask; } -static int __remove_privs(struct dentry *dentry, int kill) +static int __remove_privs(struct vfsmount *mnt, struct dentry *dentry, int kill) { struct iattr newattrs; @@ -1789,7 +1789,7 @@ static int __remove_privs(struct dentry *dentry, int kill) * Note we call this on write, so notify_change will not * encounter any conflicting delegations: */ - return notify_change(dentry, &newattrs, NULL); + return notify_change2(mnt, dentry, &newattrs, NULL); } /* @@ -1811,7 +1811,7 @@ int file_remove_privs(struct file *file) if (kill < 0) return kill; if (kill) - error = __remove_privs(dentry, kill); + error = __remove_privs(file->f_path.mnt, dentry, kill); if (!error) inode_has_no_xattr(inode); diff --git a/fs/internal.h b/fs/internal.h index f4da3341b4a37d..15fe2aac4e2cb7 100644 --- a/fs/internal.h +++ b/fs/internal.h @@ -88,9 +88,11 @@ extern struct file *get_empty_filp(void); * super.c */ extern int do_remount_sb(struct super_block *, int, void *, int); +extern int do_remount_sb2(struct vfsmount *, struct super_block *, int, + void *, int); extern bool trylock_super(struct super_block *sb); extern struct dentry *mount_fs(struct file_system_type *, - int, const char *, void *); + int, const char *, struct vfsmount *, void *); extern struct super_block *user_get_super(dev_t); /* diff --git a/fs/iomap.c b/fs/iomap.c index a8ee8c33ca782d..814ae8f9587dd5 100644 --- a/fs/iomap.c +++ b/fs/iomap.c @@ -113,6 +113,9 @@ iomap_write_begin(struct inode *inode, loff_t pos, unsigned len, unsigned flags, BUG_ON(pos + len > iomap->offset + iomap->length); + if (fatal_signal_pending(current)) + return -EINTR; + page = grab_cache_page_write_begin(inode->i_mapping, index, flags); if (!page) return -ENOMEM; diff --git a/fs/mpage.c b/fs/mpage.c index d2413af0823a24..802b4816204aef 100644 --- a/fs/mpage.c +++ b/fs/mpage.c @@ -31,6 +31,14 @@ #include #include "internal.h" +#define CREATE_TRACE_POINTS +#include + +EXPORT_TRACEPOINT_SYMBOL(android_fs_datawrite_start); +EXPORT_TRACEPOINT_SYMBOL(android_fs_datawrite_end); +EXPORT_TRACEPOINT_SYMBOL(android_fs_dataread_start); +EXPORT_TRACEPOINT_SYMBOL(android_fs_dataread_end); + /* * I/O completion handler for multipage BIOs. * @@ -48,6 +56,16 @@ static void mpage_end_io(struct bio *bio) struct bio_vec *bv; int i; + if (trace_android_fs_dataread_end_enabled() && + (bio_data_dir(bio) == READ)) { + struct page *first_page = bio->bi_io_vec[0].bv_page; + + if (first_page != NULL) + trace_android_fs_dataread_end(first_page->mapping->host, + page_offset(first_page), + bio->bi_iter.bi_size); + } + bio_for_each_segment_all(bv, bio, i) { struct page *page = bv->bv_page; page_endio(page, op_is_write(bio_op(bio)), bio->bi_error); @@ -58,6 +76,24 @@ static void mpage_end_io(struct bio *bio) static struct bio *mpage_bio_submit(int op, int op_flags, struct bio *bio) { + if (trace_android_fs_dataread_start_enabled() && (op == REQ_OP_READ)) { + struct page *first_page = bio->bi_io_vec[0].bv_page; + + if (first_page != NULL) { + char *path, pathbuf[MAX_TRACE_PATHBUF_LEN]; + + path = android_fstrace_get_pathname(pathbuf, + MAX_TRACE_PATHBUF_LEN, + first_page->mapping->host); + trace_android_fs_dataread_start( + first_page->mapping->host, + page_offset(first_page), + bio->bi_iter.bi_size, + current->pid, + path, + current->comm); + } + } bio->bi_end_io = mpage_end_io; bio_set_op_attrs(bio, op, op_flags); guard_bio_eod(op, bio); diff --git a/fs/namei.c b/fs/namei.c index 5b4eed2215304a..8fa2ddc5ed3506 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -375,9 +375,11 @@ EXPORT_SYMBOL(generic_permission); * flag in inode->i_opflags, that says "this has not special * permission function, use the fast case". */ -static inline int do_inode_permission(struct inode *inode, int mask) +static inline int do_inode_permission(struct vfsmount *mnt, struct inode *inode, int mask) { if (unlikely(!(inode->i_opflags & IOP_FASTPERM))) { + if (likely(mnt && inode->i_op->permission2)) + return inode->i_op->permission2(mnt, inode, mask); if (likely(inode->i_op->permission)) return inode->i_op->permission(inode, mask); @@ -401,7 +403,7 @@ static inline int do_inode_permission(struct inode *inode, int mask) * This does not check for a read-only file system. You probably want * inode_permission(). */ -int __inode_permission(struct inode *inode, int mask) +int __inode_permission2(struct vfsmount *mnt, struct inode *inode, int mask) { int retval; @@ -421,7 +423,7 @@ int __inode_permission(struct inode *inode, int mask) return -EACCES; } - retval = do_inode_permission(inode, mask); + retval = do_inode_permission(mnt, inode, mask); if (retval) return retval; @@ -429,7 +431,14 @@ int __inode_permission(struct inode *inode, int mask) if (retval) return retval; - return security_inode_permission(inode, mask); + retval = security_inode_permission(inode, mask); + return retval; +} +EXPORT_SYMBOL(__inode_permission2); + +int __inode_permission(struct inode *inode, int mask) +{ + return __inode_permission2(NULL, inode, mask); } EXPORT_SYMBOL(__inode_permission); @@ -465,14 +474,20 @@ static int sb_permission(struct super_block *sb, struct inode *inode, int mask) * * When checking for MAY_APPEND, MAY_WRITE must also be set in @mask. */ -int inode_permission(struct inode *inode, int mask) +int inode_permission2(struct vfsmount *mnt, struct inode *inode, int mask) { int retval; retval = sb_permission(inode->i_sb, inode, mask); if (retval) return retval; - return __inode_permission(inode, mask); + return __inode_permission2(mnt, inode, mask); +} +EXPORT_SYMBOL(inode_permission2); + +int inode_permission(struct inode *inode, int mask) +{ + return inode_permission2(NULL, inode, mask); } EXPORT_SYMBOL(inode_permission); @@ -1669,13 +1684,13 @@ static struct dentry *lookup_slow(const struct qstr *name, static inline int may_lookup(struct nameidata *nd) { if (nd->flags & LOOKUP_RCU) { - int err = inode_permission(nd->inode, MAY_EXEC|MAY_NOT_BLOCK); + int err = inode_permission2(nd->path.mnt, nd->inode, MAY_EXEC|MAY_NOT_BLOCK); if (err != -ECHILD) return err; if (unlazy_walk(nd, NULL, 0)) return -ECHILD; } - return inode_permission(nd->inode, MAY_EXEC); + return inode_permission2(nd->path.mnt, nd->inode, MAY_EXEC); } static inline int handle_dots(struct nameidata *nd, int type) @@ -2146,11 +2161,12 @@ static const char *path_init(struct nameidata *nd, unsigned flags) nd->depth = 0; if (flags & LOOKUP_ROOT) { struct dentry *root = nd->root.dentry; + struct vfsmount *mnt = nd->root.mnt; struct inode *inode = root->d_inode; if (*s) { if (!d_can_lookup(root)) return ERR_PTR(-ENOTDIR); - retval = inode_permission(inode, MAY_EXEC); + retval = inode_permission2(mnt, inode, MAY_EXEC); if (retval) return ERR_PTR(retval); } @@ -2415,6 +2431,7 @@ EXPORT_SYMBOL(vfs_path_lookup); /** * lookup_one_len - filesystem helper to lookup single pathname component * @name: pathname component to lookup + * @mnt: mount we are looking up on * @base: base directory to lookup from * @len: maximum length @len should be interpreted to * @@ -2423,7 +2440,7 @@ EXPORT_SYMBOL(vfs_path_lookup); * * The caller must hold base->i_mutex. */ -struct dentry *lookup_one_len(const char *name, struct dentry *base, int len) +struct dentry *lookup_one_len2(const char *name, struct vfsmount *mnt, struct dentry *base, int len) { struct qstr this; unsigned int c; @@ -2457,12 +2474,18 @@ struct dentry *lookup_one_len(const char *name, struct dentry *base, int len) return ERR_PTR(err); } - err = inode_permission(base->d_inode, MAY_EXEC); + err = inode_permission2(mnt, base->d_inode, MAY_EXEC); if (err) return ERR_PTR(err); return __lookup_hash(&this, base, 0); } +EXPORT_SYMBOL(lookup_one_len2); + +struct dentry *lookup_one_len(const char *name, struct dentry *base, int len) +{ + return lookup_one_len2(name, NULL, base, len); +} EXPORT_SYMBOL(lookup_one_len); /** @@ -2765,7 +2788,7 @@ EXPORT_SYMBOL(__check_sticky); * 11. We don't allow removal of NFS sillyrenamed files; it's handled by * nfs_async_unlink(). */ -static int may_delete(struct inode *dir, struct dentry *victim, bool isdir) +static int may_delete(struct vfsmount *mnt, struct inode *dir, struct dentry *victim, bool isdir) { struct inode *inode = d_backing_inode(victim); int error; @@ -2777,7 +2800,7 @@ static int may_delete(struct inode *dir, struct dentry *victim, bool isdir) BUG_ON(victim->d_parent->d_inode != dir); audit_inode_child(dir, victim, AUDIT_TYPE_CHILD_DELETE); - error = inode_permission(dir, MAY_WRITE | MAY_EXEC); + error = inode_permission2(mnt, dir, MAY_WRITE | MAY_EXEC); if (error) return error; if (IS_APPEND(dir)) @@ -2809,7 +2832,7 @@ static int may_delete(struct inode *dir, struct dentry *victim, bool isdir) * 4. We should have write and exec permissions on dir * 5. We can't do it if dir is immutable (done in permission()) */ -static inline int may_create(struct inode *dir, struct dentry *child) +static inline int may_create(struct vfsmount *mnt, struct inode *dir, struct dentry *child) { struct user_namespace *s_user_ns; audit_inode_child(dir, child, AUDIT_TYPE_CHILD_CREATE); @@ -2821,7 +2844,7 @@ static inline int may_create(struct inode *dir, struct dentry *child) if (!kuid_has_mapping(s_user_ns, current_fsuid()) || !kgid_has_mapping(s_user_ns, current_fsgid())) return -EOVERFLOW; - return inode_permission(dir, MAY_WRITE | MAY_EXEC); + return inode_permission2(mnt, dir, MAY_WRITE | MAY_EXEC); } /* @@ -2868,10 +2891,10 @@ void unlock_rename(struct dentry *p1, struct dentry *p2) } EXPORT_SYMBOL(unlock_rename); -int vfs_create(struct inode *dir, struct dentry *dentry, umode_t mode, - bool want_excl) +int vfs_create2(struct vfsmount *mnt, struct inode *dir, struct dentry *dentry, + umode_t mode, bool want_excl) { - int error = may_create(dir, dentry); + int error = may_create(mnt, dir, dentry); if (error) return error; @@ -2887,6 +2910,13 @@ int vfs_create(struct inode *dir, struct dentry *dentry, umode_t mode, fsnotify_create(dir, dentry); return error; } +EXPORT_SYMBOL(vfs_create2); + +int vfs_create(struct inode *dir, struct dentry *dentry, umode_t mode, + bool want_excl) +{ + return vfs_create2(NULL, dir, dentry, mode, want_excl); +} EXPORT_SYMBOL(vfs_create); bool may_open_dev(const struct path *path) @@ -2898,6 +2928,7 @@ bool may_open_dev(const struct path *path) static int may_open(struct path *path, int acc_mode, int flag) { struct dentry *dentry = path->dentry; + struct vfsmount *mnt = path->mnt; struct inode *inode = dentry->d_inode; int error; @@ -2922,7 +2953,7 @@ static int may_open(struct path *path, int acc_mode, int flag) break; } - error = inode_permission(inode, MAY_OPEN | acc_mode); + error = inode_permission2(mnt, inode, MAY_OPEN | acc_mode); if (error) return error; @@ -2957,7 +2988,7 @@ static int handle_truncate(struct file *filp) if (!error) error = security_path_truncate(path); if (!error) { - error = do_truncate(path->dentry, 0, + error = do_truncate2(path->mnt, path->dentry, 0, ATTR_MTIME|ATTR_CTIME|ATTR_OPEN, filp); } @@ -2978,7 +3009,7 @@ static int may_o_create(const struct path *dir, struct dentry *dentry, umode_t m if (error) return error; - error = inode_permission(dir->dentry->d_inode, MAY_WRITE | MAY_EXEC); + error = inode_permission2(dir->mnt, dir->dentry->d_inode, MAY_WRITE | MAY_EXEC); if (error) return error; @@ -3409,7 +3440,7 @@ static int do_tmpfile(struct nameidata *nd, unsigned flags, goto out; dir = path.dentry->d_inode; /* we want directory to be writable */ - error = inode_permission(dir, MAY_WRITE | MAY_EXEC); + error = inode_permission2(nd->path.mnt, dir, MAY_WRITE | MAY_EXEC); if (error) goto out2; if (!dir->i_op->tmpfile) { @@ -3662,9 +3693,9 @@ inline struct dentry *user_path_create(int dfd, const char __user *pathname, } EXPORT_SYMBOL(user_path_create); -int vfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev) +int vfs_mknod2(struct vfsmount *mnt, struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev) { - int error = may_create(dir, dentry); + int error = may_create(mnt, dir, dentry); if (error) return error; @@ -3688,6 +3719,12 @@ int vfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev) fsnotify_create(dir, dentry); return error; } +EXPORT_SYMBOL(vfs_mknod2); + +int vfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev) +{ + return vfs_mknod2(NULL, dir, dentry, mode, dev); +} EXPORT_SYMBOL(vfs_mknod); static int may_mknod(umode_t mode) @@ -3730,12 +3767,12 @@ SYSCALL_DEFINE4(mknodat, int, dfd, const char __user *, filename, umode_t, mode, goto out; switch (mode & S_IFMT) { case 0: case S_IFREG: - error = vfs_create(path.dentry->d_inode,dentry,mode,true); + error = vfs_create2(path.mnt, path.dentry->d_inode,dentry,mode,true); if (!error) ima_post_path_mknod(dentry); break; case S_IFCHR: case S_IFBLK: - error = vfs_mknod(path.dentry->d_inode,dentry,mode, + error = vfs_mknod2(path.mnt, path.dentry->d_inode,dentry,mode, new_decode_dev(dev)); break; case S_IFIFO: case S_IFSOCK: @@ -3756,9 +3793,9 @@ SYSCALL_DEFINE3(mknod, const char __user *, filename, umode_t, mode, unsigned, d return sys_mknodat(AT_FDCWD, filename, mode, dev); } -int vfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +int vfs_mkdir2(struct vfsmount *mnt, struct inode *dir, struct dentry *dentry, umode_t mode) { - int error = may_create(dir, dentry); + int error = may_create(mnt, dir, dentry); unsigned max_links = dir->i_sb->s_max_links; if (error) @@ -3780,6 +3817,12 @@ int vfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) fsnotify_mkdir(dir, dentry); return error; } +EXPORT_SYMBOL(vfs_mkdir2); + +int vfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +{ + return vfs_mkdir2(NULL, dir, dentry, mode); +} EXPORT_SYMBOL(vfs_mkdir); SYSCALL_DEFINE3(mkdirat, int, dfd, const char __user *, pathname, umode_t, mode) @@ -3798,7 +3841,7 @@ SYSCALL_DEFINE3(mkdirat, int, dfd, const char __user *, pathname, umode_t, mode) mode &= ~current_umask(); error = security_path_mkdir(&path, dentry, mode); if (!error) - error = vfs_mkdir(path.dentry->d_inode, dentry, mode); + error = vfs_mkdir2(path.mnt, path.dentry->d_inode, dentry, mode); done_path_create(&path, dentry); if (retry_estale(error, lookup_flags)) { lookup_flags |= LOOKUP_REVAL; @@ -3812,9 +3855,9 @@ SYSCALL_DEFINE2(mkdir, const char __user *, pathname, umode_t, mode) return sys_mkdirat(AT_FDCWD, pathname, mode); } -int vfs_rmdir(struct inode *dir, struct dentry *dentry) +int vfs_rmdir2(struct vfsmount *mnt, struct inode *dir, struct dentry *dentry) { - int error = may_delete(dir, dentry, 1); + int error = may_delete(mnt, dir, dentry, 1); if (error) return error; @@ -3849,6 +3892,12 @@ int vfs_rmdir(struct inode *dir, struct dentry *dentry) d_delete(dentry); return error; } +EXPORT_SYMBOL(vfs_rmdir2); + +int vfs_rmdir(struct inode *dir, struct dentry *dentry) +{ + return vfs_rmdir2(NULL, dir, dentry); +} EXPORT_SYMBOL(vfs_rmdir); static long do_rmdir(int dfd, const char __user *pathname) @@ -3894,7 +3943,7 @@ static long do_rmdir(int dfd, const char __user *pathname) error = security_path_rmdir(&path, dentry); if (error) goto exit3; - error = vfs_rmdir(path.dentry->d_inode, dentry); + error = vfs_rmdir2(path.mnt, path.dentry->d_inode, dentry); exit3: dput(dentry); exit2: @@ -3933,10 +3982,10 @@ SYSCALL_DEFINE1(rmdir, const char __user *, pathname) * be appropriate for callers that expect the underlying filesystem not * to be NFS exported. */ -int vfs_unlink(struct inode *dir, struct dentry *dentry, struct inode **delegated_inode) +int vfs_unlink2(struct vfsmount *mnt, struct inode *dir, struct dentry *dentry, struct inode **delegated_inode) { struct inode *target = dentry->d_inode; - int error = may_delete(dir, dentry, 0); + int error = may_delete(mnt, dir, dentry, 0); if (error) return error; @@ -3971,6 +4020,12 @@ int vfs_unlink(struct inode *dir, struct dentry *dentry, struct inode **delegate return error; } +EXPORT_SYMBOL(vfs_unlink2); + +int vfs_unlink(struct inode *dir, struct dentry *dentry, struct inode **delegated_inode) +{ + return vfs_unlink2(NULL, dir, dentry, delegated_inode); +} EXPORT_SYMBOL(vfs_unlink); /* @@ -4018,7 +4073,7 @@ static long do_unlinkat(int dfd, const char __user *pathname) error = security_path_unlink(&path, dentry); if (error) goto exit2; - error = vfs_unlink(path.dentry->d_inode, dentry, &delegated_inode); + error = vfs_unlink2(path.mnt, path.dentry->d_inode, dentry, &delegated_inode); exit2: dput(dentry); } @@ -4068,9 +4123,9 @@ SYSCALL_DEFINE1(unlink, const char __user *, pathname) return do_unlinkat(AT_FDCWD, pathname); } -int vfs_symlink(struct inode *dir, struct dentry *dentry, const char *oldname) +int vfs_symlink2(struct vfsmount *mnt, struct inode *dir, struct dentry *dentry, const char *oldname) { - int error = may_create(dir, dentry); + int error = may_create(mnt, dir, dentry); if (error) return error; @@ -4087,6 +4142,12 @@ int vfs_symlink(struct inode *dir, struct dentry *dentry, const char *oldname) fsnotify_create(dir, dentry); return error; } +EXPORT_SYMBOL(vfs_symlink2); + +int vfs_symlink(struct inode *dir, struct dentry *dentry, const char *oldname) +{ + return vfs_symlink2(NULL, dir, dentry, oldname); +} EXPORT_SYMBOL(vfs_symlink); SYSCALL_DEFINE3(symlinkat, const char __user *, oldname, @@ -4109,7 +4170,7 @@ SYSCALL_DEFINE3(symlinkat, const char __user *, oldname, error = security_path_symlink(&path, dentry, from->name); if (!error) - error = vfs_symlink(path.dentry->d_inode, dentry, from->name); + error = vfs_symlink2(path.mnt, path.dentry->d_inode, dentry, from->name); done_path_create(&path, dentry); if (retry_estale(error, lookup_flags)) { lookup_flags |= LOOKUP_REVAL; @@ -4144,7 +4205,7 @@ SYSCALL_DEFINE2(symlink, const char __user *, oldname, const char __user *, newn * be appropriate for callers that expect the underlying filesystem not * to be NFS exported. */ -int vfs_link(struct dentry *old_dentry, struct inode *dir, struct dentry *new_dentry, struct inode **delegated_inode) +int vfs_link2(struct vfsmount *mnt, struct dentry *old_dentry, struct inode *dir, struct dentry *new_dentry, struct inode **delegated_inode) { struct inode *inode = old_dentry->d_inode; unsigned max_links = dir->i_sb->s_max_links; @@ -4153,7 +4214,7 @@ int vfs_link(struct dentry *old_dentry, struct inode *dir, struct dentry *new_de if (!inode) return -ENOENT; - error = may_create(dir, new_dentry); + error = may_create(mnt, dir, new_dentry); if (error) return error; @@ -4203,6 +4264,12 @@ int vfs_link(struct dentry *old_dentry, struct inode *dir, struct dentry *new_de fsnotify_link(dir, inode, new_dentry); return error; } +EXPORT_SYMBOL(vfs_link2); + +int vfs_link(struct dentry *old_dentry, struct inode *dir, struct dentry *new_dentry, struct inode **delegated_inode) +{ + return vfs_link2(NULL, old_dentry, dir, new_dentry, delegated_inode); +} EXPORT_SYMBOL(vfs_link); /* @@ -4258,7 +4325,7 @@ SYSCALL_DEFINE5(linkat, int, olddfd, const char __user *, oldname, error = security_path_link(old_path.dentry, &new_path, new_dentry); if (error) goto out_dput; - error = vfs_link(old_path.dentry, new_path.dentry->d_inode, new_dentry, &delegated_inode); + error = vfs_link2(old_path.mnt, old_path.dentry, new_path.dentry->d_inode, new_dentry, &delegated_inode); out_dput: done_path_create(&new_path, new_dentry); if (delegated_inode) { @@ -4333,7 +4400,8 @@ SYSCALL_DEFINE2(link, const char __user *, oldname, const char __user *, newname * ->i_mutex on parents, which works but leads to some truly excessive * locking]. */ -int vfs_rename(struct inode *old_dir, struct dentry *old_dentry, +int vfs_rename2(struct vfsmount *mnt, + struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, struct inode **delegated_inode, unsigned int flags) { @@ -4352,19 +4420,19 @@ int vfs_rename(struct inode *old_dir, struct dentry *old_dentry, if (d_real_inode(old_dentry) == d_real_inode(new_dentry)) return 0; - error = may_delete(old_dir, old_dentry, is_dir); + error = may_delete(mnt, old_dir, old_dentry, is_dir); if (error) return error; if (!target) { - error = may_create(new_dir, new_dentry); + error = may_create(mnt, new_dir, new_dentry); } else { new_is_dir = d_is_dir(new_dentry); if (!(flags & RENAME_EXCHANGE)) - error = may_delete(new_dir, new_dentry, is_dir); + error = may_delete(mnt, new_dir, new_dentry, is_dir); else - error = may_delete(new_dir, new_dentry, new_is_dir); + error = may_delete(mnt, new_dir, new_dentry, new_is_dir); } if (error) return error; @@ -4378,12 +4446,12 @@ int vfs_rename(struct inode *old_dir, struct dentry *old_dentry, */ if (new_dir != old_dir) { if (is_dir) { - error = inode_permission(source, MAY_WRITE); + error = inode_permission2(mnt, source, MAY_WRITE); if (error) return error; } if ((flags & RENAME_EXCHANGE) && new_is_dir) { - error = inode_permission(target, MAY_WRITE); + error = inode_permission2(mnt, target, MAY_WRITE); if (error) return error; } @@ -4460,6 +4528,14 @@ int vfs_rename(struct inode *old_dir, struct dentry *old_dentry, return error; } +EXPORT_SYMBOL(vfs_rename2); + +int vfs_rename(struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry, + struct inode **delegated_inode, unsigned int flags) +{ + return vfs_rename2(NULL, old_dir, old_dentry, new_dir, new_dentry, delegated_inode, flags); +} EXPORT_SYMBOL(vfs_rename); SYSCALL_DEFINE5(renameat2, int, olddfd, const char __user *, oldname, @@ -4573,7 +4649,7 @@ SYSCALL_DEFINE5(renameat2, int, olddfd, const char __user *, oldname, &new_path, new_dentry, flags); if (error) goto exit5; - error = vfs_rename(old_path.dentry->d_inode, old_dentry, + error = vfs_rename2(old_path.mnt, old_path.dentry->d_inode, old_dentry, new_path.dentry->d_inode, new_dentry, &delegated_inode, flags); exit5: @@ -4618,7 +4694,7 @@ SYSCALL_DEFINE2(rename, const char __user *, oldname, const char __user *, newna int vfs_whiteout(struct inode *dir, struct dentry *dentry) { - int error = may_create(dir, dentry); + int error = may_create(NULL, dir, dentry); if (error) return error; diff --git a/fs/namespace.c b/fs/namespace.c index e6c234b1a6456c..4dc014ea14ea1f 100644 --- a/fs/namespace.c +++ b/fs/namespace.c @@ -580,6 +580,7 @@ int sb_prepare_remount_readonly(struct super_block *sb) static void free_vfsmnt(struct mount *mnt) { + kfree(mnt->mnt.data); kfree_const(mnt->mnt_devname); #ifdef CONFIG_SMP free_percpu(mnt->mnt_pcp); @@ -746,26 +747,50 @@ static struct mountpoint *lookup_mountpoint(struct dentry *dentry) return NULL; } -static struct mountpoint *new_mountpoint(struct dentry *dentry) +static struct mountpoint *get_mountpoint(struct dentry *dentry) { - struct hlist_head *chain = mp_hash(dentry); - struct mountpoint *mp; + struct mountpoint *mp, *new = NULL; int ret; - mp = kmalloc(sizeof(struct mountpoint), GFP_KERNEL); - if (!mp) + if (d_mountpoint(dentry)) { +mountpoint: + read_seqlock_excl(&mount_lock); + mp = lookup_mountpoint(dentry); + read_sequnlock_excl(&mount_lock); + if (mp) + goto done; + } + + if (!new) + new = kmalloc(sizeof(struct mountpoint), GFP_KERNEL); + if (!new) return ERR_PTR(-ENOMEM); + + /* Exactly one processes may set d_mounted */ ret = d_set_mounted(dentry); - if (ret) { - kfree(mp); - return ERR_PTR(ret); - } - mp->m_dentry = dentry; - mp->m_count = 1; - hlist_add_head(&mp->m_hash, chain); - INIT_HLIST_HEAD(&mp->m_list); + /* Someone else set d_mounted? */ + if (ret == -EBUSY) + goto mountpoint; + + /* The dentry is not available as a mountpoint? */ + mp = ERR_PTR(ret); + if (ret) + goto done; + + /* Add the new mountpoint to the hash table */ + read_seqlock_excl(&mount_lock); + new->m_dentry = dentry; + new->m_count = 1; + hlist_add_head(&new->m_hash, mp_hash(dentry)); + INIT_HLIST_HEAD(&new->m_list); + read_sequnlock_excl(&mount_lock); + + mp = new; + new = NULL; +done: + kfree(new); return mp; } @@ -948,11 +973,21 @@ vfs_kern_mount(struct file_system_type *type, int flags, const char *name, void if (!mnt) return ERR_PTR(-ENOMEM); + mnt->mnt.data = NULL; + if (type->alloc_mnt_data) { + mnt->mnt.data = type->alloc_mnt_data(); + if (!mnt->mnt.data) { + mnt_free_id(mnt); + free_vfsmnt(mnt); + return ERR_PTR(-ENOMEM); + } + } if (flags & MS_KERNMOUNT) mnt->mnt.mnt_flags = MNT_INTERNAL; - root = mount_fs(type, flags, name, data); + root = mount_fs(type, flags, name, &mnt->mnt, data); if (IS_ERR(root)) { + kfree(mnt->mnt.data); mnt_free_id(mnt); free_vfsmnt(mnt); return ERR_CAST(root); @@ -980,6 +1015,14 @@ static struct mount *clone_mnt(struct mount *old, struct dentry *root, if (!mnt) return ERR_PTR(-ENOMEM); + if (sb->s_op->clone_mnt_data) { + mnt->mnt.data = sb->s_op->clone_mnt_data(old->mnt.data); + if (!mnt->mnt.data) { + err = -ENOMEM; + goto out_free; + } + } + if (flag & (CL_SLAVE | CL_PRIVATE | CL_SHARED_TO_SLAVE)) mnt->mnt_group_id = 0; /* not a peer of original */ else @@ -1048,6 +1091,7 @@ static struct mount *clone_mnt(struct mount *old, struct dentry *root, return mnt; out_free: + kfree(mnt->mnt.data); mnt_free_id(mnt); free_vfsmnt(mnt); return ERR_PTR(err); @@ -1568,11 +1612,11 @@ void __detach_mounts(struct dentry *dentry) struct mount *mnt; namespace_lock(); + lock_mount_hash(); mp = lookup_mountpoint(dentry); if (IS_ERR_OR_NULL(mp)) goto out_unlock; - lock_mount_hash(); event++; while (!hlist_empty(&mp->m_list)) { mnt = hlist_entry(mp->m_list.first, struct mount, mnt_mp_list); @@ -1582,9 +1626,9 @@ void __detach_mounts(struct dentry *dentry) } else umount_tree(mnt, UMOUNT_CONNECTED); } - unlock_mount_hash(); put_mountpoint(mp); out_unlock: + unlock_mount_hash(); namespace_unlock(); } @@ -2013,9 +2057,7 @@ static struct mountpoint *lock_mount(struct path *path) namespace_lock(); mnt = lookup_mnt(path); if (likely(!mnt)) { - struct mountpoint *mp = lookup_mountpoint(dentry); - if (!mp) - mp = new_mountpoint(dentry); + struct mountpoint *mp = get_mountpoint(dentry); if (IS_ERR(mp)) { namespace_unlock(); inode_unlock(dentry->d_inode); @@ -2034,7 +2076,11 @@ static struct mountpoint *lock_mount(struct path *path) static void unlock_mount(struct mountpoint *where) { struct dentry *dentry = where->m_dentry; + + read_seqlock_excl(&mount_lock); put_mountpoint(where); + read_sequnlock_excl(&mount_lock); + namespace_unlock(); inode_unlock(dentry->d_inode); } @@ -2253,8 +2299,14 @@ static int do_remount(struct path *path, int flags, int mnt_flags, err = change_mount_flags(path->mnt, flags); else if (!capable(CAP_SYS_ADMIN)) err = -EPERM; - else - err = do_remount_sb(sb, flags, data, 0); + else { + err = do_remount_sb2(path->mnt, sb, flags, data, 0); + namespace_lock(); + lock_mount_hash(); + propagate_remount(mnt); + unlock_mount_hash(); + namespace_unlock(); + } if (!err) { lock_mount_hash(); mnt_flags |= mnt->mnt.mnt_flags & ~MNT_USER_SETTABLE_MASK; @@ -3110,9 +3162,9 @@ SYSCALL_DEFINE2(pivot_root, const char __user *, new_root, touch_mnt_namespace(current->nsproxy->mnt_ns); /* A moved mount should not expire automatically */ list_del_init(&new_mnt->mnt_expire); + put_mountpoint(root_mp); unlock_mount_hash(); chroot_fs_refs(&root, &new); - put_mountpoint(root_mp); error = 0; out4: unlock_mount(old_mp); diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c index 5f1af4cd1a336e..53e02b8bd9bd21 100644 --- a/fs/nfs/dir.c +++ b/fs/nfs/dir.c @@ -477,7 +477,7 @@ void nfs_force_use_readdirplus(struct inode *dir) { if (!list_empty(&NFS_I(dir)->open_files)) { nfs_advise_use_readdirplus(dir); - nfs_zap_mapping(dir, dir->i_mapping); + invalidate_mapping_pages(dir->i_mapping, 0, -1); } } @@ -886,17 +886,6 @@ int uncached_readdir(nfs_readdir_descriptor_t *desc) goto out; } -static bool nfs_dir_mapping_need_revalidate(struct inode *dir) -{ - struct nfs_inode *nfsi = NFS_I(dir); - - if (nfs_attribute_cache_expired(dir)) - return true; - if (nfsi->cache_validity & NFS_INO_INVALID_DATA) - return true; - return false; -} - /* The file offset position represents the dirent entry number. A last cookie cache takes care of the common case of reading the whole directory. @@ -928,7 +917,7 @@ static int nfs_readdir(struct file *file, struct dir_context *ctx) desc->decode = NFS_PROTO(inode)->decode_dirent; desc->plus = nfs_use_readdirplus(inode, ctx) ? 1 : 0; - if (ctx->pos == 0 || nfs_dir_mapping_need_revalidate(inode)) + if (ctx->pos == 0 || nfs_attribute_cache_expired(inode)) res = nfs_revalidate_mapping(inode, file->f_mapping); if (res < 0) goto out; diff --git a/fs/nfs/file.c b/fs/nfs/file.c index 9ea85ae23c320b..a1de8ef63e5699 100644 --- a/fs/nfs/file.c +++ b/fs/nfs/file.c @@ -374,7 +374,7 @@ static int nfs_write_end(struct file *file, struct address_space *mapping, */ if (!PageUptodate(page)) { unsigned pglen = nfs_page_length(page); - unsigned end = offset + len; + unsigned end = offset + copied; if (pglen == 0) { zero_user_segments(page, 0, offset, diff --git a/fs/nfs/filelayout/filelayoutdev.c b/fs/nfs/filelayout/filelayoutdev.c index 4946ef40ba875e..85ef38f9765f10 100644 --- a/fs/nfs/filelayout/filelayoutdev.c +++ b/fs/nfs/filelayout/filelayoutdev.c @@ -283,7 +283,8 @@ nfs4_fl_prepare_ds(struct pnfs_layout_segment *lseg, u32 ds_idx) s->nfs_client->cl_rpcclient->cl_auth->au_flavor); out_test_devid: - if (filelayout_test_devid_unavailable(devid)) + if (ret->ds_clp == NULL || + filelayout_test_devid_unavailable(devid)) ret = NULL; out: return ret; diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c index 98ace127bf861e..a5c38889e7ae7f 100644 --- a/fs/nfs/flexfilelayout/flexfilelayout.c +++ b/fs/nfs/flexfilelayout/flexfilelayout.c @@ -28,6 +28,9 @@ static struct group_info *ff_zero_group; +static void ff_layout_read_record_layoutstats_done(struct rpc_task *task, + struct nfs_pgio_header *hdr); + static struct pnfs_layout_hdr * ff_layout_alloc_layout_hdr(struct inode *inode, gfp_t gfp_flags) { @@ -1293,6 +1296,7 @@ static int ff_layout_read_done_cb(struct rpc_task *task, hdr->pgio_mirror_idx + 1, &hdr->pgio_mirror_idx)) goto out_eagain; + ff_layout_read_record_layoutstats_done(task, hdr); pnfs_read_resend_pnfs(hdr); return task->tk_status; case -NFS4ERR_RESET_TO_MDS: diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index 241da19b7da4a5..78ff8b63d5f70a 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -2678,7 +2678,8 @@ static inline void nfs4_exclusive_attrset(struct nfs4_opendata *opendata, sattr->ia_valid |= ATTR_MTIME; /* Except MODE, it seems harmless of setting twice. */ - if ((attrset[1] & FATTR4_WORD1_MODE)) + if (opendata->o_arg.createmode != NFS4_CREATE_EXCLUSIVE && + attrset[1] & FATTR4_WORD1_MODE) sattr->ia_valid &= ~ATTR_MODE; if (attrset[2] & FATTR4_WORD2_SECURITY_LABEL) @@ -8371,6 +8372,7 @@ nfs4_layoutget_handle_exception(struct rpc_task *task, goto out; } + nfs4_sequence_free_slot(&lgp->res.seq_res); err = nfs4_handle_exception(server, nfs4err, exception); if (!status) { if (exception->retry) diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c index 259ef85f435aa7..415d7e69bc5ec1 100644 --- a/fs/nfs/pnfs.c +++ b/fs/nfs/pnfs.c @@ -299,6 +299,14 @@ pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo) } } +static void +pnfs_clear_layoutreturn_info(struct pnfs_layout_hdr *lo) +{ + lo->plh_return_iomode = 0; + lo->plh_return_seq = 0; + clear_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags); +} + /* * Mark a pnfs_layout_hdr and all associated layout segments as invalid * @@ -317,6 +325,7 @@ pnfs_mark_layout_stateid_invalid(struct pnfs_layout_hdr *lo, }; set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags); + pnfs_clear_layoutreturn_info(lo); return pnfs_mark_matching_lsegs_invalid(lo, lseg_list, &range, 0); } @@ -411,7 +420,9 @@ pnfs_layout_remove_lseg(struct pnfs_layout_hdr *lo, list_del_init(&lseg->pls_list); /* Matched by pnfs_get_layout_hdr in pnfs_layout_insert_lseg */ atomic_dec(&lo->plh_refcount); - if (list_empty(&lo->plh_segs)) { + if (list_empty(&lo->plh_segs) && + !test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags) && + !test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) { if (atomic_read(&lo->plh_outstanding) == 0) set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags); clear_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags); @@ -816,14 +827,6 @@ pnfs_destroy_all_layouts(struct nfs_client *clp) pnfs_destroy_layouts_byclid(clp, false); } -static void -pnfs_clear_layoutreturn_info(struct pnfs_layout_hdr *lo) -{ - lo->plh_return_iomode = 0; - lo->plh_return_seq = 0; - clear_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags); -} - /* update lo->plh_stateid with new if is more recent */ void pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo, const nfs4_stateid *new, @@ -944,6 +947,7 @@ static void pnfs_clear_layoutcommit(struct inode *inode, void pnfs_clear_layoutreturn_waitbit(struct pnfs_layout_hdr *lo) { clear_bit_unlock(NFS_LAYOUT_RETURN, &lo->plh_flags); + clear_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags); smp_mb__after_atomic(); wake_up_bit(&lo->plh_flags, NFS_LAYOUT_RETURN); rpc_wake_up(&NFS_SERVER(lo->plh_inode)->roc_rpcwaitq); @@ -957,8 +961,9 @@ pnfs_prepare_layoutreturn(struct pnfs_layout_hdr *lo, /* Serialise LAYOUTGET/LAYOUTRETURN */ if (atomic_read(&lo->plh_outstanding) != 0) return false; - if (test_and_set_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) + if (test_and_set_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) return false; + set_bit(NFS_LAYOUT_RETURN, &lo->plh_flags); pnfs_get_layout_hdr(lo); if (test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) { if (stateid != NULL) { @@ -1252,13 +1257,11 @@ bool pnfs_wait_on_layoutreturn(struct inode *ino, struct rpc_task *task) * i_lock */ spin_lock(&ino->i_lock); lo = nfsi->layout; - if (lo && test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) + if (lo && test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) { + rpc_sleep_on(&NFS_SERVER(ino)->roc_rpcwaitq, task, NULL); sleep = true; + } spin_unlock(&ino->i_lock); - - if (sleep) - rpc_sleep_on(&NFS_SERVER(ino)->roc_rpcwaitq, task, NULL); - return sleep; } @@ -1950,6 +1953,8 @@ void pnfs_error_mark_layout_for_return(struct inode *inode, spin_lock(&inode->i_lock); pnfs_set_plh_return_info(lo, range.iomode, 0); + /* Block LAYOUTGET */ + set_bit(NFS_LAYOUT_RETURN, &lo->plh_flags); /* * mark all matching lsegs so that we are sure to have no live * segments at hand when sending layoutreturn. See pnfs_put_lseg() @@ -2286,6 +2291,10 @@ void pnfs_read_resend_pnfs(struct nfs_pgio_header *hdr) struct nfs_pageio_descriptor pgio; if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { + /* Prevent deadlocks with layoutreturn! */ + pnfs_put_lseg(hdr->lseg); + hdr->lseg = NULL; + nfs_pageio_init_read(&pgio, hdr->inode, false, hdr->completion_ops); hdr->task.tk_status = nfs_pageio_resend(&pgio, hdr); diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h index 5c295512c9671e..44cad8afda0e4e 100644 --- a/fs/nfs/pnfs.h +++ b/fs/nfs/pnfs.h @@ -96,6 +96,7 @@ enum { NFS_LAYOUT_RW_FAILED, /* get rw layout failed stop trying */ NFS_LAYOUT_BULK_RECALL, /* bulk recall affecting layout */ NFS_LAYOUT_RETURN, /* layoutreturn in progress */ + NFS_LAYOUT_RETURN_LOCK, /* Serialise layoutreturn */ NFS_LAYOUT_RETURN_REQUESTED, /* Return this layout ASAP */ NFS_LAYOUT_INVALID_STID, /* layout stateid id is invalid */ NFS_LAYOUT_FIRST_LAYOUTGET, /* Serialize first layoutget */ diff --git a/fs/nfs/super.c b/fs/nfs/super.c index 001796bcd6c895..ddce94ce8142dd 100644 --- a/fs/nfs/super.c +++ b/fs/nfs/super.c @@ -2904,7 +2904,7 @@ module_param(max_session_slots, ushort, 0644); MODULE_PARM_DESC(max_session_slots, "Maximum number of outstanding NFSv4.1 " "requests the client will negotiate"); module_param(max_session_cb_slots, ushort, 0644); -MODULE_PARM_DESC(max_session_slots, "Maximum number of parallel NFSv4.1 " +MODULE_PARM_DESC(max_session_cb_slots, "Maximum number of parallel NFSv4.1 " "callbacks the client will process for a given server"); module_param(send_implementation_id, ushort, 0644); MODULE_PARM_DESC(send_implementation_id, diff --git a/fs/nfsd/nfs4layouts.c b/fs/nfsd/nfs4layouts.c index 42aace4fc4c807..64813697f4c4e2 100644 --- a/fs/nfsd/nfs4layouts.c +++ b/fs/nfsd/nfs4layouts.c @@ -223,10 +223,11 @@ nfsd4_alloc_layout_stateid(struct nfsd4_compound_state *cstate, struct nfs4_layout_stateid *ls; struct nfs4_stid *stp; - stp = nfs4_alloc_stid(cstate->clp, nfs4_layout_stateid_cache); + stp = nfs4_alloc_stid(cstate->clp, nfs4_layout_stateid_cache, + nfsd4_free_layout_stateid); if (!stp) return NULL; - stp->sc_free = nfsd4_free_layout_stateid; + get_nfs4_file(fp); stp->sc_file = fp; diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index 4b4beaaa4eaac0..a0dee8ae9f97f1 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -633,8 +633,8 @@ find_or_hash_clnt_odstate(struct nfs4_file *fp, struct nfs4_clnt_odstate *new) return co; } -struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, - struct kmem_cache *slab) +struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *slab, + void (*sc_free)(struct nfs4_stid *)) { struct nfs4_stid *stid; int new_id; @@ -650,6 +650,8 @@ struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, idr_preload_end(); if (new_id < 0) goto out_free; + + stid->sc_free = sc_free; stid->sc_client = cl; stid->sc_stateid.si_opaque.so_id = new_id; stid->sc_stateid.si_opaque.so_clid = cl->cl_clientid; @@ -675,15 +677,12 @@ struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, static struct nfs4_ol_stateid * nfs4_alloc_open_stateid(struct nfs4_client *clp) { struct nfs4_stid *stid; - struct nfs4_ol_stateid *stp; - stid = nfs4_alloc_stid(clp, stateid_slab); + stid = nfs4_alloc_stid(clp, stateid_slab, nfs4_free_ol_stateid); if (!stid) return NULL; - stp = openlockstateid(stid); - stp->st_stid.sc_free = nfs4_free_ol_stateid; - return stp; + return openlockstateid(stid); } static void nfs4_free_deleg(struct nfs4_stid *stid) @@ -781,11 +780,10 @@ alloc_init_deleg(struct nfs4_client *clp, struct svc_fh *current_fh, goto out_dec; if (delegation_blocked(¤t_fh->fh_handle)) goto out_dec; - dp = delegstateid(nfs4_alloc_stid(clp, deleg_slab)); + dp = delegstateid(nfs4_alloc_stid(clp, deleg_slab, nfs4_free_deleg)); if (dp == NULL) goto out_dec; - dp->dl_stid.sc_free = nfs4_free_deleg; /* * delegation seqid's are never incremented. The 4.1 special * meaning of seqid 0 isn't meaningful, really, but let's avoid @@ -5580,7 +5578,6 @@ init_lock_stateid(struct nfs4_ol_stateid *stp, struct nfs4_lockowner *lo, stp->st_stateowner = nfs4_get_stateowner(&lo->lo_owner); get_nfs4_file(fp); stp->st_stid.sc_file = fp; - stp->st_stid.sc_free = nfs4_free_lock_stateid; stp->st_access_bmap = 0; stp->st_deny_bmap = open_stp->st_deny_bmap; stp->st_openstp = open_stp; @@ -5623,7 +5620,7 @@ find_or_create_lock_stateid(struct nfs4_lockowner *lo, struct nfs4_file *fi, lst = find_lock_stateid(lo, fi); if (lst == NULL) { spin_unlock(&clp->cl_lock); - ns = nfs4_alloc_stid(clp, stateid_slab); + ns = nfs4_alloc_stid(clp, stateid_slab, nfs4_free_lock_stateid); if (ns == NULL) return NULL; diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h index c9399366f9dfc7..4516e8b7d77630 100644 --- a/fs/nfsd/state.h +++ b/fs/nfsd/state.h @@ -603,8 +603,8 @@ extern __be32 nfs4_preprocess_stateid_op(struct svc_rqst *rqstp, __be32 nfsd4_lookup_stateid(struct nfsd4_compound_state *cstate, stateid_t *stateid, unsigned char typemask, struct nfs4_stid **s, struct nfsd_net *nn); -struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, - struct kmem_cache *slab); +struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *slab, + void (*sc_free)(struct nfs4_stid *)); void nfs4_unhash_stid(struct nfs4_stid *s); void nfs4_put_stid(struct nfs4_stid *s); void nfs4_inc_and_copy_stateid(stateid_t *dst, struct nfs4_stid *stid); diff --git a/fs/notify/fanotify/fanotify_user.c b/fs/notify/fanotify/fanotify_user.c index 7ebfca6a14272e..7f99c96014b3b4 100644 --- a/fs/notify/fanotify/fanotify_user.c +++ b/fs/notify/fanotify/fanotify_user.c @@ -488,7 +488,7 @@ static int fanotify_find_path(int dfd, const char __user *filename, } /* you can only watch an inode if you have read permissions on it */ - ret = inode_permission(path->dentry->d_inode, MAY_READ); + ret = inode_permission2(path->mnt, path->dentry->d_inode, MAY_READ); if (ret) path_put(path); out: diff --git a/fs/notify/inode_mark.c b/fs/notify/inode_mark.c index 741077deef3b55..a3645249f7ecfa 100644 --- a/fs/notify/inode_mark.c +++ b/fs/notify/inode_mark.c @@ -150,12 +150,10 @@ int fsnotify_add_inode_mark(struct fsnotify_mark *mark, */ void fsnotify_unmount_inodes(struct super_block *sb) { - struct inode *inode, *next_i, *need_iput = NULL; + struct inode *inode, *iput_inode = NULL; spin_lock(&sb->s_inode_list_lock); - list_for_each_entry_safe(inode, next_i, &sb->s_inodes, i_sb_list) { - struct inode *need_iput_tmp; - + list_for_each_entry(inode, &sb->s_inodes, i_sb_list) { /* * We cannot __iget() an inode in state I_FREEING, * I_WILL_FREE, or I_NEW which is fine because by that point @@ -178,49 +176,24 @@ void fsnotify_unmount_inodes(struct super_block *sb) continue; } - need_iput_tmp = need_iput; - need_iput = NULL; - - /* In case fsnotify_inode_delete() drops a reference. */ - if (inode != need_iput_tmp) - __iget(inode); - else - need_iput_tmp = NULL; + __iget(inode); spin_unlock(&inode->i_lock); - - /* In case the dropping of a reference would nuke next_i. */ - while (&next_i->i_sb_list != &sb->s_inodes) { - spin_lock(&next_i->i_lock); - if (!(next_i->i_state & (I_FREEING | I_WILL_FREE)) && - atomic_read(&next_i->i_count)) { - __iget(next_i); - need_iput = next_i; - spin_unlock(&next_i->i_lock); - break; - } - spin_unlock(&next_i->i_lock); - next_i = list_next_entry(next_i, i_sb_list); - } - - /* - * We can safely drop s_inode_list_lock here because either - * we actually hold references on both inode and next_i or - * end of list. Also no new inodes will be added since the - * umount has begun. - */ spin_unlock(&sb->s_inode_list_lock); - if (need_iput_tmp) - iput(need_iput_tmp); + if (iput_inode) + iput(iput_inode); /* for each watch, send FS_UNMOUNT and then remove it */ fsnotify(inode, FS_UNMOUNT, inode, FSNOTIFY_EVENT_INODE, NULL, 0); fsnotify_inode_delete(inode); - iput(inode); + iput_inode = inode; spin_lock(&sb->s_inode_list_lock); } spin_unlock(&sb->s_inode_list_lock); + + if (iput_inode) + iput(iput_inode); } diff --git a/fs/notify/inotify/inotify_user.c b/fs/notify/inotify/inotify_user.c index 69d1ea3d292a86..4da5c6a1134fc3 100644 --- a/fs/notify/inotify/inotify_user.c +++ b/fs/notify/inotify/inotify_user.c @@ -337,7 +337,7 @@ static int inotify_find_inode(const char __user *dirname, struct path *path, uns if (error) return error; /* you can only watch an inode if you have read permissions on it */ - error = inode_permission(path->dentry->d_inode, MAY_READ); + error = inode_permission2(path->mnt, path->dentry->d_inode, MAY_READ); if (error) path_put(path); return error; @@ -702,6 +702,8 @@ SYSCALL_DEFINE3(inotify_add_watch, int, fd, const char __user *, pathname, struct fsnotify_group *group; struct inode *inode; struct path path; + struct path alteredpath; + struct path *canonical_path = &path; struct fd f; int ret; unsigned flags = 0; @@ -741,13 +743,22 @@ SYSCALL_DEFINE3(inotify_add_watch, int, fd, const char __user *, pathname, if (ret) goto fput_and_out; + /* support stacked filesystems */ + if(path.dentry && path.dentry->d_op) { + if (path.dentry->d_op->d_canonical_path) { + path.dentry->d_op->d_canonical_path(&path, &alteredpath); + canonical_path = &alteredpath; + path_put(&path); + } + } + /* inode held in place by reference to path; group by fget on fd */ - inode = path.dentry->d_inode; + inode = canonical_path->dentry->d_inode; group = f.file->private_data; /* create/update an inode mark */ ret = inotify_update_watch(group, inode, mask); - path_put(&path); + path_put(canonical_path); fput_and_out: fdput(f); return ret; diff --git a/fs/ocfs2/dlmglue.c b/fs/ocfs2/dlmglue.c index 83d576f6a287b1..77d1632e905d8b 100644 --- a/fs/ocfs2/dlmglue.c +++ b/fs/ocfs2/dlmglue.c @@ -3303,6 +3303,16 @@ static int ocfs2_downconvert_lock(struct ocfs2_super *osb, mlog(ML_BASTS, "lockres %s, level %d => %d\n", lockres->l_name, lockres->l_level, new_level); + /* + * On DLM_LKF_VALBLK, fsdlm behaves differently with o2cb. It always + * expects DLM_LKF_VALBLK being set if the LKB has LVB, so that + * we can recover correctly from node failure. Otherwise, we may get + * invalid LVB in LKB, but without DLM_SBF_VALNOTVALID being set. + */ + if (!ocfs2_is_o2cb_active() && + lockres->l_ops->flags & LOCK_TYPE_USES_LVB) + lvb = 1; + if (lvb) dlm_flags |= DLM_LKF_VALBLK; diff --git a/fs/ocfs2/stackglue.c b/fs/ocfs2/stackglue.c index 52c07346bea3f8..820359096c7aa8 100644 --- a/fs/ocfs2/stackglue.c +++ b/fs/ocfs2/stackglue.c @@ -48,6 +48,12 @@ static char ocfs2_hb_ctl_path[OCFS2_MAX_HB_CTL_PATH] = "/sbin/ocfs2_hb_ctl"; */ static struct ocfs2_stack_plugin *active_stack; +inline int ocfs2_is_o2cb_active(void) +{ + return !strcmp(active_stack->sp_name, OCFS2_STACK_PLUGIN_O2CB); +} +EXPORT_SYMBOL_GPL(ocfs2_is_o2cb_active); + static struct ocfs2_stack_plugin *ocfs2_stack_lookup(const char *name) { struct ocfs2_stack_plugin *p; diff --git a/fs/ocfs2/stackglue.h b/fs/ocfs2/stackglue.h index f2dce10fae543c..e3036e1790e86d 100644 --- a/fs/ocfs2/stackglue.h +++ b/fs/ocfs2/stackglue.h @@ -298,6 +298,9 @@ void ocfs2_stack_glue_set_max_proto_version(struct ocfs2_protocol_version *max_p int ocfs2_stack_glue_register(struct ocfs2_stack_plugin *plugin); void ocfs2_stack_glue_unregister(struct ocfs2_stack_plugin *plugin); +/* In ocfs2_downconvert_lock(), we need to know which stack we are using */ +int ocfs2_is_o2cb_active(void); + extern struct kset *ocfs2_kset; #endif /* STACKGLUE_H */ diff --git a/fs/open.c b/fs/open.c index d3ed8171e8e092..568749b035faf3 100644 --- a/fs/open.c +++ b/fs/open.c @@ -34,8 +34,8 @@ #include "internal.h" -int do_truncate(struct dentry *dentry, loff_t length, unsigned int time_attrs, - struct file *filp) +int do_truncate2(struct vfsmount *mnt, struct dentry *dentry, loff_t length, + unsigned int time_attrs, struct file *filp) { int ret; struct iattr newattrs; @@ -60,18 +60,25 @@ int do_truncate(struct dentry *dentry, loff_t length, unsigned int time_attrs, inode_lock(dentry->d_inode); /* Note any delegations or leases have already been broken: */ - ret = notify_change(dentry, &newattrs, NULL); + ret = notify_change2(mnt, dentry, &newattrs, NULL); inode_unlock(dentry->d_inode); return ret; } +int do_truncate(struct dentry *dentry, loff_t length, unsigned int time_attrs, + struct file *filp) +{ + return do_truncate2(NULL, dentry, length, time_attrs, filp); +} long vfs_truncate(const struct path *path, loff_t length) { struct inode *inode; + struct vfsmount *mnt; struct dentry *upperdentry; long error; inode = path->dentry->d_inode; + mnt = path->mnt; /* For directories it's -EISDIR, for other non-regulars - -EINVAL */ if (S_ISDIR(inode->i_mode)) @@ -83,7 +90,7 @@ long vfs_truncate(const struct path *path, loff_t length) if (error) goto out; - error = inode_permission(inode, MAY_WRITE); + error = inode_permission2(mnt, inode, MAY_WRITE); if (error) goto mnt_drop_write_and_out; @@ -117,7 +124,7 @@ long vfs_truncate(const struct path *path, loff_t length) if (!error) error = security_path_truncate(path); if (!error) - error = do_truncate(path->dentry, length, 0, NULL); + error = do_truncate2(mnt, path->dentry, length, 0, NULL); put_write_and_out: put_write_access(upperdentry->d_inode); @@ -166,6 +173,7 @@ static long do_sys_ftruncate(unsigned int fd, loff_t length, int small) { struct inode *inode; struct dentry *dentry; + struct vfsmount *mnt; struct fd f; int error; @@ -182,6 +190,7 @@ static long do_sys_ftruncate(unsigned int fd, loff_t length, int small) small = 0; dentry = f.file->f_path.dentry; + mnt = f.file->f_path.mnt; inode = dentry->d_inode; error = -EINVAL; if (!S_ISREG(inode->i_mode) || !(f.file->f_mode & FMODE_WRITE)) @@ -201,7 +210,7 @@ static long do_sys_ftruncate(unsigned int fd, loff_t length, int small) if (!error) error = security_path_truncate(&f.file->f_path); if (!error) - error = do_truncate(dentry, length, ATTR_MTIME|ATTR_CTIME, f.file); + error = do_truncate2(mnt, dentry, length, ATTR_MTIME|ATTR_CTIME, f.file); sb_end_write(inode->i_sb); out_putf: fdput(f); @@ -357,6 +366,7 @@ SYSCALL_DEFINE3(faccessat, int, dfd, const char __user *, filename, int, mode) struct cred *override_cred; struct path path; struct inode *inode; + struct vfsmount *mnt; int res; unsigned int lookup_flags = LOOKUP_FOLLOW; @@ -387,6 +397,7 @@ SYSCALL_DEFINE3(faccessat, int, dfd, const char __user *, filename, int, mode) goto out; inode = d_backing_inode(path.dentry); + mnt = path.mnt; if ((mode & MAY_EXEC) && S_ISREG(inode->i_mode)) { /* @@ -398,7 +409,7 @@ SYSCALL_DEFINE3(faccessat, int, dfd, const char __user *, filename, int, mode) goto out_path_release; } - res = inode_permission(inode, mode | MAY_ACCESS); + res = inode_permission2(mnt, inode, mode | MAY_ACCESS); /* SuS v2 requires we report a read only fs too */ if (res || !(mode & S_IWOTH) || special_file(inode->i_mode)) goto out_path_release; @@ -442,7 +453,7 @@ SYSCALL_DEFINE1(chdir, const char __user *, filename) if (error) goto out; - error = inode_permission(path.dentry->d_inode, MAY_EXEC | MAY_CHDIR); + error = inode_permission2(path.mnt, path.dentry->d_inode, MAY_EXEC | MAY_CHDIR); if (error) goto dput_and_out; @@ -462,6 +473,7 @@ SYSCALL_DEFINE1(fchdir, unsigned int, fd) { struct fd f = fdget_raw(fd); struct inode *inode; + struct vfsmount *mnt; int error = -EBADF; error = -EBADF; @@ -469,12 +481,13 @@ SYSCALL_DEFINE1(fchdir, unsigned int, fd) goto out; inode = file_inode(f.file); + mnt = f.file->f_path.mnt; error = -ENOTDIR; if (!S_ISDIR(inode->i_mode)) goto out_putf; - error = inode_permission(inode, MAY_EXEC | MAY_CHDIR); + error = inode_permission2(mnt, inode, MAY_EXEC | MAY_CHDIR); if (!error) set_fs_pwd(current->fs, &f.file->f_path); out_putf: @@ -493,7 +506,7 @@ SYSCALL_DEFINE1(chroot, const char __user *, filename) if (error) goto out; - error = inode_permission(path.dentry->d_inode, MAY_EXEC | MAY_CHDIR); + error = inode_permission2(path.mnt, path.dentry->d_inode, MAY_EXEC | MAY_CHDIR); if (error) goto dput_and_out; @@ -533,7 +546,7 @@ static int chmod_common(const struct path *path, umode_t mode) goto out_unlock; newattrs.ia_mode = (mode & S_IALLUGO) | (inode->i_mode & ~S_IALLUGO); newattrs.ia_valid = ATTR_MODE | ATTR_CTIME; - error = notify_change(path->dentry, &newattrs, &delegated_inode); + error = notify_change2(path->mnt, path->dentry, &newattrs, &delegated_inode); out_unlock: inode_unlock(inode); if (delegated_inode) { @@ -613,7 +626,7 @@ static int chown_common(const struct path *path, uid_t user, gid_t group) inode_lock(inode); error = security_path_chown(path, uid, gid); if (!error) - error = notify_change(path->dentry, &newattrs, &delegated_inode); + error = notify_change2(path->mnt, path->dentry, &newattrs, &delegated_inode); inode_unlock(inode); if (delegated_inode) { error = break_deleg_wait(&delegated_inode); diff --git a/fs/pnode.c b/fs/pnode.c index 234a9ac49958ed..83b5bb1fdbb8a1 100644 --- a/fs/pnode.c +++ b/fs/pnode.c @@ -458,3 +458,32 @@ int propagate_umount(struct list_head *list) __propagate_umount(mnt); return 0; } + +/* + * Iterates over all slaves, and slaves of slaves. + */ +static struct mount *next_descendent(struct mount *root, struct mount *cur) +{ + if (!IS_MNT_NEW(cur) && !list_empty(&cur->mnt_slave_list)) + return first_slave(cur); + do { + if (cur->mnt_slave.next != &cur->mnt_master->mnt_slave_list) + return next_slave(cur); + cur = cur->mnt_master; + } while (cur != root); + return NULL; +} + +void propagate_remount(struct mount *mnt) +{ + struct mount *m = mnt; + struct super_block *sb = mnt->mnt.mnt_sb; + + if (sb->s_op->copy_mnt_data) { + m = next_descendent(mnt, m); + while (m) { + sb->s_op->copy_mnt_data(m->mnt.data, mnt->mnt.data); + m = next_descendent(mnt, m); + } + } +} diff --git a/fs/pnode.h b/fs/pnode.h index 550f5a8b4fcf46..03a8001c4af2d5 100644 --- a/fs/pnode.h +++ b/fs/pnode.h @@ -44,6 +44,7 @@ int propagate_mnt(struct mount *, struct mountpoint *, struct mount *, int propagate_umount(struct list_head *); int propagate_mount_busy(struct mount *, int); void propagate_mount_unlock(struct mount *); +void propagate_remount(struct mount *); void mnt_release_group_id(struct mount *); int get_dominating_id(struct mount *mnt, const struct path *root); unsigned int mnt_get_count(struct mount *mnt); diff --git a/fs/posix_acl.c b/fs/posix_acl.c index 595522022aca04..c9d48dc784953f 100644 --- a/fs/posix_acl.c +++ b/fs/posix_acl.c @@ -922,11 +922,10 @@ int simple_set_acl(struct inode *inode, struct posix_acl *acl, int type) int error; if (type == ACL_TYPE_ACCESS) { - error = posix_acl_equiv_mode(acl, &inode->i_mode); - if (error < 0) - return 0; - if (error == 0) - acl = NULL; + error = posix_acl_update_mode(inode, + &inode->i_mode, &acl); + if (error) + return error; } inode->i_ctime = current_time(inode); diff --git a/fs/proc/base.c b/fs/proc/base.c index ca651ac0066088..c01eeaade636ec 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -2912,8 +2912,8 @@ static const struct pid_entry tgid_base_stuff[] = { ONE("cgroup", S_IRUGO, proc_cgroup_show), #endif ONE("oom_score", S_IRUGO, proc_oom_score), - REG("oom_adj", S_IRUGO|S_IWUSR, proc_oom_adj_operations), - REG("oom_score_adj", S_IRUGO|S_IWUSR, proc_oom_score_adj_operations), + REG("oom_adj", S_IRUSR, proc_oom_adj_operations), + REG("oom_score_adj", S_IRUSR, proc_oom_score_adj_operations), #ifdef CONFIG_AUDITSYSCALL REG("loginuid", S_IWUSR|S_IRUGO, proc_loginuid_operations), REG("sessionid", S_IRUGO, proc_sessionid_operations), @@ -3301,8 +3301,8 @@ static const struct pid_entry tid_base_stuff[] = { ONE("cgroup", S_IRUGO, proc_cgroup_show), #endif ONE("oom_score", S_IRUGO, proc_oom_score), - REG("oom_adj", S_IRUGO|S_IWUSR, proc_oom_adj_operations), - REG("oom_score_adj", S_IRUGO|S_IWUSR, proc_oom_score_adj_operations), + REG("oom_adj", S_IRUSR, proc_oom_adj_operations), + REG("oom_score_adj", S_IRUSR, proc_oom_score_adj_operations), #ifdef CONFIG_AUDITSYSCALL REG("loginuid", S_IWUSR|S_IRUGO, proc_loginuid_operations), REG("sessionid", S_IRUGO, proc_sessionid_operations), diff --git a/fs/proc/proc_sysctl.c b/fs/proc/proc_sysctl.c index 55313d9948954b..d4e37acd48217d 100644 --- a/fs/proc/proc_sysctl.c +++ b/fs/proc/proc_sysctl.c @@ -709,7 +709,7 @@ static int proc_sys_readdir(struct file *file, struct dir_context *ctx) ctl_dir = container_of(head, struct ctl_dir, header); if (!dir_emit_dots(file, ctx)) - return 0; + goto out; pos = 2; @@ -719,6 +719,7 @@ static int proc_sys_readdir(struct file *file, struct dir_context *ctx) break; } } +out: sysctl_head_finish(head); return 0; } diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 35b92d81692f09..65d28f9ed232ae 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -127,6 +127,56 @@ static void release_task_mempolicy(struct proc_maps_private *priv) } #endif +static void seq_print_vma_name(struct seq_file *m, struct vm_area_struct *vma) +{ + const char __user *name = vma_get_anon_name(vma); + struct mm_struct *mm = vma->vm_mm; + + unsigned long page_start_vaddr; + unsigned long page_offset; + unsigned long num_pages; + unsigned long max_len = NAME_MAX; + int i; + + page_start_vaddr = (unsigned long)name & PAGE_MASK; + page_offset = (unsigned long)name - page_start_vaddr; + num_pages = DIV_ROUND_UP(page_offset + max_len, PAGE_SIZE); + + seq_puts(m, "[anon:"); + + for (i = 0; i < num_pages; i++) { + int len; + int write_len; + const char *kaddr; + long pages_pinned; + struct page *page; + + pages_pinned = get_user_pages_remote(current, mm, + page_start_vaddr, 1, 0, &page, NULL); + if (pages_pinned < 1) { + seq_puts(m, "]"); + return; + } + + kaddr = (const char *)kmap(page); + len = min(max_len, PAGE_SIZE - page_offset); + write_len = strnlen(kaddr + page_offset, len); + seq_write(m, kaddr + page_offset, write_len); + kunmap(page); + put_page(page); + + /* if strnlen hit a null terminator then we're done */ + if (write_len != len) + break; + + max_len -= len; + page_offset = 0; + page_start_vaddr += PAGE_SIZE; + } + + seq_putc(m, ']'); +} + static void vma_stop(struct proc_maps_private *priv) { struct mm_struct *mm = priv->mm; @@ -345,8 +395,15 @@ show_map_vma(struct seq_file *m, struct vm_area_struct *vma, int is_pid) goto done; } - if (is_stack(priv, vma)) + if (is_stack(priv, vma)) { name = "[stack]"; + goto done; + } + + if (vma_get_anon_name(vma)) { + seq_pad(m, ' '); + seq_print_vma_name(m, vma); + } } done: @@ -760,6 +817,12 @@ static int show_smap(struct seq_file *m, void *v, int is_pid) show_map_vma(m, vma, is_pid); + if (vma_get_anon_name(vma)) { + seq_puts(m, "Name: "); + seq_print_vma_name(m, vma); + seq_putc(m, '\n'); + } + seq_printf(m, "Size: %8lu kB\n" "Rss: %8lu kB\n" diff --git a/fs/proc_namespace.c b/fs/proc_namespace.c index 3f1190d1899153..6863773aff2527 100644 --- a/fs/proc_namespace.c +++ b/fs/proc_namespace.c @@ -118,7 +118,9 @@ static int show_vfsmnt(struct seq_file *m, struct vfsmount *mnt) if (err) goto out; show_mnt_opts(m, mnt); - if (sb->s_op->show_options) + if (sb->s_op->show_options2) + err = sb->s_op->show_options2(mnt, m, mnt_path.dentry); + else if (sb->s_op->show_options) err = sb->s_op->show_options(m, mnt_path.dentry); seq_puts(m, " 0 0\n"); out: @@ -180,7 +182,9 @@ static int show_mountinfo(struct seq_file *m, struct vfsmount *mnt) err = show_sb_opts(m, sb); if (err) goto out; - if (sb->s_op->show_options) + if (sb->s_op->show_options2) { + err = sb->s_op->show_options2(mnt, m, mnt->mnt_root); + } else if (sb->s_op->show_options) err = sb->s_op->show_options(m, mnt->mnt_root); seq_putc(m, '\n'); out: diff --git a/fs/pstore/ram.c b/fs/pstore/ram.c index 6ad831b9d1b87a..bd2dc34da78905 100644 --- a/fs/pstore/ram.c +++ b/fs/pstore/ram.c @@ -549,6 +549,12 @@ static int ramoops_parse_dt(struct platform_device *pdev, return 0; } +void notrace ramoops_console_write_buf(const char *buf, size_t size) +{ + struct ramoops_context *cxt = &oops_cxt; + persistent_ram_write(cxt->cprz, buf, size); +} + static int ramoops_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; diff --git a/fs/sdcardfs/Kconfig b/fs/sdcardfs/Kconfig new file mode 100644 index 00000000000000..a1c103316ac794 --- /dev/null +++ b/fs/sdcardfs/Kconfig @@ -0,0 +1,13 @@ +config SDCARD_FS + tristate "sdcard file system" + depends on CONFIGFS_FS + default n + help + Sdcardfs is based on Wrapfs file system. + +config SDCARD_FS_FADV_NOACTIVE + bool "sdcardfs fadvise noactive support" + depends on FADV_NOACTIVE + default y + help + Sdcardfs supports fadvise noactive mode. diff --git a/fs/sdcardfs/Makefile b/fs/sdcardfs/Makefile new file mode 100644 index 00000000000000..b84fbb2b45a47a --- /dev/null +++ b/fs/sdcardfs/Makefile @@ -0,0 +1,7 @@ +SDCARDFS_VERSION="0.1" + +EXTRA_CFLAGS += -DSDCARDFS_VERSION=\"$(SDCARDFS_VERSION)\" + +obj-$(CONFIG_SDCARD_FS) += sdcardfs.o + +sdcardfs-y := dentry.o file.o inode.o main.o super.o lookup.o mmap.o packagelist.o derived_perm.o diff --git a/fs/sdcardfs/dentry.c b/fs/sdcardfs/dentry.c new file mode 100644 index 00000000000000..f22de8add10ce3 --- /dev/null +++ b/fs/sdcardfs/dentry.c @@ -0,0 +1,185 @@ +/* + * fs/sdcardfs/dentry.c + * + * Copyright (c) 2013 Samsung Electronics Co. Ltd + * Authors: Daeho Jeong, Woojoong Lee, Seunghwan Hyun, + * Sunghwan Yun, Sungjong Seo + * + * This program has been developed as a stackable file system based on + * the WrapFS which written by + * + * Copyright (c) 1998-2011 Erez Zadok + * Copyright (c) 2009 Shrikar Archak + * Copyright (c) 2003-2011 Stony Brook University + * Copyright (c) 2003-2011 The Research Foundation of SUNY + * + * This file is dual licensed. It may be redistributed and/or modified + * under the terms of the Apache 2.0 License OR version 2 of the GNU + * General Public License. + */ + +#include "sdcardfs.h" +#include "linux/ctype.h" + +/* + * returns: -ERRNO if error (returned to user) + * 0: tell VFS to invalidate dentry + * 1: dentry is valid + */ +static int sdcardfs_d_revalidate(struct dentry *dentry, unsigned int flags) +{ + int err = 1; + struct path parent_lower_path, lower_path; + struct dentry *parent_dentry = NULL; + struct dentry *parent_lower_dentry = NULL; + struct dentry *lower_cur_parent_dentry = NULL; + struct dentry *lower_dentry = NULL; + + if (flags & LOOKUP_RCU) + return -ECHILD; + + spin_lock(&dentry->d_lock); + if (IS_ROOT(dentry)) { + spin_unlock(&dentry->d_lock); + return 1; + } + spin_unlock(&dentry->d_lock); + + /* check uninitialized obb_dentry and + * whether the base obbpath has been changed or not */ + if (is_obbpath_invalid(dentry)) { + d_drop(dentry); + return 0; + } + + parent_dentry = dget_parent(dentry); + sdcardfs_get_lower_path(parent_dentry, &parent_lower_path); + sdcardfs_get_real_lower(dentry, &lower_path); + parent_lower_dentry = parent_lower_path.dentry; + lower_dentry = lower_path.dentry; + lower_cur_parent_dentry = dget_parent(lower_dentry); + + spin_lock(&lower_dentry->d_lock); + if (d_unhashed(lower_dentry)) { + spin_unlock(&lower_dentry->d_lock); + d_drop(dentry); + err = 0; + goto out; + } + spin_unlock(&lower_dentry->d_lock); + + if (parent_lower_dentry != lower_cur_parent_dentry) { + d_drop(dentry); + err = 0; + goto out; + } + + if (dentry < lower_dentry) { + spin_lock(&dentry->d_lock); + spin_lock(&lower_dentry->d_lock); + } else { + spin_lock(&lower_dentry->d_lock); + spin_lock(&dentry->d_lock); + } + + if (dentry->d_name.len != lower_dentry->d_name.len) { + __d_drop(dentry); + err = 0; + } else if (strncasecmp(dentry->d_name.name, lower_dentry->d_name.name, + dentry->d_name.len) != 0) { + __d_drop(dentry); + err = 0; + } + + if (dentry < lower_dentry) { + spin_unlock(&lower_dentry->d_lock); + spin_unlock(&dentry->d_lock); + } else { + spin_unlock(&dentry->d_lock); + spin_unlock(&lower_dentry->d_lock); + } + +out: + dput(parent_dentry); + dput(lower_cur_parent_dentry); + sdcardfs_put_lower_path(parent_dentry, &parent_lower_path); + sdcardfs_put_real_lower(dentry, &lower_path); + return err; +} + +static void sdcardfs_d_release(struct dentry *dentry) +{ + /* release and reset the lower paths */ + if(has_graft_path(dentry)) { + sdcardfs_put_reset_orig_path(dentry); + } + sdcardfs_put_reset_lower_path(dentry); + free_dentry_private_data(dentry); + return; +} + +static int sdcardfs_hash_ci(const struct dentry *dentry, + struct qstr *qstr) +{ + /* + * This function is copy of vfat_hashi. + * FIXME Should we support national language? + * Refer to vfat_hashi() + * struct nls_table *t = MSDOS_SB(dentry->d_sb)->nls_io; + */ + const unsigned char *name; + unsigned int len; + unsigned long hash; + + name = qstr->name; + //len = vfat_striptail_len(qstr); + len = qstr->len; + + hash = init_name_hash(dentry); + while (len--) + //hash = partial_name_hash(nls_tolower(t, *name++), hash); + hash = partial_name_hash(tolower(*name++), hash); + qstr->hash = end_name_hash(hash); + + return 0; +} + +/* + * Case insensitive compare of two vfat names. + */ +static int sdcardfs_cmp_ci(const struct dentry *dentry, + unsigned int len, const char *str, const struct qstr *name) +{ + /* This function is copy of vfat_cmpi */ + // FIXME Should we support national language? + //struct nls_table *t = MSDOS_SB(parent->d_sb)->nls_io; + //unsigned int alen, blen; + + /* A filename cannot end in '.' or we treat it like it has none */ + /* + alen = vfat_striptail_len(name); + blen = __vfat_striptail_len(len, str); + if (alen == blen) { + if (nls_strnicmp(t, name->name, str, alen) == 0) + return 0; + } + */ + if (name->len == len) { + if (strncasecmp(name->name, str, len) == 0) + return 0; + } + return 1; +} + +static void sdcardfs_canonical_path(const struct path *path, struct path *actual_path) { + sdcardfs_get_real_lower(path->dentry, actual_path); +} + +const struct dentry_operations sdcardfs_ci_dops = { + .d_revalidate = sdcardfs_d_revalidate, + .d_release = sdcardfs_d_release, + .d_hash = sdcardfs_hash_ci, + .d_compare = sdcardfs_cmp_ci, + .d_canonical_path = sdcardfs_canonical_path, +}; + diff --git a/fs/sdcardfs/derived_perm.c b/fs/sdcardfs/derived_perm.c new file mode 100644 index 00000000000000..9408a5477adab9 --- /dev/null +++ b/fs/sdcardfs/derived_perm.c @@ -0,0 +1,325 @@ +/* + * fs/sdcardfs/derived_perm.c + * + * Copyright (c) 2013 Samsung Electronics Co. Ltd + * Authors: Daeho Jeong, Woojoong Lee, Seunghwan Hyun, + * Sunghwan Yun, Sungjong Seo + * + * This program has been developed as a stackable file system based on + * the WrapFS which written by + * + * Copyright (c) 1998-2011 Erez Zadok + * Copyright (c) 2009 Shrikar Archak + * Copyright (c) 2003-2011 Stony Brook University + * Copyright (c) 2003-2011 The Research Foundation of SUNY + * + * This file is dual licensed. It may be redistributed and/or modified + * under the terms of the Apache 2.0 License OR version 2 of the GNU + * General Public License. + */ + +#include "sdcardfs.h" + +/* copy derived state from parent inode */ +static void inherit_derived_state(struct inode *parent, struct inode *child) +{ + struct sdcardfs_inode_info *pi = SDCARDFS_I(parent); + struct sdcardfs_inode_info *ci = SDCARDFS_I(child); + + ci->perm = PERM_INHERIT; + ci->userid = pi->userid; + ci->d_uid = pi->d_uid; + ci->under_android = pi->under_android; + set_top(ci, pi->top); +} + +/* helper function for derived state */ +void setup_derived_state(struct inode *inode, perm_t perm, userid_t userid, + uid_t uid, bool under_android, struct inode *top) +{ + struct sdcardfs_inode_info *info = SDCARDFS_I(inode); + + info->perm = perm; + info->userid = userid; + info->d_uid = uid; + info->under_android = under_android; + set_top(info, top); +} + +/* While renaming, there is a point where we want the path from dentry, but the name from newdentry */ +void get_derived_permission_new(struct dentry *parent, struct dentry *dentry, struct dentry *newdentry) +{ + struct sdcardfs_inode_info *info = SDCARDFS_I(d_inode(dentry)); + struct sdcardfs_inode_info *parent_info= SDCARDFS_I(d_inode(parent)); + appid_t appid; + + /* By default, each inode inherits from its parent. + * the properties are maintained on its private fields + * because the inode attributes will be modified with that of + * its lower inode. + * The derived state will be updated on the last + * stage of each system call by fix_derived_permission(inode). + */ + + inherit_derived_state(d_inode(parent), d_inode(dentry)); + + /* Derive custom permissions based on parent and current node */ + switch (parent_info->perm) { + case PERM_INHERIT: + /* Already inherited above */ + break; + case PERM_PRE_ROOT: + /* Legacy internal layout places users at top level */ + info->perm = PERM_ROOT; + info->userid = simple_strtoul(newdentry->d_name.name, NULL, 10); + set_top(info, &info->vfs_inode); + break; + case PERM_ROOT: + /* Assume masked off by default. */ + if (!strcasecmp(newdentry->d_name.name, "Android")) { + /* App-specific directories inside; let anyone traverse */ + info->perm = PERM_ANDROID; + info->under_android = true; + set_top(info, &info->vfs_inode); + } + break; + case PERM_ANDROID: + if (!strcasecmp(newdentry->d_name.name, "data")) { + /* App-specific directories inside; let anyone traverse */ + info->perm = PERM_ANDROID_DATA; + set_top(info, &info->vfs_inode); + } else if (!strcasecmp(newdentry->d_name.name, "obb")) { + /* App-specific directories inside; let anyone traverse */ + info->perm = PERM_ANDROID_OBB; + set_top(info, &info->vfs_inode); + /* Single OBB directory is always shared */ + } else if (!strcasecmp(newdentry->d_name.name, "media")) { + /* App-specific directories inside; let anyone traverse */ + info->perm = PERM_ANDROID_MEDIA; + set_top(info, &info->vfs_inode); + } + break; + case PERM_ANDROID_DATA: + case PERM_ANDROID_OBB: + case PERM_ANDROID_MEDIA: + appid = get_appid(newdentry->d_name.name); + if (appid != 0) { + info->d_uid = multiuser_get_uid(parent_info->userid, appid); + } + set_top(info, &info->vfs_inode); + break; + } +} + +void get_derived_permission(struct dentry *parent, struct dentry *dentry) +{ + get_derived_permission_new(parent, dentry, dentry); +} + +static int descendant_may_need_fixup(perm_t perm) { + if (perm == PERM_PRE_ROOT || perm == PERM_ROOT || perm == PERM_ANDROID) + return 1; + return 0; +} + +static int needs_fixup(perm_t perm) { + if (perm == PERM_ANDROID_DATA || perm == PERM_ANDROID_OBB + || perm == PERM_ANDROID_MEDIA) + return 1; + return 0; +} + +void fixup_perms_recursive(struct dentry *dentry, const char* name, size_t len) { + struct dentry *child; + struct sdcardfs_inode_info *info; + if (!dget(dentry)) + return; + if (!d_inode(dentry)) { + dput(dentry); + return; + } + info = SDCARDFS_I(d_inode(dentry)); + + if (needs_fixup(info->perm)) { + spin_lock(&dentry->d_lock); + list_for_each_entry(child, &dentry->d_subdirs, d_child) { + dget(child); + if (!strncasecmp(child->d_name.name, name, len)) { + if (child->d_inode) { + get_derived_permission(dentry, child); + fixup_tmp_permissions(child->d_inode); + dput(child); + break; + } + } + dput(child); + } + spin_unlock(&dentry->d_lock); + } else if (descendant_may_need_fixup(info->perm)) { + spin_lock(&dentry->d_lock); + list_for_each_entry(child, &dentry->d_subdirs, d_child) { + fixup_perms_recursive(child, name, len); + } + spin_unlock(&dentry->d_lock); + } + dput(dentry); +} + +void fixup_top_recursive(struct dentry *parent) { + struct dentry *dentry; + struct sdcardfs_inode_info *info; + if (!d_inode(parent)) + return; + info = SDCARDFS_I(d_inode(parent)); + spin_lock(&parent->d_lock); + list_for_each_entry(dentry, &parent->d_subdirs, d_child) { + if (d_inode(dentry)) { + if (SDCARDFS_I(d_inode(parent))->top != SDCARDFS_I(d_inode(dentry))->top) { + get_derived_permission(parent, dentry); + fixup_tmp_permissions(d_inode(dentry)); + fixup_top_recursive(dentry); + } + } + } + spin_unlock(&parent->d_lock); +} + +/* main function for updating derived permission */ +inline void update_derived_permission_lock(struct dentry *dentry) +{ + struct dentry *parent; + + if(!dentry || !d_inode(dentry)) { + printk(KERN_ERR "sdcardfs: %s: invalid dentry\n", __func__); + return; + } + /* FIXME: + * 1. need to check whether the dentry is updated or not + * 2. remove the root dentry update + */ + if(IS_ROOT(dentry)) { + //setup_default_pre_root_state(d_inode(dentry)); + } else { + parent = dget_parent(dentry); + if(parent) { + get_derived_permission(parent, dentry); + dput(parent); + } + } + fixup_tmp_permissions(d_inode(dentry)); +} + +int need_graft_path(struct dentry *dentry) +{ + int ret = 0; + struct dentry *parent = dget_parent(dentry); + struct sdcardfs_inode_info *parent_info= SDCARDFS_I(d_inode(parent)); + struct sdcardfs_sb_info *sbi = SDCARDFS_SB(dentry->d_sb); + + if(parent_info->perm == PERM_ANDROID && + !strcasecmp(dentry->d_name.name, "obb")) { + + /* /Android/obb is the base obbpath of DERIVED_UNIFIED */ + if(!(sbi->options.multiuser == false + && parent_info->userid == 0)) { + ret = 1; + } + } + dput(parent); + return ret; +} + +int is_obbpath_invalid(struct dentry *dent) +{ + int ret = 0; + struct sdcardfs_dentry_info *di = SDCARDFS_D(dent); + struct sdcardfs_sb_info *sbi = SDCARDFS_SB(dent->d_sb); + char *path_buf, *obbpath_s; + + /* check the base obbpath has been changed. + * this routine can check an uninitialized obb dentry as well. + * regarding the uninitialized obb, refer to the sdcardfs_mkdir() */ + spin_lock(&di->lock); + if(di->orig_path.dentry) { + if(!di->lower_path.dentry) { + ret = 1; + } else { + path_get(&di->lower_path); + //lower_parent = lock_parent(lower_path->dentry); + + path_buf = kmalloc(PATH_MAX, GFP_ATOMIC); + if(!path_buf) { + ret = 1; + printk(KERN_ERR "sdcardfs: fail to allocate path_buf in %s.\n", __func__); + } else { + obbpath_s = d_path(&di->lower_path, path_buf, PATH_MAX); + if (d_unhashed(di->lower_path.dentry) || + strcasecmp(sbi->obbpath_s, obbpath_s)) { + ret = 1; + } + kfree(path_buf); + } + + //unlock_dir(lower_parent); + path_put(&di->lower_path); + } + } + spin_unlock(&di->lock); + return ret; +} + +int is_base_obbpath(struct dentry *dentry) +{ + int ret = 0; + struct dentry *parent = dget_parent(dentry); + struct sdcardfs_inode_info *parent_info= SDCARDFS_I(d_inode(parent)); + struct sdcardfs_sb_info *sbi = SDCARDFS_SB(dentry->d_sb); + + spin_lock(&SDCARDFS_D(dentry)->lock); + if (sbi->options.multiuser) { + if(parent_info->perm == PERM_PRE_ROOT && + !strcasecmp(dentry->d_name.name, "obb")) { + ret = 1; + } + } else if (parent_info->perm == PERM_ANDROID && + !strcasecmp(dentry->d_name.name, "obb")) { + ret = 1; + } + spin_unlock(&SDCARDFS_D(dentry)->lock); + return ret; +} + +/* The lower_path will be stored to the dentry's orig_path + * and the base obbpath will be copyed to the lower_path variable. + * if an error returned, there's no change in the lower_path + * returns: -ERRNO if error (0: no error) */ +int setup_obb_dentry(struct dentry *dentry, struct path *lower_path) +{ + int err = 0; + struct sdcardfs_sb_info *sbi = SDCARDFS_SB(dentry->d_sb); + struct path obbpath; + + /* A local obb dentry must have its own orig_path to support rmdir + * and mkdir of itself. Usually, we expect that the sbi->obbpath + * is avaiable on this stage. */ + sdcardfs_set_orig_path(dentry, lower_path); + + err = kern_path(sbi->obbpath_s, + LOOKUP_FOLLOW | LOOKUP_DIRECTORY, &obbpath); + + if(!err) { + /* the obbpath base has been found */ + printk(KERN_INFO "sdcardfs: the sbi->obbpath is found\n"); + pathcpy(lower_path, &obbpath); + } else { + /* if the sbi->obbpath is not available, we can optionally + * setup the lower_path with its orig_path. + * but, the current implementation just returns an error + * because the sdcard daemon also regards this case as + * a lookup fail. */ + printk(KERN_INFO "sdcardfs: the sbi->obbpath is not available\n"); + } + return err; +} + + diff --git a/fs/sdcardfs/file.c b/fs/sdcardfs/file.c new file mode 100644 index 00000000000000..7750a0472389be --- /dev/null +++ b/fs/sdcardfs/file.c @@ -0,0 +1,356 @@ +/* + * fs/sdcardfs/file.c + * + * Copyright (c) 2013 Samsung Electronics Co. Ltd + * Authors: Daeho Jeong, Woojoong Lee, Seunghwan Hyun, + * Sunghwan Yun, Sungjong Seo + * + * This program has been developed as a stackable file system based on + * the WrapFS which written by + * + * Copyright (c) 1998-2011 Erez Zadok + * Copyright (c) 2009 Shrikar Archak + * Copyright (c) 2003-2011 Stony Brook University + * Copyright (c) 2003-2011 The Research Foundation of SUNY + * + * This file is dual licensed. It may be redistributed and/or modified + * under the terms of the Apache 2.0 License OR version 2 of the GNU + * General Public License. + */ + +#include "sdcardfs.h" +#ifdef CONFIG_SDCARD_FS_FADV_NOACTIVE +#include +#endif + +static ssize_t sdcardfs_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + int err; + struct file *lower_file; + struct dentry *dentry = file->f_path.dentry; +#ifdef CONFIG_SDCARD_FS_FADV_NOACTIVE + struct backing_dev_info *bdi; +#endif + + lower_file = sdcardfs_lower_file(file); + +#ifdef CONFIG_SDCARD_FS_FADV_NOACTIVE + if (file->f_mode & FMODE_NOACTIVE) { + if (!(lower_file->f_mode & FMODE_NOACTIVE)) { + bdi = lower_file->f_mapping->backing_dev_info; + lower_file->f_ra.ra_pages = bdi->ra_pages * 2; + spin_lock(&lower_file->f_lock); + lower_file->f_mode |= FMODE_NOACTIVE; + spin_unlock(&lower_file->f_lock); + } + } +#endif + + err = vfs_read(lower_file, buf, count, ppos); + /* update our inode atime upon a successful lower read */ + if (err >= 0) + fsstack_copy_attr_atime(d_inode(dentry), + file_inode(lower_file)); + + return err; +} + +static ssize_t sdcardfs_write(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + int err; + struct file *lower_file; + struct dentry *dentry = file->f_path.dentry; + + /* check disk space */ + if (!check_min_free_space(dentry, count, 0)) { + printk(KERN_INFO "No minimum free space.\n"); + return -ENOSPC; + } + + lower_file = sdcardfs_lower_file(file); + err = vfs_write(lower_file, buf, count, ppos); + /* update our inode times+sizes upon a successful lower write */ + if (err >= 0) { + fsstack_copy_inode_size(d_inode(dentry), + file_inode(lower_file)); + fsstack_copy_attr_times(d_inode(dentry), + file_inode(lower_file)); + } + + return err; +} + +static int sdcardfs_readdir(struct file *file, struct dir_context *ctx) +{ + int err; + struct file *lower_file = NULL; + struct dentry *dentry = file->f_path.dentry; + + lower_file = sdcardfs_lower_file(file); + + lower_file->f_pos = file->f_pos; + err = iterate_dir(lower_file, ctx); + file->f_pos = lower_file->f_pos; + if (err >= 0) /* copy the atime */ + fsstack_copy_attr_atime(d_inode(dentry), + file_inode(lower_file)); + return err; +} + +static long sdcardfs_unlocked_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + long err = -ENOTTY; + struct file *lower_file; + + lower_file = sdcardfs_lower_file(file); + + /* XXX: use vfs_ioctl if/when VFS exports it */ + if (!lower_file || !lower_file->f_op) + goto out; + if (lower_file->f_op->unlocked_ioctl) + err = lower_file->f_op->unlocked_ioctl(lower_file, cmd, arg); + +out: + return err; +} + +#ifdef CONFIG_COMPAT +static long sdcardfs_compat_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + long err = -ENOTTY; + struct file *lower_file; + + lower_file = sdcardfs_lower_file(file); + + /* XXX: use vfs_ioctl if/when VFS exports it */ + if (!lower_file || !lower_file->f_op) + goto out; + if (lower_file->f_op->compat_ioctl) + err = lower_file->f_op->compat_ioctl(lower_file, cmd, arg); + +out: + return err; +} +#endif + +static int sdcardfs_mmap(struct file *file, struct vm_area_struct *vma) +{ + int err = 0; + bool willwrite; + struct file *lower_file; + const struct vm_operations_struct *saved_vm_ops = NULL; + + /* this might be deferred to mmap's writepage */ + willwrite = ((vma->vm_flags | VM_SHARED | VM_WRITE) == vma->vm_flags); + + /* + * File systems which do not implement ->writepage may use + * generic_file_readonly_mmap as their ->mmap op. If you call + * generic_file_readonly_mmap with VM_WRITE, you'd get an -EINVAL. + * But we cannot call the lower ->mmap op, so we can't tell that + * writeable mappings won't work. Therefore, our only choice is to + * check if the lower file system supports the ->writepage, and if + * not, return EINVAL (the same error that + * generic_file_readonly_mmap returns in that case). + */ + lower_file = sdcardfs_lower_file(file); + if (willwrite && !lower_file->f_mapping->a_ops->writepage) { + err = -EINVAL; + printk(KERN_ERR "sdcardfs: lower file system does not " + "support writeable mmap\n"); + goto out; + } + + /* + * find and save lower vm_ops. + * + * XXX: the VFS should have a cleaner way of finding the lower vm_ops + */ + if (!SDCARDFS_F(file)->lower_vm_ops) { + err = lower_file->f_op->mmap(lower_file, vma); + if (err) { + printk(KERN_ERR "sdcardfs: lower mmap failed %d\n", err); + goto out; + } + saved_vm_ops = vma->vm_ops; /* save: came from lower ->mmap */ + err = do_munmap(current->mm, vma->vm_start, + vma->vm_end - vma->vm_start); + if (err) { + printk(KERN_ERR "sdcardfs: do_munmap failed %d\n", err); + goto out; + } + } + + /* + * Next 3 lines are all I need from generic_file_mmap. I definitely + * don't want its test for ->readpage which returns -ENOEXEC. + */ + file_accessed(file); + vma->vm_ops = &sdcardfs_vm_ops; + + file->f_mapping->a_ops = &sdcardfs_aops; /* set our aops */ + if (!SDCARDFS_F(file)->lower_vm_ops) /* save for our ->fault */ + SDCARDFS_F(file)->lower_vm_ops = saved_vm_ops; + +out: + return err; +} + +static int sdcardfs_open(struct inode *inode, struct file *file) +{ + int err = 0; + struct file *lower_file = NULL; + struct path lower_path; + struct dentry *dentry = file->f_path.dentry; + struct dentry *parent = dget_parent(dentry); + struct sdcardfs_sb_info *sbi = SDCARDFS_SB(dentry->d_sb); + const struct cred *saved_cred = NULL; + + /* don't open unhashed/deleted files */ + if (d_unhashed(dentry)) { + err = -ENOENT; + goto out_err; + } + + if(!check_caller_access_to_name(d_inode(parent), dentry->d_name.name)) { + printk(KERN_INFO "%s: need to check the caller's gid in packages.list\n" + " dentry: %s, task:%s\n", + __func__, dentry->d_name.name, current->comm); + err = -EACCES; + goto out_err; + } + + /* save current_cred and override it */ + OVERRIDE_CRED(sbi, saved_cred); + + file->private_data = + kzalloc(sizeof(struct sdcardfs_file_info), GFP_KERNEL); + if (!SDCARDFS_F(file)) { + err = -ENOMEM; + goto out_revert_cred; + } + + /* open lower object and link sdcardfs's file struct to lower's */ + sdcardfs_get_lower_path(file->f_path.dentry, &lower_path); + lower_file = dentry_open(&lower_path, file->f_flags, current_cred()); + path_put(&lower_path); + if (IS_ERR(lower_file)) { + err = PTR_ERR(lower_file); + lower_file = sdcardfs_lower_file(file); + if (lower_file) { + sdcardfs_set_lower_file(file, NULL); + fput(lower_file); /* fput calls dput for lower_dentry */ + } + } else { + sdcardfs_set_lower_file(file, lower_file); + } + + if (err) + kfree(SDCARDFS_F(file)); + else { + sdcardfs_copy_and_fix_attrs(inode, sdcardfs_lower_inode(inode)); + } + +out_revert_cred: + REVERT_CRED(saved_cred); +out_err: + dput(parent); + return err; +} + +static int sdcardfs_flush(struct file *file, fl_owner_t id) +{ + int err = 0; + struct file *lower_file = NULL; + + lower_file = sdcardfs_lower_file(file); + if (lower_file && lower_file->f_op && lower_file->f_op->flush) { + filemap_write_and_wait(file->f_mapping); + err = lower_file->f_op->flush(lower_file, id); + } + + return err; +} + +/* release all lower object references & free the file info structure */ +static int sdcardfs_file_release(struct inode *inode, struct file *file) +{ + struct file *lower_file; + + lower_file = sdcardfs_lower_file(file); + if (lower_file) { + sdcardfs_set_lower_file(file, NULL); + fput(lower_file); + } + + kfree(SDCARDFS_F(file)); + return 0; +} + +static int sdcardfs_fsync(struct file *file, loff_t start, loff_t end, + int datasync) +{ + int err; + struct file *lower_file; + struct path lower_path; + struct dentry *dentry = file->f_path.dentry; + + err = __generic_file_fsync(file, start, end, datasync); + if (err) + goto out; + + lower_file = sdcardfs_lower_file(file); + sdcardfs_get_lower_path(dentry, &lower_path); + err = vfs_fsync_range(lower_file, start, end, datasync); + sdcardfs_put_lower_path(dentry, &lower_path); +out: + return err; +} + +static int sdcardfs_fasync(int fd, struct file *file, int flag) +{ + int err = 0; + struct file *lower_file = NULL; + + lower_file = sdcardfs_lower_file(file); + if (lower_file->f_op && lower_file->f_op->fasync) + err = lower_file->f_op->fasync(fd, lower_file, flag); + + return err; +} + +const struct file_operations sdcardfs_main_fops = { + .llseek = generic_file_llseek, + .read = sdcardfs_read, + .write = sdcardfs_write, + .unlocked_ioctl = sdcardfs_unlocked_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = sdcardfs_compat_ioctl, +#endif + .mmap = sdcardfs_mmap, + .open = sdcardfs_open, + .flush = sdcardfs_flush, + .release = sdcardfs_file_release, + .fsync = sdcardfs_fsync, + .fasync = sdcardfs_fasync, +}; + +/* trimmed directory options */ +const struct file_operations sdcardfs_dir_fops = { + .llseek = generic_file_llseek, + .read = generic_read_dir, + .iterate = sdcardfs_readdir, + .unlocked_ioctl = sdcardfs_unlocked_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = sdcardfs_compat_ioctl, +#endif + .open = sdcardfs_open, + .release = sdcardfs_file_release, + .flush = sdcardfs_flush, + .fsync = sdcardfs_fsync, + .fasync = sdcardfs_fasync, +}; diff --git a/fs/sdcardfs/inode.c b/fs/sdcardfs/inode.c new file mode 100644 index 00000000000000..5b311708195b48 --- /dev/null +++ b/fs/sdcardfs/inode.c @@ -0,0 +1,935 @@ +/* + * fs/sdcardfs/inode.c + * + * Copyright (c) 2013 Samsung Electronics Co. Ltd + * Authors: Daeho Jeong, Woojoong Lee, Seunghwan Hyun, + * Sunghwan Yun, Sungjong Seo + * + * This program has been developed as a stackable file system based on + * the WrapFS which written by + * + * Copyright (c) 1998-2011 Erez Zadok + * Copyright (c) 2009 Shrikar Archak + * Copyright (c) 2003-2011 Stony Brook University + * Copyright (c) 2003-2011 The Research Foundation of SUNY + * + * This file is dual licensed. It may be redistributed and/or modified + * under the terms of the Apache 2.0 License OR version 2 of the GNU + * General Public License. + */ + +#include "sdcardfs.h" +#include + +/* Do not directly use this function. Use OVERRIDE_CRED() instead. */ +const struct cred * override_fsids(struct sdcardfs_sb_info* sbi) +{ + struct cred * cred; + const struct cred * old_cred; + + cred = prepare_creds(); + if (!cred) + return NULL; + + cred->fsuid = make_kuid(&init_user_ns, sbi->options.fs_low_uid); + cred->fsgid = make_kgid(&init_user_ns, sbi->options.fs_low_gid); + + old_cred = override_creds(cred); + + return old_cred; +} + +/* Do not directly use this function, use REVERT_CRED() instead. */ +void revert_fsids(const struct cred * old_cred) +{ + const struct cred * cur_cred; + + cur_cred = current->cred; + revert_creds(old_cred); + put_cred(cur_cred); +} + +static int sdcardfs_create(struct inode *dir, struct dentry *dentry, + umode_t mode, bool want_excl) +{ + int err; + struct dentry *lower_dentry; + struct vfsmount *lower_dentry_mnt; + struct dentry *lower_parent_dentry = NULL; + struct path lower_path; + const struct cred *saved_cred = NULL; + struct fs_struct *saved_fs; + struct fs_struct *copied_fs; + + if(!check_caller_access_to_name(dir, dentry->d_name.name)) { + printk(KERN_INFO "%s: need to check the caller's gid in packages.list\n" + " dentry: %s, task:%s\n", + __func__, dentry->d_name.name, current->comm); + err = -EACCES; + goto out_eacces; + } + + /* save current_cred and override it */ + OVERRIDE_CRED(SDCARDFS_SB(dir->i_sb), saved_cred); + + sdcardfs_get_lower_path(dentry, &lower_path); + lower_dentry = lower_path.dentry; + lower_dentry_mnt = lower_path.mnt; + lower_parent_dentry = lock_parent(lower_dentry); + + /* set last 16bytes of mode field to 0664 */ + mode = (mode & S_IFMT) | 00664; + + /* temporarily change umask for lower fs write */ + saved_fs = current->fs; + copied_fs = copy_fs_struct(current->fs); + if (!copied_fs) { + err = -ENOMEM; + goto out_unlock; + } + current->fs = copied_fs; + current->fs->umask = 0; + err = vfs_create2(lower_dentry_mnt, d_inode(lower_parent_dentry), lower_dentry, mode, want_excl); + if (err) + goto out; + + err = sdcardfs_interpose(dentry, dir->i_sb, &lower_path, SDCARDFS_I(dir)->userid); + if (err) + goto out; + fsstack_copy_attr_times(dir, sdcardfs_lower_inode(dir)); + fsstack_copy_inode_size(dir, d_inode(lower_parent_dentry)); + +out: + current->fs = saved_fs; + free_fs_struct(copied_fs); +out_unlock: + unlock_dir(lower_parent_dentry); + sdcardfs_put_lower_path(dentry, &lower_path); + REVERT_CRED(saved_cred); +out_eacces: + return err; +} + +#if 0 +static int sdcardfs_link(struct dentry *old_dentry, struct inode *dir, + struct dentry *new_dentry) +{ + struct dentry *lower_old_dentry; + struct dentry *lower_new_dentry; + struct dentry *lower_dir_dentry; + u64 file_size_save; + int err; + struct path lower_old_path, lower_new_path; + + OVERRIDE_CRED(SDCARDFS_SB(dir->i_sb)); + + file_size_save = i_size_read(d_inode(old_dentry)); + sdcardfs_get_lower_path(old_dentry, &lower_old_path); + sdcardfs_get_lower_path(new_dentry, &lower_new_path); + lower_old_dentry = lower_old_path.dentry; + lower_new_dentry = lower_new_path.dentry; + lower_dir_dentry = lock_parent(lower_new_dentry); + + err = vfs_link(lower_old_dentry, d_inode(lower_dir_dentry), + lower_new_dentry, NULL); + if (err || !d_inode(lower_new_dentry)) + goto out; + + err = sdcardfs_interpose(new_dentry, dir->i_sb, &lower_new_path); + if (err) + goto out; + fsstack_copy_attr_times(dir, d_inode(lower_new_dentry)); + fsstack_copy_inode_size(dir, d_inode(lower_new_dentry)); + set_nlink(d_inode(old_dentry), + sdcardfs_lower_inode(d_inode(old_dentry))->i_nlink); + i_size_write(d_inode(new_dentry), file_size_save); +out: + unlock_dir(lower_dir_dentry); + sdcardfs_put_lower_path(old_dentry, &lower_old_path); + sdcardfs_put_lower_path(new_dentry, &lower_new_path); + REVERT_CRED(); + return err; +} +#endif + +static int sdcardfs_unlink(struct inode *dir, struct dentry *dentry) +{ + int err; + struct dentry *lower_dentry; + struct vfsmount *lower_mnt; + struct inode *lower_dir_inode = sdcardfs_lower_inode(dir); + struct dentry *lower_dir_dentry; + struct path lower_path; + const struct cred *saved_cred = NULL; + + if(!check_caller_access_to_name(dir, dentry->d_name.name)) { + printk(KERN_INFO "%s: need to check the caller's gid in packages.list\n" + " dentry: %s, task:%s\n", + __func__, dentry->d_name.name, current->comm); + err = -EACCES; + goto out_eacces; + } + + /* save current_cred and override it */ + OVERRIDE_CRED(SDCARDFS_SB(dir->i_sb), saved_cred); + + sdcardfs_get_lower_path(dentry, &lower_path); + lower_dentry = lower_path.dentry; + lower_mnt = lower_path.mnt; + dget(lower_dentry); + lower_dir_dentry = lock_parent(lower_dentry); + + err = vfs_unlink2(lower_mnt, lower_dir_inode, lower_dentry, NULL); + + /* + * Note: unlinking on top of NFS can cause silly-renamed files. + * Trying to delete such files results in EBUSY from NFS + * below. Silly-renamed files will get deleted by NFS later on, so + * we just need to detect them here and treat such EBUSY errors as + * if the upper file was successfully deleted. + */ + if (err == -EBUSY && lower_dentry->d_flags & DCACHE_NFSFS_RENAMED) + err = 0; + if (err) + goto out; + fsstack_copy_attr_times(dir, lower_dir_inode); + fsstack_copy_inode_size(dir, lower_dir_inode); + set_nlink(d_inode(dentry), + sdcardfs_lower_inode(d_inode(dentry))->i_nlink); + d_inode(dentry)->i_ctime = dir->i_ctime; + d_drop(dentry); /* this is needed, else LTP fails (VFS won't do it) */ +out: + unlock_dir(lower_dir_dentry); + dput(lower_dentry); + sdcardfs_put_lower_path(dentry, &lower_path); + REVERT_CRED(saved_cred); +out_eacces: + return err; +} + +#if 0 +static int sdcardfs_symlink(struct inode *dir, struct dentry *dentry, + const char *symname) +{ + int err; + struct dentry *lower_dentry; + struct dentry *lower_parent_dentry = NULL; + struct path lower_path; + + OVERRIDE_CRED(SDCARDFS_SB(dir->i_sb)); + + sdcardfs_get_lower_path(dentry, &lower_path); + lower_dentry = lower_path.dentry; + lower_parent_dentry = lock_parent(lower_dentry); + + err = vfs_symlink(d_inode(lower_parent_dentry), lower_dentry, symname); + if (err) + goto out; + err = sdcardfs_interpose(dentry, dir->i_sb, &lower_path); + if (err) + goto out; + fsstack_copy_attr_times(dir, sdcardfs_lower_inode(dir)); + fsstack_copy_inode_size(dir, d_inode(lower_parent_dentry)); + +out: + unlock_dir(lower_parent_dentry); + sdcardfs_put_lower_path(dentry, &lower_path); + REVERT_CRED(); + return err; +} +#endif + +static int touch(char *abs_path, mode_t mode) { + struct file *filp = filp_open(abs_path, O_RDWR|O_CREAT|O_EXCL|O_NOFOLLOW, mode); + if (IS_ERR(filp)) { + if (PTR_ERR(filp) == -EEXIST) { + return 0; + } + else { + printk(KERN_ERR "sdcardfs: failed to open(%s): %ld\n", + abs_path, PTR_ERR(filp)); + return PTR_ERR(filp); + } + } + filp_close(filp, current->files); + return 0; +} + +static int sdcardfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +{ + int err; + int make_nomedia_in_obb = 0; + struct dentry *lower_dentry; + struct vfsmount *lower_mnt; + struct dentry *lower_parent_dentry = NULL; + struct path lower_path; + struct sdcardfs_sb_info *sbi = SDCARDFS_SB(dentry->d_sb); + const struct cred *saved_cred = NULL; + struct sdcardfs_inode_info *pi = SDCARDFS_I(dir); + int touch_err = 0; + struct fs_struct *saved_fs; + struct fs_struct *copied_fs; + + if(!check_caller_access_to_name(dir, dentry->d_name.name)) { + printk(KERN_INFO "%s: need to check the caller's gid in packages.list\n" + " dentry: %s, task:%s\n", + __func__, dentry->d_name.name, current->comm); + err = -EACCES; + goto out_eacces; + } + + /* save current_cred and override it */ + OVERRIDE_CRED(SDCARDFS_SB(dir->i_sb), saved_cred); + + /* check disk space */ + if (!check_min_free_space(dentry, 0, 1)) { + printk(KERN_INFO "sdcardfs: No minimum free space.\n"); + err = -ENOSPC; + goto out_revert; + } + + /* the lower_dentry is negative here */ + sdcardfs_get_lower_path(dentry, &lower_path); + lower_dentry = lower_path.dentry; + lower_mnt = lower_path.mnt; + lower_parent_dentry = lock_parent(lower_dentry); + + /* set last 16bytes of mode field to 0775 */ + mode = (mode & S_IFMT) | 00775; + + /* temporarily change umask for lower fs write */ + saved_fs = current->fs; + copied_fs = copy_fs_struct(current->fs); + if (!copied_fs) { + err = -ENOMEM; + unlock_dir(lower_parent_dentry); + goto out_unlock; + } + current->fs = copied_fs; + current->fs->umask = 0; + err = vfs_mkdir2(lower_mnt, d_inode(lower_parent_dentry), lower_dentry, mode); + + if (err) { + unlock_dir(lower_parent_dentry); + goto out; + } + + /* if it is a local obb dentry, setup it with the base obbpath */ + if(need_graft_path(dentry)) { + + err = setup_obb_dentry(dentry, &lower_path); + if(err) { + /* if the sbi->obbpath is not available, the lower_path won't be + * changed by setup_obb_dentry() but the lower path is saved to + * its orig_path. this dentry will be revalidated later. + * but now, the lower_path should be NULL */ + sdcardfs_put_reset_lower_path(dentry); + + /* the newly created lower path which saved to its orig_path or + * the lower_path is the base obbpath. + * therefore, an additional path_get is required */ + path_get(&lower_path); + } else + make_nomedia_in_obb = 1; + } + + err = sdcardfs_interpose(dentry, dir->i_sb, &lower_path, pi->userid); + if (err) { + unlock_dir(lower_parent_dentry); + goto out; + } + + fsstack_copy_attr_times(dir, sdcardfs_lower_inode(dir)); + fsstack_copy_inode_size(dir, d_inode(lower_parent_dentry)); + /* update number of links on parent directory */ + set_nlink(dir, sdcardfs_lower_inode(dir)->i_nlink); + + unlock_dir(lower_parent_dentry); + + if ((!sbi->options.multiuser) && (!strcasecmp(dentry->d_name.name, "obb")) + && (pi->perm == PERM_ANDROID) && (pi->userid == 0)) + make_nomedia_in_obb = 1; + + /* When creating /Android/data and /Android/obb, mark them as .nomedia */ + if (make_nomedia_in_obb || + ((pi->perm == PERM_ANDROID) && (!strcasecmp(dentry->d_name.name, "data")))) { + set_fs_pwd(current->fs, &lower_path); + touch_err = touch(".nomedia", 0664); + if (touch_err) { + printk(KERN_ERR "sdcardfs: failed to create .nomedia in %s: %d\n", + lower_path.dentry->d_name.name, touch_err); + goto out; + } + } +out: + current->fs = saved_fs; + free_fs_struct(copied_fs); +out_unlock: + sdcardfs_put_lower_path(dentry, &lower_path); +out_revert: + REVERT_CRED(saved_cred); +out_eacces: + return err; +} + +static int sdcardfs_rmdir(struct inode *dir, struct dentry *dentry) +{ + struct dentry *lower_dentry; + struct dentry *lower_dir_dentry; + struct vfsmount *lower_mnt; + int err; + struct path lower_path; + const struct cred *saved_cred = NULL; + + if(!check_caller_access_to_name(dir, dentry->d_name.name)) { + printk(KERN_INFO "%s: need to check the caller's gid in packages.list\n" + " dentry: %s, task:%s\n", + __func__, dentry->d_name.name, current->comm); + err = -EACCES; + goto out_eacces; + } + + /* save current_cred and override it */ + OVERRIDE_CRED(SDCARDFS_SB(dir->i_sb), saved_cred); + + /* sdcardfs_get_real_lower(): in case of remove an user's obb dentry + * the dentry on the original path should be deleted. */ + sdcardfs_get_real_lower(dentry, &lower_path); + + lower_dentry = lower_path.dentry; + lower_mnt = lower_path.mnt; + lower_dir_dentry = lock_parent(lower_dentry); + + err = vfs_rmdir2(lower_mnt, d_inode(lower_dir_dentry), lower_dentry); + if (err) + goto out; + + d_drop(dentry); /* drop our dentry on success (why not VFS's job?) */ + if (d_inode(dentry)) + clear_nlink(d_inode(dentry)); + fsstack_copy_attr_times(dir, d_inode(lower_dir_dentry)); + fsstack_copy_inode_size(dir, d_inode(lower_dir_dentry)); + set_nlink(dir, d_inode(lower_dir_dentry)->i_nlink); + +out: + unlock_dir(lower_dir_dentry); + sdcardfs_put_real_lower(dentry, &lower_path); + REVERT_CRED(saved_cred); +out_eacces: + return err; +} + +#if 0 +static int sdcardfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, + dev_t dev) +{ + int err; + struct dentry *lower_dentry; + struct dentry *lower_parent_dentry = NULL; + struct path lower_path; + + OVERRIDE_CRED(SDCARDFS_SB(dir->i_sb)); + + sdcardfs_get_lower_path(dentry, &lower_path); + lower_dentry = lower_path.dentry; + lower_parent_dentry = lock_parent(lower_dentry); + + err = vfs_mknod(d_inode(lower_parent_dentry), lower_dentry, mode, dev); + if (err) + goto out; + + err = sdcardfs_interpose(dentry, dir->i_sb, &lower_path); + if (err) + goto out; + fsstack_copy_attr_times(dir, sdcardfs_lower_inode(dir)); + fsstack_copy_inode_size(dir, d_inode(lower_parent_dentry)); + +out: + unlock_dir(lower_parent_dentry); + sdcardfs_put_lower_path(dentry, &lower_path); + REVERT_CRED(); + return err; +} +#endif + +/* + * The locking rules in sdcardfs_rename are complex. We could use a simpler + * superblock-level name-space lock for renames and copy-ups. + */ +static int sdcardfs_rename(struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry, + unsigned int flags) +{ + int err = 0; + struct dentry *lower_old_dentry = NULL; + struct dentry *lower_new_dentry = NULL; + struct dentry *lower_old_dir_dentry = NULL; + struct dentry *lower_new_dir_dentry = NULL; + struct vfsmount *lower_mnt = NULL; + struct dentry *trap = NULL; + struct dentry *new_parent = NULL; + struct path lower_old_path, lower_new_path; + const struct cred *saved_cred = NULL; + + if (flags) + return -EINVAL; + + if(!check_caller_access_to_name(old_dir, old_dentry->d_name.name) || + !check_caller_access_to_name(new_dir, new_dentry->d_name.name)) { + printk(KERN_INFO "%s: need to check the caller's gid in packages.list\n" + " new_dentry: %s, task:%s\n", + __func__, new_dentry->d_name.name, current->comm); + err = -EACCES; + goto out_eacces; + } + + /* save current_cred and override it */ + OVERRIDE_CRED(SDCARDFS_SB(old_dir->i_sb), saved_cred); + + sdcardfs_get_real_lower(old_dentry, &lower_old_path); + sdcardfs_get_lower_path(new_dentry, &lower_new_path); + lower_old_dentry = lower_old_path.dentry; + lower_new_dentry = lower_new_path.dentry; + lower_mnt = lower_old_path.mnt; + lower_old_dir_dentry = dget_parent(lower_old_dentry); + lower_new_dir_dentry = dget_parent(lower_new_dentry); + + trap = lock_rename(lower_old_dir_dentry, lower_new_dir_dentry); + /* source should not be ancestor of target */ + if (trap == lower_old_dentry) { + err = -EINVAL; + goto out; + } + /* target should not be ancestor of source */ + if (trap == lower_new_dentry) { + err = -ENOTEMPTY; + goto out; + } + + err = vfs_rename2(lower_mnt, + d_inode(lower_old_dir_dentry), lower_old_dentry, + d_inode(lower_new_dir_dentry), lower_new_dentry, + NULL, 0); + if (err) + goto out; + + /* Copy attrs from lower dir, but i_uid/i_gid */ + sdcardfs_copy_and_fix_attrs(new_dir, d_inode(lower_new_dir_dentry)); + fsstack_copy_inode_size(new_dir, d_inode(lower_new_dir_dentry)); + + if (new_dir != old_dir) { + sdcardfs_copy_and_fix_attrs(old_dir, d_inode(lower_old_dir_dentry)); + fsstack_copy_inode_size(old_dir, d_inode(lower_old_dir_dentry)); + + /* update the derived permission of the old_dentry + * with its new parent + */ + new_parent = dget_parent(new_dentry); + if(new_parent) { + if(d_inode(old_dentry)) { + update_derived_permission_lock(old_dentry); + } + dput(new_parent); + } + } + /* At this point, not all dentry information has been moved, so + * we pass along new_dentry for the name.*/ + get_derived_permission_new(new_dentry->d_parent, old_dentry, new_dentry); + fixup_tmp_permissions(d_inode(old_dentry)); + fixup_top_recursive(old_dentry); +out: + unlock_rename(lower_old_dir_dentry, lower_new_dir_dentry); + dput(lower_old_dir_dentry); + dput(lower_new_dir_dentry); + sdcardfs_put_real_lower(old_dentry, &lower_old_path); + sdcardfs_put_lower_path(new_dentry, &lower_new_path); + REVERT_CRED(saved_cred); +out_eacces: + return err; +} + +#if 0 +static int sdcardfs_readlink(struct dentry *dentry, char __user *buf, int bufsiz) +{ + int err; + struct dentry *lower_dentry; + struct path lower_path; + /* XXX readlink does not requires overriding credential */ + + sdcardfs_get_lower_path(dentry, &lower_path); + lower_dentry = lower_path.dentry; + if (!d_inode(lower_dentry)->i_op || + !d_inode(lower_dentry)->i_op->readlink) { + err = -EINVAL; + goto out; + } + + err = d_inode(lower_dentry)->i_op->readlink(lower_dentry, + buf, bufsiz); + if (err < 0) + goto out; + fsstack_copy_attr_atime(d_inode(dentry), d_inode(lower_dentry)); + +out: + sdcardfs_put_lower_path(dentry, &lower_path); + return err; +} +#endif + +#if 0 +static const char *sdcardfs_follow_link(struct dentry *dentry, void **cookie) +{ + char *buf; + int len = PAGE_SIZE, err; + mm_segment_t old_fs; + + /* This is freed by the put_link method assuming a successful call. */ + buf = kmalloc(len, GFP_KERNEL); + if (!buf) { + buf = ERR_PTR(-ENOMEM); + return buf; + } + + /* read the symlink, and then we will follow it */ + old_fs = get_fs(); + set_fs(KERNEL_DS); + err = sdcardfs_readlink(dentry, buf, len); + set_fs(old_fs); + if (err < 0) { + kfree(buf); + buf = ERR_PTR(err); + } else { + buf[err] = '\0'; + } + return *cookie = buf; +} +#endif + +static int sdcardfs_permission_wrn(struct inode *inode, int mask) +{ + WARN(1, "sdcardfs does not support permission. Use permission2.\n"); + return -EINVAL; +} + +void copy_attrs(struct inode *dest, const struct inode *src) +{ + dest->i_mode = src->i_mode; + dest->i_uid = src->i_uid; + dest->i_gid = src->i_gid; + dest->i_rdev = src->i_rdev; + dest->i_atime = src->i_atime; + dest->i_mtime = src->i_mtime; + dest->i_ctime = src->i_ctime; + dest->i_blkbits = src->i_blkbits; + dest->i_flags = src->i_flags; +#ifdef CONFIG_FS_POSIX_ACL + dest->i_acl = src->i_acl; +#endif +#ifdef CONFIG_SECURITY + dest->i_security = src->i_security; +#endif +} + +static int sdcardfs_permission(struct vfsmount *mnt, struct inode *inode, int mask) +{ + int err; + struct inode tmp; + struct inode *top = grab_top(SDCARDFS_I(inode)); + + if (!top) { + release_top(SDCARDFS_I(inode)); + WARN(1, "Top value was null!\n"); + return -EINVAL; + } + + /* + * Permission check on sdcardfs inode. + * Calling process should have AID_SDCARD_RW permission + * Since generic_permission only needs i_mode, i_uid, + * i_gid, and i_sb, we can create a fake inode to pass + * this information down in. + * + * The underlying code may attempt to take locks in some + * cases for features we're not using, but if that changes, + * locks must be dealt with to avoid undefined behavior. + */ + copy_attrs(&tmp, inode); + tmp.i_uid = make_kuid(&init_user_ns, SDCARDFS_I(top)->d_uid); + tmp.i_gid = make_kgid(&init_user_ns, get_gid(mnt, SDCARDFS_I(top))); + tmp.i_mode = (inode->i_mode & S_IFMT) | get_mode(mnt, SDCARDFS_I(top)); + release_top(SDCARDFS_I(inode)); + tmp.i_sb = inode->i_sb; + if (IS_POSIXACL(inode)) + printk(KERN_WARNING "%s: This may be undefined behavior... \n", __func__); + err = generic_permission(&tmp, mask); + /* XXX + * Original sdcardfs code calls inode_permission(lower_inode,.. ) + * for checking inode permission. But doing such things here seems + * duplicated work, because the functions called after this func, + * such as vfs_create, vfs_unlink, vfs_rename, and etc, + * does exactly same thing, i.e., they calls inode_permission(). + * So we just let they do the things. + * If there are any security hole, just uncomment following if block. + */ +#if 0 + if (!err) { + /* + * Permission check on lower_inode(=EXT4). + * we check it with AID_MEDIA_RW permission + */ + struct inode *lower_inode; + OVERRIDE_CRED(SDCARDFS_SB(inode->sb)); + + lower_inode = sdcardfs_lower_inode(inode); + err = inode_permission(lower_inode, mask); + + REVERT_CRED(); + } +#endif + return err; + +} + +static int sdcardfs_setattr_wrn(struct dentry *dentry, struct iattr *ia) +{ + WARN(1, "sdcardfs does not support setattr. User setattr2.\n"); + return -EINVAL; +} + +static int sdcardfs_setattr(struct vfsmount *mnt, struct dentry *dentry, struct iattr *ia) +{ + int err; + struct dentry *lower_dentry; + struct vfsmount *lower_mnt; + struct inode *inode; + struct inode *lower_inode; + struct path lower_path; + struct iattr lower_ia; + struct dentry *parent; + struct inode tmp; + struct dentry tmp_d; + struct inode *top; + const struct cred *saved_cred = NULL; + + inode = d_inode(dentry); + top = grab_top(SDCARDFS_I(inode)); + + if (!top) { + release_top(SDCARDFS_I(inode)); + return -EINVAL; + } + + /* + * Permission check on sdcardfs inode. + * Calling process should have AID_SDCARD_RW permission + * Since generic_permission only needs i_mode, i_uid, + * i_gid, and i_sb, we can create a fake inode to pass + * this information down in. + * + * The underlying code may attempt to take locks in some + * cases for features we're not using, but if that changes, + * locks must be dealt with to avoid undefined behavior. + * + */ + copy_attrs(&tmp, inode); + tmp.i_uid = make_kuid(&init_user_ns, SDCARDFS_I(top)->d_uid); + tmp.i_gid = make_kgid(&init_user_ns, get_gid(mnt, SDCARDFS_I(top))); + tmp.i_mode = (inode->i_mode & S_IFMT) | get_mode(mnt, SDCARDFS_I(top)); + tmp.i_size = i_size_read(inode); + release_top(SDCARDFS_I(inode)); + tmp.i_sb = inode->i_sb; + tmp_d.d_inode = &tmp; + + /* + * Check if user has permission to change dentry. We don't check if + * this user can change the lower inode: that should happen when + * calling notify_change on the lower inode. + */ + err = setattr_prepare(&tmp_d, ia); + + if (!err) { + /* check the Android group ID */ + parent = dget_parent(dentry); + if(!check_caller_access_to_name(d_inode(parent), dentry->d_name.name)) { + printk(KERN_INFO "%s: need to check the caller's gid in packages.list\n" + " dentry: %s, task:%s\n", + __func__, dentry->d_name.name, current->comm); + err = -EACCES; + } + dput(parent); + } + + if (err) + goto out_err; + + /* save current_cred and override it */ + OVERRIDE_CRED(SDCARDFS_SB(dentry->d_sb), saved_cred); + + sdcardfs_get_lower_path(dentry, &lower_path); + lower_dentry = lower_path.dentry; + lower_mnt = lower_path.mnt; + lower_inode = sdcardfs_lower_inode(inode); + + /* prepare our own lower struct iattr (with the lower file) */ + memcpy(&lower_ia, ia, sizeof(lower_ia)); + if (ia->ia_valid & ATTR_FILE) + lower_ia.ia_file = sdcardfs_lower_file(ia->ia_file); + + lower_ia.ia_valid &= ~(ATTR_UID | ATTR_GID | ATTR_MODE); + + /* + * If shrinking, first truncate upper level to cancel writing dirty + * pages beyond the new eof; and also if its' maxbytes is more + * limiting (fail with -EFBIG before making any change to the lower + * level). There is no need to vmtruncate the upper level + * afterwards in the other cases: we fsstack_copy_inode_size from + * the lower level. + */ + if (current->mm) + down_write(¤t->mm->mmap_sem); + if (ia->ia_valid & ATTR_SIZE) { + err = inode_newsize_ok(&tmp, ia->ia_size); + if (err) { + if (current->mm) + up_write(¤t->mm->mmap_sem); + goto out; + } + truncate_setsize(inode, ia->ia_size); + } + + /* + * mode change is for clearing setuid/setgid bits. Allow lower fs + * to interpret this in its own way. + */ + if (lower_ia.ia_valid & (ATTR_KILL_SUID | ATTR_KILL_SGID)) + lower_ia.ia_valid &= ~ATTR_MODE; + + /* notify the (possibly copied-up) lower inode */ + /* + * Note: we use d_inode(lower_dentry), because lower_inode may be + * unlinked (no inode->i_sb and i_ino==0. This happens if someone + * tries to open(), unlink(), then ftruncate() a file. + */ + inode_lock(d_inode(lower_dentry)); + err = notify_change2(lower_mnt, lower_dentry, &lower_ia, /* note: lower_ia */ + NULL); + inode_unlock(d_inode(lower_dentry)); + if (current->mm) + up_write(¤t->mm->mmap_sem); + if (err) + goto out; + + /* get attributes from the lower inode and update derived permissions */ + sdcardfs_copy_and_fix_attrs(inode, lower_inode); + + /* + * Not running fsstack_copy_inode_size(inode, lower_inode), because + * VFS should update our inode size, and notify_change on + * lower_inode should update its size. + */ + +out: + sdcardfs_put_lower_path(dentry, &lower_path); + REVERT_CRED(saved_cred); +out_err: + return err; +} + +static int sdcardfs_fillattr(struct vfsmount *mnt, struct inode *inode, struct kstat *stat) +{ + struct sdcardfs_inode_info *info = SDCARDFS_I(inode); + struct inode *top = grab_top(info); + if (!top) + return -EINVAL; + + stat->dev = inode->i_sb->s_dev; + stat->ino = inode->i_ino; + stat->mode = (inode->i_mode & S_IFMT) | get_mode(mnt, SDCARDFS_I(top)); + stat->nlink = inode->i_nlink; + stat->uid = make_kuid(&init_user_ns, SDCARDFS_I(top)->d_uid); + stat->gid = make_kgid(&init_user_ns, get_gid(mnt, SDCARDFS_I(top))); + stat->rdev = inode->i_rdev; + stat->size = i_size_read(inode); + stat->atime = inode->i_atime; + stat->mtime = inode->i_mtime; + stat->ctime = inode->i_ctime; + stat->blksize = (1 << inode->i_blkbits); + stat->blocks = inode->i_blocks; + release_top(info); + return 0; +} + +static int sdcardfs_getattr(struct vfsmount *mnt, struct dentry *dentry, + struct kstat *stat) +{ + struct dentry *lower_dentry; + struct inode *inode; + struct inode *lower_inode; + struct path lower_path; + struct dentry *parent; + int err; + + parent = dget_parent(dentry); + if(!check_caller_access_to_name(d_inode(parent), dentry->d_name.name)) { + printk(KERN_INFO "%s: need to check the caller's gid in packages.list\n" + " dentry: %s, task:%s\n", + __func__, dentry->d_name.name, current->comm); + dput(parent); + return -EACCES; + } + dput(parent); + + inode = d_inode(dentry); + + sdcardfs_get_lower_path(dentry, &lower_path); + lower_dentry = lower_path.dentry; + lower_inode = sdcardfs_lower_inode(inode); + + sdcardfs_copy_and_fix_attrs(inode, lower_inode); + fsstack_copy_inode_size(inode, lower_inode); + + err = sdcardfs_fillattr(mnt, inode, stat); + sdcardfs_put_lower_path(dentry, &lower_path); + return err; +} + +const struct inode_operations sdcardfs_symlink_iops = { + .permission2 = sdcardfs_permission, + .setattr2 = sdcardfs_setattr, + /* XXX Following operations are implemented, + * but FUSE(sdcard) or FAT does not support them + * These methods are *NOT* perfectly tested. + .readlink = sdcardfs_readlink, + .follow_link = sdcardfs_follow_link, + .put_link = kfree_put_link, + */ +}; + +const struct inode_operations sdcardfs_dir_iops = { + .create = sdcardfs_create, + .lookup = sdcardfs_lookup, + .permission = sdcardfs_permission_wrn, + .permission2 = sdcardfs_permission, + .unlink = sdcardfs_unlink, + .mkdir = sdcardfs_mkdir, + .rmdir = sdcardfs_rmdir, + .rename = sdcardfs_rename, + .setattr = sdcardfs_setattr_wrn, + .setattr2 = sdcardfs_setattr, + .getattr = sdcardfs_getattr, + /* XXX Following operations are implemented, + * but FUSE(sdcard) or FAT does not support them + * These methods are *NOT* perfectly tested. + .symlink = sdcardfs_symlink, + .link = sdcardfs_link, + .mknod = sdcardfs_mknod, + */ +}; + +const struct inode_operations sdcardfs_main_iops = { + .permission = sdcardfs_permission_wrn, + .permission2 = sdcardfs_permission, + .setattr = sdcardfs_setattr_wrn, + .setattr2 = sdcardfs_setattr, + .getattr = sdcardfs_getattr, +}; diff --git a/fs/sdcardfs/lookup.c b/fs/sdcardfs/lookup.c new file mode 100644 index 00000000000000..d271617290eeb2 --- /dev/null +++ b/fs/sdcardfs/lookup.c @@ -0,0 +1,406 @@ +/* + * fs/sdcardfs/lookup.c + * + * Copyright (c) 2013 Samsung Electronics Co. Ltd + * Authors: Daeho Jeong, Woojoong Lee, Seunghwan Hyun, + * Sunghwan Yun, Sungjong Seo + * + * This program has been developed as a stackable file system based on + * the WrapFS which written by + * + * Copyright (c) 1998-2011 Erez Zadok + * Copyright (c) 2009 Shrikar Archak + * Copyright (c) 2003-2011 Stony Brook University + * Copyright (c) 2003-2011 The Research Foundation of SUNY + * + * This file is dual licensed. It may be redistributed and/or modified + * under the terms of the Apache 2.0 License OR version 2 of the GNU + * General Public License. + */ + +#include "sdcardfs.h" +#include "linux/delay.h" + +/* The dentry cache is just so we have properly sized dentries */ +static struct kmem_cache *sdcardfs_dentry_cachep; + +int sdcardfs_init_dentry_cache(void) +{ + sdcardfs_dentry_cachep = + kmem_cache_create("sdcardfs_dentry", + sizeof(struct sdcardfs_dentry_info), + 0, SLAB_RECLAIM_ACCOUNT, NULL); + + return sdcardfs_dentry_cachep ? 0 : -ENOMEM; +} + +void sdcardfs_destroy_dentry_cache(void) +{ + if (sdcardfs_dentry_cachep) + kmem_cache_destroy(sdcardfs_dentry_cachep); +} + +void free_dentry_private_data(struct dentry *dentry) +{ + if (!dentry || !dentry->d_fsdata) + return; + kmem_cache_free(sdcardfs_dentry_cachep, dentry->d_fsdata); + dentry->d_fsdata = NULL; +} + +/* allocate new dentry private data */ +int new_dentry_private_data(struct dentry *dentry) +{ + struct sdcardfs_dentry_info *info = SDCARDFS_D(dentry); + + /* use zalloc to init dentry_info.lower_path */ + info = kmem_cache_zalloc(sdcardfs_dentry_cachep, GFP_ATOMIC); + if (!info) + return -ENOMEM; + + spin_lock_init(&info->lock); + dentry->d_fsdata = info; + + return 0; +} + +struct inode_data { + struct inode *lower_inode; + userid_t id; +}; + +static int sdcardfs_inode_test(struct inode *inode, void *candidate_data/*void *candidate_lower_inode*/) +{ + struct inode *current_lower_inode = sdcardfs_lower_inode(inode); + userid_t current_userid = SDCARDFS_I(inode)->userid; + if (current_lower_inode == ((struct inode_data *)candidate_data)->lower_inode && + current_userid == ((struct inode_data *)candidate_data)->id) + return 1; /* found a match */ + else + return 0; /* no match */ +} + +static int sdcardfs_inode_set(struct inode *inode, void *lower_inode) +{ + /* we do actual inode initialization in sdcardfs_iget */ + return 0; +} + +struct inode *sdcardfs_iget(struct super_block *sb, struct inode *lower_inode, userid_t id) +{ + struct sdcardfs_inode_info *info; + struct inode_data data; + struct inode *inode; /* the new inode to return */ + int err; + + data.id = id; + data.lower_inode = lower_inode; + inode = iget5_locked(sb, /* our superblock */ + /* + * hashval: we use inode number, but we can + * also use "(unsigned long)lower_inode" + * instead. + */ + lower_inode->i_ino, /* hashval */ + sdcardfs_inode_test, /* inode comparison function */ + sdcardfs_inode_set, /* inode init function */ + &data); /* data passed to test+set fxns */ + if (!inode) { + err = -EACCES; + iput(lower_inode); + return ERR_PTR(err); + } + /* if found a cached inode, then just return it */ + if (!(inode->i_state & I_NEW)) + return inode; + + /* initialize new inode */ + info = SDCARDFS_I(inode); + + inode->i_ino = lower_inode->i_ino; + if (!igrab(lower_inode)) { + err = -ESTALE; + return ERR_PTR(err); + } + sdcardfs_set_lower_inode(inode, lower_inode); + + inode->i_version++; + + /* use different set of inode ops for symlinks & directories */ + if (S_ISDIR(lower_inode->i_mode)) + inode->i_op = &sdcardfs_dir_iops; + else if (S_ISLNK(lower_inode->i_mode)) + inode->i_op = &sdcardfs_symlink_iops; + else + inode->i_op = &sdcardfs_main_iops; + + /* use different set of file ops for directories */ + if (S_ISDIR(lower_inode->i_mode)) + inode->i_fop = &sdcardfs_dir_fops; + else + inode->i_fop = &sdcardfs_main_fops; + + inode->i_mapping->a_ops = &sdcardfs_aops; + + inode->i_atime.tv_sec = 0; + inode->i_atime.tv_nsec = 0; + inode->i_mtime.tv_sec = 0; + inode->i_mtime.tv_nsec = 0; + inode->i_ctime.tv_sec = 0; + inode->i_ctime.tv_nsec = 0; + + /* properly initialize special inodes */ + if (S_ISBLK(lower_inode->i_mode) || S_ISCHR(lower_inode->i_mode) || + S_ISFIFO(lower_inode->i_mode) || S_ISSOCK(lower_inode->i_mode)) + init_special_inode(inode, lower_inode->i_mode, + lower_inode->i_rdev); + + /* all well, copy inode attributes */ + sdcardfs_copy_and_fix_attrs(inode, lower_inode); + fsstack_copy_inode_size(inode, lower_inode); + + unlock_new_inode(inode); + return inode; +} + +/* + * Connect a sdcardfs inode dentry/inode with several lower ones. This is + * the classic stackable file system "vnode interposition" action. + * + * @dentry: sdcardfs's dentry which interposes on lower one + * @sb: sdcardfs's super_block + * @lower_path: the lower path (caller does path_get/put) + */ +int sdcardfs_interpose(struct dentry *dentry, struct super_block *sb, + struct path *lower_path, userid_t id) +{ + int err = 0; + struct inode *inode; + struct inode *lower_inode; + struct super_block *lower_sb; + + lower_inode = d_inode(lower_path->dentry); + lower_sb = sdcardfs_lower_super(sb); + + /* check that the lower file system didn't cross a mount point */ + if (lower_inode->i_sb != lower_sb) { + err = -EXDEV; + goto out; + } + + /* + * We allocate our new inode below by calling sdcardfs_iget, + * which will initialize some of the new inode's fields + */ + + /* inherit lower inode number for sdcardfs's inode */ + inode = sdcardfs_iget(sb, lower_inode, id); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + goto out; + } + + d_add(dentry, inode); + update_derived_permission_lock(dentry); +out: + return err; +} + +/* + * Main driver function for sdcardfs's lookup. + * + * Returns: NULL (ok), ERR_PTR if an error occurred. + * Fills in lower_parent_path with on success. + */ +static struct dentry *__sdcardfs_lookup(struct dentry *dentry, + unsigned int flags, struct path *lower_parent_path, userid_t id) +{ + int err = 0; + struct vfsmount *lower_dir_mnt; + struct dentry *lower_dir_dentry = NULL; + struct dentry *lower_dentry; + const char *name; + struct path lower_path; + struct qstr this; + struct sdcardfs_sb_info *sbi; + + sbi = SDCARDFS_SB(dentry->d_sb); + /* must initialize dentry operations */ + d_set_d_op(dentry, &sdcardfs_ci_dops); + + if (IS_ROOT(dentry)) + goto out; + + name = dentry->d_name.name; + + /* now start the actual lookup procedure */ + lower_dir_dentry = lower_parent_path->dentry; + lower_dir_mnt = lower_parent_path->mnt; + + /* Use vfs_path_lookup to check if the dentry exists or not */ + err = vfs_path_lookup(lower_dir_dentry, lower_dir_mnt, name, 0, + &lower_path); + /* check for other cases */ + if (err == -ENOENT) { + struct dentry *child; + struct dentry *match = NULL; + inode_lock(d_inode(lower_dir_dentry)); + spin_lock(&lower_dir_dentry->d_lock); + list_for_each_entry(child, &lower_dir_dentry->d_subdirs, d_child) { + if (child && d_inode(child)) { + if (strcasecmp(child->d_name.name, name)==0) { + match = dget(child); + break; + } + } + } + spin_unlock(&lower_dir_dentry->d_lock); + inode_unlock(d_inode(lower_dir_dentry)); + if (match) { + err = vfs_path_lookup(lower_dir_dentry, + lower_dir_mnt, + match->d_name.name, 0, + &lower_path); + dput(match); + } + } + + /* no error: handle positive dentries */ + if (!err) { + /* check if the dentry is an obb dentry + * if true, the lower_inode must be replaced with + * the inode of the graft path */ + + if(need_graft_path(dentry)) { + + /* setup_obb_dentry() + * The lower_path will be stored to the dentry's orig_path + * and the base obbpath will be copyed to the lower_path variable. + * if an error returned, there's no change in the lower_path + * returns: -ERRNO if error (0: no error) */ + err = setup_obb_dentry(dentry, &lower_path); + + if(err) { + /* if the sbi->obbpath is not available, we can optionally + * setup the lower_path with its orig_path. + * but, the current implementation just returns an error + * because the sdcard daemon also regards this case as + * a lookup fail. */ + printk(KERN_INFO "sdcardfs: base obbpath is not available\n"); + sdcardfs_put_reset_orig_path(dentry); + goto out; + } + } + + sdcardfs_set_lower_path(dentry, &lower_path); + err = sdcardfs_interpose(dentry, dentry->d_sb, &lower_path, id); + if (err) /* path_put underlying path on error */ + sdcardfs_put_reset_lower_path(dentry); + goto out; + } + + /* + * We don't consider ENOENT an error, and we want to return a + * negative dentry. + */ + if (err && err != -ENOENT) + goto out; + + /* instatiate a new negative dentry */ + this.name = name; + this.len = strlen(name); + this.hash = full_name_hash(dentry, this.name, this.len); + lower_dentry = d_lookup(lower_dir_dentry, &this); + if (lower_dentry) + goto setup_lower; + + lower_dentry = d_alloc(lower_dir_dentry, &this); + if (!lower_dentry) { + err = -ENOMEM; + goto out; + } + d_add(lower_dentry, NULL); /* instantiate and hash */ + +setup_lower: + lower_path.dentry = lower_dentry; + lower_path.mnt = mntget(lower_dir_mnt); + sdcardfs_set_lower_path(dentry, &lower_path); + + /* + * If the intent is to create a file, then don't return an error, so + * the VFS will continue the process of making this negative dentry + * into a positive one. + */ + if (flags & (LOOKUP_CREATE|LOOKUP_RENAME_TARGET)) + err = 0; + +out: + return ERR_PTR(err); +} + +/* + * On success: + * fills dentry object appropriate values and returns NULL. + * On fail (== error) + * returns error ptr + * + * @dir : Parent inode. + * @dentry : Target dentry to lookup. we should set each of fields. + * (dentry->d_name is initialized already) + * @nd : nameidata of parent inode + */ +struct dentry *sdcardfs_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + struct dentry *ret = NULL, *parent; + struct path lower_parent_path; + int err = 0; + const struct cred *saved_cred = NULL; + + parent = dget_parent(dentry); + + if(!check_caller_access_to_name(d_inode(parent), dentry->d_name.name)) { + ret = ERR_PTR(-EACCES); + printk(KERN_INFO "%s: need to check the caller's gid in packages.list\n" + " dentry: %s, task:%s\n", + __func__, dentry->d_name.name, current->comm); + goto out_err; + } + + /* save current_cred and override it */ + OVERRIDE_CRED_PTR(SDCARDFS_SB(dir->i_sb), saved_cred); + + sdcardfs_get_lower_path(parent, &lower_parent_path); + + /* allocate dentry private data. We free it in ->d_release */ + err = new_dentry_private_data(dentry); + if (err) { + ret = ERR_PTR(err); + goto out; + } + + ret = __sdcardfs_lookup(dentry, flags, &lower_parent_path, SDCARDFS_I(dir)->userid); + if (IS_ERR(ret)) + { + goto out; + } + if (ret) + dentry = ret; + if (d_inode(dentry)) { + fsstack_copy_attr_times(d_inode(dentry), + sdcardfs_lower_inode(d_inode(dentry))); + /* get derived permission */ + get_derived_permission(parent, dentry); + fixup_tmp_permissions(d_inode(dentry)); + } + /* update parent directory's atime */ + fsstack_copy_attr_atime(d_inode(parent), + sdcardfs_lower_inode(d_inode(parent))); + +out: + sdcardfs_put_lower_path(parent, &lower_parent_path); + REVERT_CRED(saved_cred); +out_err: + dput(parent); + return ret; +} diff --git a/fs/sdcardfs/main.c b/fs/sdcardfs/main.c new file mode 100644 index 00000000000000..7a8eae29e44dcf --- /dev/null +++ b/fs/sdcardfs/main.c @@ -0,0 +1,477 @@ +/* + * fs/sdcardfs/main.c + * + * Copyright (c) 2013 Samsung Electronics Co. Ltd + * Authors: Daeho Jeong, Woojoong Lee, Seunghwan Hyun, + * Sunghwan Yun, Sungjong Seo + * + * This program has been developed as a stackable file system based on + * the WrapFS which written by + * + * Copyright (c) 1998-2011 Erez Zadok + * Copyright (c) 2009 Shrikar Archak + * Copyright (c) 2003-2011 Stony Brook University + * Copyright (c) 2003-2011 The Research Foundation of SUNY + * + * This file is dual licensed. It may be redistributed and/or modified + * under the terms of the Apache 2.0 License OR version 2 of the GNU + * General Public License. + */ + +#include "sdcardfs.h" +#include +#include +#include + +enum { + Opt_fsuid, + Opt_fsgid, + Opt_gid, + Opt_debug, + Opt_mask, + Opt_multiuser, // May need? + Opt_userid, + Opt_reserved_mb, + Opt_err, +}; + +static const match_table_t sdcardfs_tokens = { + {Opt_fsuid, "fsuid=%u"}, + {Opt_fsgid, "fsgid=%u"}, + {Opt_gid, "gid=%u"}, + {Opt_debug, "debug"}, + {Opt_mask, "mask=%u"}, + {Opt_userid, "userid=%d"}, + {Opt_multiuser, "multiuser"}, + {Opt_reserved_mb, "reserved_mb=%u"}, + {Opt_err, NULL} +}; + +static int parse_options(struct super_block *sb, char *options, int silent, + int *debug, struct sdcardfs_vfsmount_options *vfsopts, + struct sdcardfs_mount_options *opts) +{ + char *p; + substring_t args[MAX_OPT_ARGS]; + int option; + + /* by default, we use AID_MEDIA_RW as uid, gid */ + opts->fs_low_uid = AID_MEDIA_RW; + opts->fs_low_gid = AID_MEDIA_RW; + vfsopts->mask = 0; + opts->multiuser = false; + opts->fs_user_id = 0; + vfsopts->gid = 0; + /* by default, 0MB is reserved */ + opts->reserved_mb = 0; + + *debug = 0; + + if (!options) + return 0; + + while ((p = strsep(&options, ",")) != NULL) { + int token; + if (!*p) + continue; + + token = match_token(p, sdcardfs_tokens, args); + + switch (token) { + case Opt_debug: + *debug = 1; + break; + case Opt_fsuid: + if (match_int(&args[0], &option)) + return 0; + opts->fs_low_uid = option; + break; + case Opt_fsgid: + if (match_int(&args[0], &option)) + return 0; + opts->fs_low_gid = option; + break; + case Opt_gid: + if (match_int(&args[0], &option)) + return 0; + vfsopts->gid = option; + break; + case Opt_userid: + if (match_int(&args[0], &option)) + return 0; + opts->fs_user_id = option; + break; + case Opt_mask: + if (match_int(&args[0], &option)) + return 0; + vfsopts->mask = option; + break; + case Opt_multiuser: + opts->multiuser = true; + break; + case Opt_reserved_mb: + if (match_int(&args[0], &option)) + return 0; + opts->reserved_mb = option; + break; + /* unknown option */ + default: + if (!silent) { + printk( KERN_ERR "Unrecognized mount option \"%s\" " + "or missing value", p); + } + return -EINVAL; + } + } + + if (*debug) { + printk( KERN_INFO "sdcardfs : options - debug:%d\n", *debug); + printk( KERN_INFO "sdcardfs : options - uid:%d\n", + opts->fs_low_uid); + printk( KERN_INFO "sdcardfs : options - gid:%d\n", + opts->fs_low_gid); + } + + return 0; +} + +int parse_options_remount(struct super_block *sb, char *options, int silent, + struct sdcardfs_vfsmount_options *vfsopts) +{ + char *p; + substring_t args[MAX_OPT_ARGS]; + int option; + int debug; + + if (!options) + return 0; + + while ((p = strsep(&options, ",")) != NULL) { + int token; + if (!*p) + continue; + + token = match_token(p, sdcardfs_tokens, args); + + switch (token) { + case Opt_debug: + debug = 1; + break; + case Opt_gid: + if (match_int(&args[0], &option)) + return 0; + vfsopts->gid = option; + + break; + case Opt_mask: + if (match_int(&args[0], &option)) + return 0; + vfsopts->mask = option; + break; + case Opt_multiuser: + case Opt_userid: + case Opt_fsuid: + case Opt_fsgid: + case Opt_reserved_mb: + printk( KERN_WARNING "Option \"%s\" can't be changed during remount\n", p); + break; + /* unknown option */ + default: + if (!silent) { + printk( KERN_ERR "Unrecognized mount option \"%s\" " + "or missing value", p); + } + return -EINVAL; + } + } + + if (debug) { + printk( KERN_INFO "sdcardfs : options - debug:%d\n", debug); + printk( KERN_INFO "sdcardfs : options - gid:%d\n", vfsopts->gid); + printk( KERN_INFO "sdcardfs : options - mask:%d\n", vfsopts->mask); + } + + return 0; +} + +#if 0 +/* + * our custom d_alloc_root work-alike + * + * we can't use d_alloc_root if we want to use our own interpose function + * unchanged, so we simply call our own "fake" d_alloc_root + */ +static struct dentry *sdcardfs_d_alloc_root(struct super_block *sb) +{ + struct dentry *ret = NULL; + + if (sb) { + static const struct qstr name = { + .name = "/", + .len = 1 + }; + + ret = d_alloc(NULL, &name); + if (ret) { + d_set_d_op(ret, &sdcardfs_ci_dops); + ret->d_sb = sb; + ret->d_parent = ret; + } + } + return ret; +} +#endif + +DEFINE_MUTEX(sdcardfs_super_list_lock); +LIST_HEAD(sdcardfs_super_list); +EXPORT_SYMBOL_GPL(sdcardfs_super_list_lock); +EXPORT_SYMBOL_GPL(sdcardfs_super_list); + +/* + * There is no need to lock the sdcardfs_super_info's rwsem as there is no + * way anyone can have a reference to the superblock at this point in time. + */ +static int sdcardfs_read_super(struct vfsmount *mnt, struct super_block *sb, + const char *dev_name, void *raw_data, int silent) +{ + int err = 0; + int debug; + struct super_block *lower_sb; + struct path lower_path; + struct sdcardfs_sb_info *sb_info; + struct sdcardfs_vfsmount_options *mnt_opt = mnt->data; + struct inode *inode; + + printk(KERN_INFO "sdcardfs version 2.0\n"); + + if (!dev_name) { + printk(KERN_ERR + "sdcardfs: read_super: missing dev_name argument\n"); + err = -EINVAL; + goto out; + } + + printk(KERN_INFO "sdcardfs: dev_name -> %s\n", dev_name); + printk(KERN_INFO "sdcardfs: options -> %s\n", (char *)raw_data); + printk(KERN_INFO "sdcardfs: mnt -> %p\n", mnt); + + /* parse lower path */ + err = kern_path(dev_name, LOOKUP_FOLLOW | LOOKUP_DIRECTORY, + &lower_path); + if (err) { + printk(KERN_ERR "sdcardfs: error accessing lower directory '%s'\n", dev_name); + goto out; + } + + /* allocate superblock private data */ + sb->s_fs_info = kzalloc(sizeof(struct sdcardfs_sb_info), GFP_KERNEL); + if (!SDCARDFS_SB(sb)) { + printk(KERN_CRIT "sdcardfs: read_super: out of memory\n"); + err = -ENOMEM; + goto out_free; + } + + sb_info = sb->s_fs_info; + /* parse options */ + err = parse_options(sb, raw_data, silent, &debug, mnt_opt, &sb_info->options); + if (err) { + printk(KERN_ERR "sdcardfs: invalid options\n"); + goto out_freesbi; + } + + /* set the lower superblock field of upper superblock */ + lower_sb = lower_path.dentry->d_sb; + atomic_inc(&lower_sb->s_active); + sdcardfs_set_lower_super(sb, lower_sb); + + /* inherit maxbytes from lower file system */ + sb->s_maxbytes = lower_sb->s_maxbytes; + + /* + * Our c/m/atime granularity is 1 ns because we may stack on file + * systems whose granularity is as good. + */ + sb->s_time_gran = 1; + + sb->s_magic = SDCARDFS_SUPER_MAGIC; + sb->s_op = &sdcardfs_sops; + + /* get a new inode and allocate our root dentry */ + inode = sdcardfs_iget(sb, d_inode(lower_path.dentry), 0); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + goto out_sput; + } + sb->s_root = d_make_root(inode); + if (!sb->s_root) { + err = -ENOMEM; + goto out_iput; + } + d_set_d_op(sb->s_root, &sdcardfs_ci_dops); + + /* link the upper and lower dentries */ + sb->s_root->d_fsdata = NULL; + err = new_dentry_private_data(sb->s_root); + if (err) + goto out_freeroot; + + /* set the lower dentries for s_root */ + sdcardfs_set_lower_path(sb->s_root, &lower_path); + + /* + * No need to call interpose because we already have a positive + * dentry, which was instantiated by d_make_root. Just need to + * d_rehash it. + */ + d_rehash(sb->s_root); + + /* setup permission policy */ + sb_info->obbpath_s = kzalloc(PATH_MAX, GFP_KERNEL); + mutex_lock(&sdcardfs_super_list_lock); + if(sb_info->options.multiuser) { + setup_derived_state(d_inode(sb->s_root), PERM_PRE_ROOT, sb_info->options.fs_user_id, AID_ROOT, false, d_inode(sb->s_root)); + snprintf(sb_info->obbpath_s, PATH_MAX, "%s/obb", dev_name); + /*err = prepare_dir(sb_info->obbpath_s, + sb_info->options.fs_low_uid, + sb_info->options.fs_low_gid, 00755);*/ + } else { + setup_derived_state(d_inode(sb->s_root), PERM_ROOT, sb_info->options.fs_user_id, AID_ROOT, false, d_inode(sb->s_root)); + snprintf(sb_info->obbpath_s, PATH_MAX, "%s/Android/obb", dev_name); + } + fixup_tmp_permissions(d_inode(sb->s_root)); + sb_info->sb = sb; + list_add(&sb_info->list, &sdcardfs_super_list); + mutex_unlock(&sdcardfs_super_list_lock); + + if (!silent) + printk(KERN_INFO "sdcardfs: mounted on top of %s type %s\n", + dev_name, lower_sb->s_type->name); + goto out; /* all is well */ + + /* no longer needed: free_dentry_private_data(sb->s_root); */ +out_freeroot: + dput(sb->s_root); +out_iput: + iput(inode); +out_sput: + /* drop refs we took earlier */ + atomic_dec(&lower_sb->s_active); +out_freesbi: + kfree(SDCARDFS_SB(sb)); + sb->s_fs_info = NULL; +out_free: + path_put(&lower_path); + +out: + return err; +} + +/* A feature which supports mount_nodev() with options */ +static struct dentry *mount_nodev_with_options(struct vfsmount *mnt, + struct file_system_type *fs_type, int flags, const char *dev_name, void *data, + int (*fill_super)(struct vfsmount *, struct super_block *, const char *, void *, int)) + +{ + int error; + struct super_block *s = sget(fs_type, NULL, set_anon_super, flags, NULL); + + if (IS_ERR(s)) + return ERR_CAST(s); + + s->s_flags = flags; + + error = fill_super(mnt, s, dev_name, data, flags & MS_SILENT ? 1 : 0); + if (error) { + deactivate_locked_super(s); + return ERR_PTR(error); + } + s->s_flags |= MS_ACTIVE; + return dget(s->s_root); +} + +static struct dentry *sdcardfs_mount(struct vfsmount *mnt, + struct file_system_type *fs_type, int flags, + const char *dev_name, void *raw_data) +{ + /* + * dev_name is a lower_path_name, + * raw_data is a option string. + */ + return mount_nodev_with_options(mnt, fs_type, flags, dev_name, + raw_data, sdcardfs_read_super); +} + +static struct dentry *sdcardfs_mount_wrn(struct file_system_type *fs_type, int flags, + const char *dev_name, void *raw_data) +{ + WARN(1, "sdcardfs does not support mount. Use mount2.\n"); + return ERR_PTR(-EINVAL); +} + +void *sdcardfs_alloc_mnt_data(void) { + return kmalloc(sizeof(struct sdcardfs_vfsmount_options), GFP_KERNEL); +} + +void sdcardfs_kill_sb(struct super_block *sb) { + struct sdcardfs_sb_info *sbi; + if (sb->s_magic == SDCARDFS_SUPER_MAGIC) { + sbi = SDCARDFS_SB(sb); + mutex_lock(&sdcardfs_super_list_lock); + list_del(&sbi->list); + mutex_unlock(&sdcardfs_super_list_lock); + } + generic_shutdown_super(sb); +} + +static struct file_system_type sdcardfs_fs_type = { + .owner = THIS_MODULE, + .name = SDCARDFS_NAME, + .mount = sdcardfs_mount_wrn, + .mount2 = sdcardfs_mount, + .alloc_mnt_data = sdcardfs_alloc_mnt_data, + .kill_sb = sdcardfs_kill_sb, + .fs_flags = 0, +}; + +static int __init init_sdcardfs_fs(void) +{ + int err; + + pr_info("Registering sdcardfs " SDCARDFS_VERSION "\n"); + + err = sdcardfs_init_inode_cache(); + if (err) + goto out; + err = sdcardfs_init_dentry_cache(); + if (err) + goto out; + err = packagelist_init(); + if (err) + goto out; + err = register_filesystem(&sdcardfs_fs_type); +out: + if (err) { + sdcardfs_destroy_inode_cache(); + sdcardfs_destroy_dentry_cache(); + packagelist_exit(); + } + return err; +} + +static void __exit exit_sdcardfs_fs(void) +{ + sdcardfs_destroy_inode_cache(); + sdcardfs_destroy_dentry_cache(); + packagelist_exit(); + unregister_filesystem(&sdcardfs_fs_type); + pr_info("Completed sdcardfs module unload\n"); +} + +MODULE_AUTHOR("Erez Zadok, Filesystems and Storage Lab, Stony Brook University" + " (http://www.fsl.cs.sunysb.edu/)"); +MODULE_DESCRIPTION("Wrapfs " SDCARDFS_VERSION + " (http://wrapfs.filesystems.org/)"); +MODULE_LICENSE("GPL"); + +module_init(init_sdcardfs_fs); +module_exit(exit_sdcardfs_fs); diff --git a/fs/sdcardfs/mmap.c b/fs/sdcardfs/mmap.c new file mode 100644 index 00000000000000..ac5f3deae088e3 --- /dev/null +++ b/fs/sdcardfs/mmap.c @@ -0,0 +1,80 @@ +/* + * fs/sdcardfs/mmap.c + * + * Copyright (c) 2013 Samsung Electronics Co. Ltd + * Authors: Daeho Jeong, Woojoong Lee, Seunghwan Hyun, + * Sunghwan Yun, Sungjong Seo + * + * This program has been developed as a stackable file system based on + * the WrapFS which written by + * + * Copyright (c) 1998-2011 Erez Zadok + * Copyright (c) 2009 Shrikar Archak + * Copyright (c) 2003-2011 Stony Brook University + * Copyright (c) 2003-2011 The Research Foundation of SUNY + * + * This file is dual licensed. It may be redistributed and/or modified + * under the terms of the Apache 2.0 License OR version 2 of the GNU + * General Public License. + */ + +#include "sdcardfs.h" + +static int sdcardfs_fault(struct vm_area_struct *vma, struct vm_fault *vmf) +{ + int err; + struct file *file, *lower_file; + const struct vm_operations_struct *lower_vm_ops; + struct vm_area_struct lower_vma; + + memcpy(&lower_vma, vma, sizeof(struct vm_area_struct)); + file = lower_vma.vm_file; + lower_vm_ops = SDCARDFS_F(file)->lower_vm_ops; + BUG_ON(!lower_vm_ops); + + lower_file = sdcardfs_lower_file(file); + /* + * XXX: vm_ops->fault may be called in parallel. Because we have to + * resort to temporarily changing the vma->vm_file to point to the + * lower file, a concurrent invocation of sdcardfs_fault could see a + * different value. In this workaround, we keep a different copy of + * the vma structure in our stack, so we never expose a different + * value of the vma->vm_file called to us, even temporarily. A + * better fix would be to change the calling semantics of ->fault to + * take an explicit file pointer. + */ + lower_vma.vm_file = lower_file; + err = lower_vm_ops->fault(&lower_vma, vmf); + return err; +} + +static ssize_t sdcardfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter) +{ + /* + * This function returns zero on purpose in order to support direct IO. + * __dentry_open checks a_ops->direct_IO and returns EINVAL if it is null. + * + * However, this function won't be called by certain file operations + * including generic fs functions. * reads and writes are delivered to + * the lower file systems and the direct IOs will be handled by them. + * + * NOTE: exceptionally, on the recent kernels (since Linux 3.8.x), + * swap_writepage invokes this function directly. + */ + printk(KERN_INFO "%s, operation is not supported\n", __func__); + return 0; +} + +/* + * XXX: the default address_space_ops for sdcardfs is empty. We cannot set + * our inode->i_mapping->a_ops to NULL because too many code paths expect + * the a_ops vector to be non-NULL. + */ +const struct address_space_operations sdcardfs_aops = { + /* empty on purpose */ + .direct_IO = sdcardfs_direct_IO, +}; + +const struct vm_operations_struct sdcardfs_vm_ops = { + .fault = sdcardfs_fault, +}; diff --git a/fs/sdcardfs/multiuser.h b/fs/sdcardfs/multiuser.h new file mode 100644 index 00000000000000..923ba101dfa9f1 --- /dev/null +++ b/fs/sdcardfs/multiuser.h @@ -0,0 +1,37 @@ +/* + * fs/sdcardfs/multiuser.h + * + * Copyright (c) 2013 Samsung Electronics Co. Ltd + * Authors: Daeho Jeong, Woojoong Lee, Seunghwan Hyun, + * Sunghwan Yun, Sungjong Seo + * + * This program has been developed as a stackable file system based on + * the WrapFS which written by + * + * Copyright (c) 1998-2011 Erez Zadok + * Copyright (c) 2009 Shrikar Archak + * Copyright (c) 2003-2011 Stony Brook University + * Copyright (c) 2003-2011 The Research Foundation of SUNY + * + * This file is dual licensed. It may be redistributed and/or modified + * under the terms of the Apache 2.0 License OR version 2 of the GNU + * General Public License. + */ + +#define MULTIUSER_APP_PER_USER_RANGE 100000 + +typedef uid_t userid_t; +typedef uid_t appid_t; + +static inline userid_t multiuser_get_user_id(uid_t uid) { + return uid / MULTIUSER_APP_PER_USER_RANGE; +} + +static inline appid_t multiuser_get_app_id(uid_t uid) { + return uid % MULTIUSER_APP_PER_USER_RANGE; +} + +static inline uid_t multiuser_get_uid(userid_t userId, appid_t appId) { + return userId * MULTIUSER_APP_PER_USER_RANGE + (appId % MULTIUSER_APP_PER_USER_RANGE); +} + diff --git a/fs/sdcardfs/packagelist.c b/fs/sdcardfs/packagelist.c new file mode 100644 index 00000000000000..03776fa5f26c74 --- /dev/null +++ b/fs/sdcardfs/packagelist.c @@ -0,0 +1,426 @@ +/* + * fs/sdcardfs/packagelist.c + * + * Copyright (c) 2013 Samsung Electronics Co. Ltd + * Authors: Daeho Jeong, Woojoong Lee, Seunghwan Hyun, + * Sunghwan Yun, Sungjong Seo + * + * This program has been developed as a stackable file system based on + * the WrapFS which written by + * + * Copyright (c) 1998-2011 Erez Zadok + * Copyright (c) 2009 Shrikar Archak + * Copyright (c) 2003-2011 Stony Brook University + * Copyright (c) 2003-2011 The Research Foundation of SUNY + * + * This file is dual licensed. It may be redistributed and/or modified + * under the terms of the Apache 2.0 License OR version 2 of the GNU + * General Public License. + */ + +#include "sdcardfs.h" +#include +#include + + +#include +#include +#include + +#include + +struct hashtable_entry { + struct hlist_node hlist; + const char *key; + atomic_t value; +}; + +static DEFINE_HASHTABLE(package_to_appid, 8); + +static struct kmem_cache *hashtable_entry_cachep; + +static unsigned int str_hash(const char *key) { + int i; + unsigned int h = strlen(key); + char *data = (char *)key; + + for (i = 0; i < strlen(key); i++) { + h = h * 31 + *data; + data++; + } + return h; +} + +appid_t get_appid(const char *app_name) +{ + struct hashtable_entry *hash_cur; + unsigned int hash = str_hash(app_name); + appid_t ret_id; + + rcu_read_lock(); + hash_for_each_possible_rcu(package_to_appid, hash_cur, hlist, hash) { + if (!strcasecmp(app_name, hash_cur->key)) { + ret_id = atomic_read(&hash_cur->value); + rcu_read_unlock(); + return ret_id; + } + } + rcu_read_unlock(); + return 0; +} + +/* Kernel has already enforced everything we returned through + * derive_permissions_locked(), so this is used to lock down access + * even further, such as enforcing that apps hold sdcard_rw. */ +int check_caller_access_to_name(struct inode *parent_node, const char* name) { + + /* Always block security-sensitive files at root */ + if (parent_node && SDCARDFS_I(parent_node)->perm == PERM_ROOT) { + if (!strcasecmp(name, "autorun.inf") + || !strcasecmp(name, ".android_secure") + || !strcasecmp(name, "android_secure")) { + return 0; + } + } + + /* Root always has access; access for any other UIDs should always + * be controlled through packages.list. */ + if (from_kuid(&init_user_ns, current_fsuid()) == 0) { + return 1; + } + + /* No extra permissions to enforce */ + return 1; +} + +/* This function is used when file opening. The open flags must be + * checked before calling check_caller_access_to_name() */ +int open_flags_to_access_mode(int open_flags) { + if((open_flags & O_ACCMODE) == O_RDONLY) { + return 0; /* R_OK */ + } else if ((open_flags & O_ACCMODE) == O_WRONLY) { + return 1; /* W_OK */ + } else { + /* Probably O_RDRW, but treat as default to be safe */ + return 1; /* R_OK | W_OK */ + } +} + +static struct hashtable_entry *alloc_packagelist_entry(const char *key, + appid_t value) +{ + struct hashtable_entry *ret = kmem_cache_alloc(hashtable_entry_cachep, + GFP_KERNEL); + if (!ret) + return NULL; + + ret->key = kstrdup(key, GFP_KERNEL); + if (!ret->key) { + kmem_cache_free(hashtable_entry_cachep, ret); + return NULL; + } + + atomic_set(&ret->value, value); + return ret; +} + +static int insert_packagelist_entry_locked(const char *key, appid_t value) +{ + struct hashtable_entry *hash_cur; + struct hashtable_entry *new_entry; + unsigned int hash = str_hash(key); + + hash_for_each_possible_rcu(package_to_appid, hash_cur, hlist, hash) { + if (!strcasecmp(key, hash_cur->key)) { + atomic_set(&hash_cur->value, value); + return 0; + } + } + new_entry = alloc_packagelist_entry(key, value); + if (!new_entry) + return -ENOMEM; + hash_add_rcu(package_to_appid, &new_entry->hlist, hash); + return 0; +} + +static void fixup_perms(struct super_block *sb, const char *key) { + if (sb && sb->s_magic == SDCARDFS_SUPER_MAGIC) { + fixup_perms_recursive(sb->s_root, key, strlen(key)); + } +} + +static void fixup_all_perms(const char *key) +{ + struct sdcardfs_sb_info *sbinfo; + list_for_each_entry(sbinfo, &sdcardfs_super_list, list) + if (sbinfo) + fixup_perms(sbinfo->sb, key); +} + +static int insert_packagelist_entry(const char *key, appid_t value) +{ + int err; + + mutex_lock(&sdcardfs_super_list_lock); + err = insert_packagelist_entry_locked(key, value); + if (!err) + fixup_all_perms(key); + mutex_unlock(&sdcardfs_super_list_lock); + + return err; +} + +static void free_packagelist_entry(struct hashtable_entry *entry) +{ + kfree(entry->key); + hash_del_rcu(&entry->hlist); + kmem_cache_free(hashtable_entry_cachep, entry); +} + +static void remove_packagelist_entry_locked(const char *key) +{ + struct hashtable_entry *hash_cur; + unsigned int hash = str_hash(key); + + hash_for_each_possible_rcu(package_to_appid, hash_cur, hlist, hash) { + if (!strcasecmp(key, hash_cur->key)) { + hash_del_rcu(&hash_cur->hlist); + synchronize_rcu(); + free_packagelist_entry(hash_cur); + return; + } + } +} + +static void remove_packagelist_entry(const char *key) +{ + mutex_lock(&sdcardfs_super_list_lock); + remove_packagelist_entry_locked(key); + fixup_all_perms(key); + mutex_unlock(&sdcardfs_super_list_lock); + return; +} + +static void packagelist_destroy(void) +{ + struct hashtable_entry *hash_cur; + struct hlist_node *h_t; + HLIST_HEAD(free_list); + int i; + mutex_lock(&sdcardfs_super_list_lock); + hash_for_each_rcu(package_to_appid, i, hash_cur, hlist) { + hash_del_rcu(&hash_cur->hlist); + hlist_add_head(&hash_cur->hlist, &free_list); + + } + synchronize_rcu(); + hlist_for_each_entry_safe(hash_cur, h_t, &free_list, hlist) + free_packagelist_entry(hash_cur); + mutex_unlock(&sdcardfs_super_list_lock); + printk(KERN_INFO "sdcardfs: destroyed packagelist pkgld\n"); +} + +struct package_appid { + struct config_item item; + int add_pid; +}; + +static inline struct package_appid *to_package_appid(struct config_item *item) +{ + return item ? container_of(item, struct package_appid, item) : NULL; +} + +static ssize_t package_appid_attr_show(struct config_item *item, + char *page) +{ + return scnprintf(page, PAGE_SIZE, "%u\n", get_appid(item->ci_name)); +} + +static ssize_t package_appid_attr_store(struct config_item *item, + const char *page, size_t count) +{ + struct package_appid *package_appid = to_package_appid(item); + unsigned int tmp; + int ret; + + ret = kstrtouint(page, 10, &tmp); + if (ret) + return ret; + + ret = insert_packagelist_entry(item->ci_name, tmp); + package_appid->add_pid = tmp; + if (ret) + return ret; + + return count; +} + +static struct configfs_attribute package_appid_attr_add_pid = { + .ca_owner = THIS_MODULE, + .ca_name = "appid", + .ca_mode = S_IRUGO | S_IWUGO, + .show = package_appid_attr_show, + .store = package_appid_attr_store, +}; + +static struct configfs_attribute *package_appid_attrs[] = { + &package_appid_attr_add_pid, + NULL, +}; + +static void package_appid_release(struct config_item *item) +{ + printk(KERN_INFO "sdcardfs: removing %s\n", item->ci_dentry->d_name.name); + /* item->ci_name is freed already, so we rely on the dentry */ + remove_packagelist_entry(item->ci_dentry->d_name.name); + kfree(to_package_appid(item)); +} + +static struct configfs_item_operations package_appid_item_ops = { + .release = package_appid_release, +}; + +static struct config_item_type package_appid_type = { + .ct_item_ops = &package_appid_item_ops, + .ct_attrs = package_appid_attrs, + .ct_owner = THIS_MODULE, +}; + + +struct sdcardfs_packages { + struct config_group group; +}; + +static inline struct sdcardfs_packages *to_sdcardfs_packages(struct config_item *item) +{ + return item ? container_of(to_config_group(item), struct sdcardfs_packages, group) : NULL; +} + +static struct config_item *sdcardfs_packages_make_item(struct config_group *group, const char *name) +{ + struct package_appid *package_appid; + + package_appid = kzalloc(sizeof(struct package_appid), GFP_KERNEL); + if (!package_appid) + return ERR_PTR(-ENOMEM); + + config_item_init_type_name(&package_appid->item, name, + &package_appid_type); + + package_appid->add_pid = 0; + + return &package_appid->item; +} + +static ssize_t packages_attr_show(struct config_item *item, + char *page) +{ + struct hashtable_entry *hash_cur; + int i; + int count = 0, written = 0; + const char errormsg[] = "\n"; + + rcu_read_lock(); + hash_for_each_rcu(package_to_appid, i, hash_cur, hlist) { + written = scnprintf(page + count, PAGE_SIZE - sizeof(errormsg) - count, "%s %d\n", + (const char *)hash_cur->key, atomic_read(&hash_cur->value)); + if (count + written == PAGE_SIZE - sizeof(errormsg)) { + count += scnprintf(page + count, PAGE_SIZE - count, errormsg); + break; + } + count += written; + } + rcu_read_unlock(); + + return count; +} + +static struct configfs_attribute sdcardfs_packages_attr_description = { + .ca_owner = THIS_MODULE, + .ca_name = "packages_gid.list", + .ca_mode = S_IRUGO, + .show = packages_attr_show, +}; + +static struct configfs_attribute *sdcardfs_packages_attrs[] = { + &sdcardfs_packages_attr_description, + NULL, +}; + +static void sdcardfs_packages_release(struct config_item *item) +{ + + printk(KERN_INFO "sdcardfs: destroyed something?\n"); + kfree(to_sdcardfs_packages(item)); +} + +static struct configfs_item_operations sdcardfs_packages_item_ops = { + .release = sdcardfs_packages_release, +}; + +/* + * Note that, since no extra work is required on ->drop_item(), + * no ->drop_item() is provided. + */ +static struct configfs_group_operations sdcardfs_packages_group_ops = { + .make_item = sdcardfs_packages_make_item, +}; + +static struct config_item_type sdcardfs_packages_type = { + .ct_item_ops = &sdcardfs_packages_item_ops, + .ct_group_ops = &sdcardfs_packages_group_ops, + .ct_attrs = sdcardfs_packages_attrs, + .ct_owner = THIS_MODULE, +}; + +static struct configfs_subsystem sdcardfs_packages_subsys = { + .su_group = { + .cg_item = { + .ci_namebuf = "sdcardfs", + .ci_type = &sdcardfs_packages_type, + }, + }, +}; + +static int configfs_sdcardfs_init(void) +{ + int ret; + struct configfs_subsystem *subsys = &sdcardfs_packages_subsys; + + config_group_init(&subsys->su_group); + mutex_init(&subsys->su_mutex); + ret = configfs_register_subsystem(subsys); + if (ret) { + printk(KERN_ERR "Error %d while registering subsystem %s\n", + ret, + subsys->su_group.cg_item.ci_namebuf); + } + return ret; +} + +static void configfs_sdcardfs_exit(void) +{ + configfs_unregister_subsystem(&sdcardfs_packages_subsys); +} + +int packagelist_init(void) +{ + hashtable_entry_cachep = + kmem_cache_create("packagelist_hashtable_entry", + sizeof(struct hashtable_entry), 0, 0, NULL); + if (!hashtable_entry_cachep) { + printk(KERN_ERR "sdcardfs: failed creating pkgl_hashtable entry slab cache\n"); + return -ENOMEM; + } + + configfs_sdcardfs_init(); + return 0; +} + +void packagelist_exit(void) +{ + configfs_sdcardfs_exit(); + packagelist_destroy(); + if (hashtable_entry_cachep) + kmem_cache_destroy(hashtable_entry_cachep); +} diff --git a/fs/sdcardfs/sdcardfs.h b/fs/sdcardfs/sdcardfs.h new file mode 100644 index 00000000000000..66a97ef8d26159 --- /dev/null +++ b/fs/sdcardfs/sdcardfs.h @@ -0,0 +1,580 @@ +/* + * fs/sdcardfs/sdcardfs.h + * + * The sdcardfs v2.0 + * This file system replaces the sdcard daemon on Android + * On version 2.0, some of the daemon functions have been ported + * to support the multi-user concepts of Android 4.4 + * + * Copyright (c) 2013 Samsung Electronics Co. Ltd + * Authors: Daeho Jeong, Woojoong Lee, Seunghwan Hyun, + * Sunghwan Yun, Sungjong Seo + * + * This program has been developed as a stackable file system based on + * the WrapFS which written by + * + * Copyright (c) 1998-2011 Erez Zadok + * Copyright (c) 2009 Shrikar Archak + * Copyright (c) 2003-2011 Stony Brook University + * Copyright (c) 2003-2011 The Research Foundation of SUNY + * + * This file is dual licensed. It may be redistributed and/or modified + * under the terms of the Apache 2.0 License OR version 2 of the GNU + * General Public License. + */ + +#ifndef _SDCARDFS_H_ +#define _SDCARDFS_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "multiuser.h" + +/* the file system name */ +#define SDCARDFS_NAME "sdcardfs" + +/* sdcardfs root inode number */ +#define SDCARDFS_ROOT_INO 1 + +/* useful for tracking code reachability */ +#define UDBG printk(KERN_DEFAULT "DBG:%s:%s:%d\n", __FILE__, __func__, __LINE__) + +#define SDCARDFS_DIRENT_SIZE 256 + +/* temporary static uid settings for development */ +#define AID_ROOT 0 /* uid for accessing /mnt/sdcard & extSdcard */ +#define AID_MEDIA_RW 1023 /* internal media storage write access */ + +#define AID_SDCARD_RW 1015 /* external storage write access */ +#define AID_SDCARD_R 1028 /* external storage read access */ +#define AID_SDCARD_PICS 1033 /* external storage photos access */ +#define AID_SDCARD_AV 1034 /* external storage audio/video access */ +#define AID_SDCARD_ALL 1035 /* access all users external storage */ + +#define AID_PACKAGE_INFO 1027 + + +/* + * Permissions are handled by our permission function. + * We don't want anyone who happens to look at our inode value to prematurely + * block access, so store more permissive values. These are probably never + * used. + */ +#define fixup_tmp_permissions(x) \ + do { \ + (x)->i_uid = make_kuid(&init_user_ns, SDCARDFS_I(x)->d_uid); \ + (x)->i_gid = make_kgid(&init_user_ns, AID_SDCARD_RW); \ + (x)->i_mode = ((x)->i_mode & S_IFMT) | 0775;\ + } while (0) + +/* OVERRIDE_CRED() and REVERT_CRED() + * OVERRID_CRED() + * backup original task->cred + * and modifies task->cred->fsuid/fsgid to specified value. + * REVERT_CRED() + * restore original task->cred->fsuid/fsgid. + * These two macro should be used in pair, and OVERRIDE_CRED() should be + * placed at the beginning of a function, right after variable declaration. + */ +#define OVERRIDE_CRED(sdcardfs_sbi, saved_cred) \ + saved_cred = override_fsids(sdcardfs_sbi); \ + if (!saved_cred) { return -ENOMEM; } + +#define OVERRIDE_CRED_PTR(sdcardfs_sbi, saved_cred) \ + saved_cred = override_fsids(sdcardfs_sbi); \ + if (!saved_cred) { return ERR_PTR(-ENOMEM); } + +#define REVERT_CRED(saved_cred) revert_fsids(saved_cred) + +#define DEBUG_CRED() \ + printk("KAKJAGI: %s:%d fsuid %d fsgid %d\n", \ + __FUNCTION__, __LINE__, \ + (int)current->cred->fsuid, \ + (int)current->cred->fsgid); + +/* Android 5.0 support */ + +/* Permission mode for a specific node. Controls how file permissions + * are derived for children nodes. */ +typedef enum { + /* Nothing special; this node should just inherit from its parent. */ + PERM_INHERIT, + /* This node is one level above a normal root; used for legacy layouts + * which use the first level to represent user_id. */ + PERM_PRE_ROOT, + /* This node is "/" */ + PERM_ROOT, + /* This node is "/Android" */ + PERM_ANDROID, + /* This node is "/Android/data" */ + PERM_ANDROID_DATA, + /* This node is "/Android/obb" */ + PERM_ANDROID_OBB, + /* This node is "/Android/media" */ + PERM_ANDROID_MEDIA, +} perm_t; + +struct sdcardfs_sb_info; +struct sdcardfs_mount_options; + +/* Do not directly use this function. Use OVERRIDE_CRED() instead. */ +const struct cred * override_fsids(struct sdcardfs_sb_info* sbi); +/* Do not directly use this function, use REVERT_CRED() instead. */ +void revert_fsids(const struct cred * old_cred); + +/* operations vectors defined in specific files */ +extern const struct file_operations sdcardfs_main_fops; +extern const struct file_operations sdcardfs_dir_fops; +extern const struct inode_operations sdcardfs_main_iops; +extern const struct inode_operations sdcardfs_dir_iops; +extern const struct inode_operations sdcardfs_symlink_iops; +extern const struct super_operations sdcardfs_sops; +extern const struct dentry_operations sdcardfs_ci_dops; +extern const struct address_space_operations sdcardfs_aops, sdcardfs_dummy_aops; +extern const struct vm_operations_struct sdcardfs_vm_ops; + +extern int sdcardfs_init_inode_cache(void); +extern void sdcardfs_destroy_inode_cache(void); +extern int sdcardfs_init_dentry_cache(void); +extern void sdcardfs_destroy_dentry_cache(void); +extern int new_dentry_private_data(struct dentry *dentry); +extern void free_dentry_private_data(struct dentry *dentry); +extern struct dentry *sdcardfs_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags); +extern struct inode *sdcardfs_iget(struct super_block *sb, + struct inode *lower_inode, userid_t id); +extern int sdcardfs_interpose(struct dentry *dentry, struct super_block *sb, + struct path *lower_path, userid_t id); + +/* file private data */ +struct sdcardfs_file_info { + struct file *lower_file; + const struct vm_operations_struct *lower_vm_ops; +}; + +/* sdcardfs inode data in memory */ +struct sdcardfs_inode_info { + struct inode *lower_inode; + /* state derived based on current position in hierachy */ + perm_t perm; + userid_t userid; + uid_t d_uid; + bool under_android; + /* top folder for ownership */ + struct inode *top; + + struct inode vfs_inode; +}; + + +/* sdcardfs dentry data in memory */ +struct sdcardfs_dentry_info { + spinlock_t lock; /* protects lower_path */ + struct path lower_path; + struct path orig_path; +}; + +struct sdcardfs_mount_options { + uid_t fs_low_uid; + gid_t fs_low_gid; + userid_t fs_user_id; + bool multiuser; + unsigned int reserved_mb; +}; + +struct sdcardfs_vfsmount_options { + gid_t gid; + mode_t mask; +}; + +extern int parse_options_remount(struct super_block *sb, char *options, int silent, + struct sdcardfs_vfsmount_options *vfsopts); + +/* sdcardfs super-block data in memory */ +struct sdcardfs_sb_info { + struct super_block *sb; + struct super_block *lower_sb; + /* derived perm policy : some of options have been added + * to sdcardfs_mount_options (Android 4.4 support) */ + struct sdcardfs_mount_options options; + spinlock_t lock; /* protects obbpath */ + char *obbpath_s; + struct path obbpath; + void *pkgl_id; + struct list_head list; +}; + +/* + * inode to private data + * + * Since we use containers and the struct inode is _inside_ the + * sdcardfs_inode_info structure, SDCARDFS_I will always (given a non-NULL + * inode pointer), return a valid non-NULL pointer. + */ +static inline struct sdcardfs_inode_info *SDCARDFS_I(const struct inode *inode) +{ + return container_of(inode, struct sdcardfs_inode_info, vfs_inode); +} + +/* dentry to private data */ +#define SDCARDFS_D(dent) ((struct sdcardfs_dentry_info *)(dent)->d_fsdata) + +/* superblock to private data */ +#define SDCARDFS_SB(super) ((struct sdcardfs_sb_info *)(super)->s_fs_info) + +/* file to private Data */ +#define SDCARDFS_F(file) ((struct sdcardfs_file_info *)((file)->private_data)) + +/* file to lower file */ +static inline struct file *sdcardfs_lower_file(const struct file *f) +{ + return SDCARDFS_F(f)->lower_file; +} + +static inline void sdcardfs_set_lower_file(struct file *f, struct file *val) +{ + SDCARDFS_F(f)->lower_file = val; +} + +/* inode to lower inode. */ +static inline struct inode *sdcardfs_lower_inode(const struct inode *i) +{ + return SDCARDFS_I(i)->lower_inode; +} + +static inline void sdcardfs_set_lower_inode(struct inode *i, struct inode *val) +{ + SDCARDFS_I(i)->lower_inode = val; +} + +/* superblock to lower superblock */ +static inline struct super_block *sdcardfs_lower_super( + const struct super_block *sb) +{ + return SDCARDFS_SB(sb)->lower_sb; +} + +static inline void sdcardfs_set_lower_super(struct super_block *sb, + struct super_block *val) +{ + SDCARDFS_SB(sb)->lower_sb = val; +} + +/* path based (dentry/mnt) macros */ +static inline void pathcpy(struct path *dst, const struct path *src) +{ + dst->dentry = src->dentry; + dst->mnt = src->mnt; +} + +/* sdcardfs_get_pname functions calls path_get() + * therefore, the caller must call "proper" path_put functions + */ +#define SDCARDFS_DENT_FUNC(pname) \ +static inline void sdcardfs_get_##pname(const struct dentry *dent, \ + struct path *pname) \ +{ \ + spin_lock(&SDCARDFS_D(dent)->lock); \ + pathcpy(pname, &SDCARDFS_D(dent)->pname); \ + path_get(pname); \ + spin_unlock(&SDCARDFS_D(dent)->lock); \ + return; \ +} \ +static inline void sdcardfs_put_##pname(const struct dentry *dent, \ + struct path *pname) \ +{ \ + path_put(pname); \ + return; \ +} \ +static inline void sdcardfs_set_##pname(const struct dentry *dent, \ + struct path *pname) \ +{ \ + spin_lock(&SDCARDFS_D(dent)->lock); \ + pathcpy(&SDCARDFS_D(dent)->pname, pname); \ + spin_unlock(&SDCARDFS_D(dent)->lock); \ + return; \ +} \ +static inline void sdcardfs_reset_##pname(const struct dentry *dent) \ +{ \ + spin_lock(&SDCARDFS_D(dent)->lock); \ + SDCARDFS_D(dent)->pname.dentry = NULL; \ + SDCARDFS_D(dent)->pname.mnt = NULL; \ + spin_unlock(&SDCARDFS_D(dent)->lock); \ + return; \ +} \ +static inline void sdcardfs_put_reset_##pname(const struct dentry *dent) \ +{ \ + struct path pname; \ + spin_lock(&SDCARDFS_D(dent)->lock); \ + if(SDCARDFS_D(dent)->pname.dentry) { \ + pathcpy(&pname, &SDCARDFS_D(dent)->pname); \ + SDCARDFS_D(dent)->pname.dentry = NULL; \ + SDCARDFS_D(dent)->pname.mnt = NULL; \ + spin_unlock(&SDCARDFS_D(dent)->lock); \ + path_put(&pname); \ + } else \ + spin_unlock(&SDCARDFS_D(dent)->lock); \ + return; \ +} + +SDCARDFS_DENT_FUNC(lower_path) +SDCARDFS_DENT_FUNC(orig_path) + +/* grab a refererence if we aren't linking to ourself */ +static inline void set_top(struct sdcardfs_inode_info *info, struct inode *top) +{ + struct inode *old_top = NULL; + BUG_ON(IS_ERR_OR_NULL(top)); + if (info->top && info->top != &info->vfs_inode) { + old_top = info->top; + } + if (top != &info->vfs_inode) + igrab(top); + info->top = top; + iput(old_top); +} + +static inline struct inode *grab_top(struct sdcardfs_inode_info *info) +{ + struct inode *top = info->top; + if (top) { + return igrab(top); + } else { + return NULL; + } +} + +static inline void release_top(struct sdcardfs_inode_info *info) +{ + iput(info->top); +} + +static inline int get_gid(struct vfsmount *mnt, struct sdcardfs_inode_info *info) { + struct sdcardfs_vfsmount_options *opts = mnt->data; + + if (opts->gid == AID_SDCARD_RW) { + /* As an optimization, certain trusted system components only run + * as owner but operate across all users. Since we're now handing + * out the sdcard_rw GID only to trusted apps, we're okay relaxing + * the user boundary enforcement for the default view. The UIDs + * assigned to app directories are still multiuser aware. */ + return AID_SDCARD_RW; + } else { + return multiuser_get_uid(info->userid, opts->gid); + } +} +static inline int get_mode(struct vfsmount *mnt, struct sdcardfs_inode_info *info) { + int owner_mode; + int filtered_mode; + struct sdcardfs_vfsmount_options *opts = mnt->data; + int visible_mode = 0775 & ~opts->mask; + + + if (info->perm == PERM_PRE_ROOT) { + /* Top of multi-user view should always be visible to ensure + * secondary users can traverse inside. */ + visible_mode = 0711; + } else if (info->under_android) { + /* Block "other" access to Android directories, since only apps + * belonging to a specific user should be in there; we still + * leave +x open for the default view. */ + if (opts->gid == AID_SDCARD_RW) { + visible_mode = visible_mode & ~0006; + } else { + visible_mode = visible_mode & ~0007; + } + } + owner_mode = info->lower_inode->i_mode & 0700; + filtered_mode = visible_mode & (owner_mode | (owner_mode >> 3) | (owner_mode >> 6)); + return filtered_mode; +} + +static inline int has_graft_path(const struct dentry *dent) +{ + int ret = 0; + + spin_lock(&SDCARDFS_D(dent)->lock); + if (SDCARDFS_D(dent)->orig_path.dentry != NULL) + ret = 1; + spin_unlock(&SDCARDFS_D(dent)->lock); + + return ret; +} + +static inline void sdcardfs_get_real_lower(const struct dentry *dent, + struct path *real_lower) +{ + /* in case of a local obb dentry + * the orig_path should be returned + */ + if(has_graft_path(dent)) + sdcardfs_get_orig_path(dent, real_lower); + else + sdcardfs_get_lower_path(dent, real_lower); +} + +static inline void sdcardfs_put_real_lower(const struct dentry *dent, + struct path *real_lower) +{ + if(has_graft_path(dent)) + sdcardfs_put_orig_path(dent, real_lower); + else + sdcardfs_put_lower_path(dent, real_lower); +} + +extern struct mutex sdcardfs_super_list_lock; +extern struct list_head sdcardfs_super_list; + +/* for packagelist.c */ +extern appid_t get_appid(const char *app_name); +extern int check_caller_access_to_name(struct inode *parent_node, const char* name); +extern int open_flags_to_access_mode(int open_flags); +extern int packagelist_init(void); +extern void packagelist_exit(void); + +/* for derived_perm.c */ +extern void setup_derived_state(struct inode *inode, perm_t perm, userid_t userid, + uid_t uid, bool under_android, struct inode *top); +extern void get_derived_permission(struct dentry *parent, struct dentry *dentry); +extern void get_derived_permission_new(struct dentry *parent, struct dentry *dentry, struct dentry *newdentry); +extern void fixup_top_recursive(struct dentry *parent); +extern void fixup_perms_recursive(struct dentry *dentry, const char *name, size_t len); + +extern void update_derived_permission_lock(struct dentry *dentry); +extern int need_graft_path(struct dentry *dentry); +extern int is_base_obbpath(struct dentry *dentry); +extern int is_obbpath_invalid(struct dentry *dentry); +extern int setup_obb_dentry(struct dentry *dentry, struct path *lower_path); + +/* locking helpers */ +static inline struct dentry *lock_parent(struct dentry *dentry) +{ + struct dentry *dir = dget_parent(dentry); + inode_lock_nested(d_inode(dir), I_MUTEX_PARENT); + return dir; +} + +static inline void unlock_dir(struct dentry *dir) +{ + inode_unlock(d_inode(dir)); + dput(dir); +} + +static inline int prepare_dir(const char *path_s, uid_t uid, gid_t gid, mode_t mode) +{ + int err; + struct dentry *dent; + struct iattr attrs; + struct path parent; + + dent = kern_path_locked(path_s, &parent); + if (IS_ERR(dent)) { + err = PTR_ERR(dent); + if (err == -EEXIST) + err = 0; + goto out_unlock; + } + + err = vfs_mkdir2(parent.mnt, d_inode(parent.dentry), dent, mode); + if (err) { + if (err == -EEXIST) + err = 0; + goto out_dput; + } + + attrs.ia_uid = make_kuid(&init_user_ns, uid); + attrs.ia_gid = make_kgid(&init_user_ns, gid); + attrs.ia_valid = ATTR_UID | ATTR_GID; + inode_lock(d_inode(dent)); + notify_change2(parent.mnt, dent, &attrs, NULL); + inode_unlock(d_inode(dent)); + +out_dput: + dput(dent); + +out_unlock: + /* parent dentry locked by lookup_create */ + inode_unlock(d_inode(parent.dentry)); + path_put(&parent); + return err; +} + +/* + * Return 1, if a disk has enough free space, otherwise 0. + * We assume that any files can not be overwritten. + */ +static inline int check_min_free_space(struct dentry *dentry, size_t size, int dir) +{ + int err; + struct path lower_path; + struct kstatfs statfs; + u64 avail; + struct sdcardfs_sb_info *sbi = SDCARDFS_SB(dentry->d_sb); + + if (sbi->options.reserved_mb) { + /* Get fs stat of lower filesystem. */ + sdcardfs_get_lower_path(dentry, &lower_path); + err = vfs_statfs(&lower_path, &statfs); + sdcardfs_put_lower_path(dentry, &lower_path); + + if (unlikely(err)) + return 0; + + /* Invalid statfs informations. */ + if (unlikely(statfs.f_bsize == 0)) + return 0; + + /* if you are checking directory, set size to f_bsize. */ + if (unlikely(dir)) + size = statfs.f_bsize; + + /* available size */ + avail = statfs.f_bavail * statfs.f_bsize; + + /* not enough space */ + if ((u64)size > avail) + return 0; + + /* enough space */ + if ((avail - size) > (sbi->options.reserved_mb * 1024 * 1024)) + return 1; + + return 0; + } else + return 1; +} + +/* + * Copies attrs and maintains sdcardfs managed attrs + * Since our permission check handles all special permissions, set those to be open + */ +static inline void sdcardfs_copy_and_fix_attrs(struct inode *dest, const struct inode *src) +{ + dest->i_mode = (src->i_mode & S_IFMT) | S_IRWXU | S_IRWXG | + S_IROTH | S_IXOTH; /* 0775 */ + dest->i_uid = make_kuid(&init_user_ns, SDCARDFS_I(dest)->d_uid); + dest->i_gid = make_kgid(&init_user_ns, AID_SDCARD_RW); + dest->i_rdev = src->i_rdev; + dest->i_atime = src->i_atime; + dest->i_mtime = src->i_mtime; + dest->i_ctime = src->i_ctime; + dest->i_blkbits = src->i_blkbits; + dest->i_flags = src->i_flags; + set_nlink(dest, src->i_nlink); +} +#endif /* not _SDCARDFS_H_ */ diff --git a/fs/sdcardfs/super.c b/fs/sdcardfs/super.c new file mode 100644 index 00000000000000..edda32b68dc08d --- /dev/null +++ b/fs/sdcardfs/super.c @@ -0,0 +1,275 @@ +/* + * fs/sdcardfs/super.c + * + * Copyright (c) 2013 Samsung Electronics Co. Ltd + * Authors: Daeho Jeong, Woojoong Lee, Seunghwan Hyun, + * Sunghwan Yun, Sungjong Seo + * + * This program has been developed as a stackable file system based on + * the WrapFS which written by + * + * Copyright (c) 1998-2011 Erez Zadok + * Copyright (c) 2009 Shrikar Archak + * Copyright (c) 2003-2011 Stony Brook University + * Copyright (c) 2003-2011 The Research Foundation of SUNY + * + * This file is dual licensed. It may be redistributed and/or modified + * under the terms of the Apache 2.0 License OR version 2 of the GNU + * General Public License. + */ + +#include "sdcardfs.h" + +/* + * The inode cache is used with alloc_inode for both our inode info and the + * vfs inode. + */ +static struct kmem_cache *sdcardfs_inode_cachep; + +/* final actions when unmounting a file system */ +static void sdcardfs_put_super(struct super_block *sb) +{ + struct sdcardfs_sb_info *spd; + struct super_block *s; + + spd = SDCARDFS_SB(sb); + if (!spd) + return; + + if(spd->obbpath_s) { + kfree(spd->obbpath_s); + path_put(&spd->obbpath); + } + + /* decrement lower super references */ + s = sdcardfs_lower_super(sb); + sdcardfs_set_lower_super(sb, NULL); + atomic_dec(&s->s_active); + + kfree(spd); + sb->s_fs_info = NULL; +} + +static int sdcardfs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + int err; + struct path lower_path; + u32 min_blocks; + struct sdcardfs_sb_info *sbi = SDCARDFS_SB(dentry->d_sb); + + sdcardfs_get_lower_path(dentry, &lower_path); + err = vfs_statfs(&lower_path, buf); + sdcardfs_put_lower_path(dentry, &lower_path); + + if (sbi->options.reserved_mb) { + /* Invalid statfs informations. */ + if (buf->f_bsize == 0) { + printk(KERN_ERR "Returned block size is zero.\n"); + return -EINVAL; + } + + min_blocks = ((sbi->options.reserved_mb * 1024 * 1024)/buf->f_bsize); + buf->f_blocks -= min_blocks; + + if (buf->f_bavail > min_blocks) + buf->f_bavail -= min_blocks; + else + buf->f_bavail = 0; + + /* Make reserved blocks invisiable to media storage */ + buf->f_bfree = buf->f_bavail; + } + + /* set return buf to our f/s to avoid confusing user-level utils */ + buf->f_type = SDCARDFS_SUPER_MAGIC; + + return err; +} + +/* + * @flags: numeric mount options + * @options: mount options string + */ +static int sdcardfs_remount_fs(struct super_block *sb, int *flags, char *options) +{ + int err = 0; + + /* + * The VFS will take care of "ro" and "rw" flags among others. We + * can safely accept a few flags (RDONLY, MANDLOCK), and honor + * SILENT, but anything else left over is an error. + */ + if ((*flags & ~(MS_RDONLY | MS_MANDLOCK | MS_SILENT)) != 0) { + printk(KERN_ERR + "sdcardfs: remount flags 0x%x unsupported\n", *flags); + err = -EINVAL; + } + + return err; +} + +/* + * @mnt: mount point we are remounting + * @sb: superblock we are remounting + * @flags: numeric mount options + * @options: mount options string + */ +static int sdcardfs_remount_fs2(struct vfsmount *mnt, struct super_block *sb, + int *flags, char *options) +{ + int err = 0; + + /* + * The VFS will take care of "ro" and "rw" flags among others. We + * can safely accept a few flags (RDONLY, MANDLOCK), and honor + * SILENT, but anything else left over is an error. + */ + if ((*flags & ~(MS_RDONLY | MS_MANDLOCK | MS_SILENT | MS_REMOUNT)) != 0) { + printk(KERN_ERR + "sdcardfs: remount flags 0x%x unsupported\n", *flags); + err = -EINVAL; + } + printk(KERN_INFO "Remount options were %s for vfsmnt %p.\n", options, mnt); + err = parse_options_remount(sb, options, *flags & ~MS_SILENT, mnt->data); + + + return err; +} + +static void* sdcardfs_clone_mnt_data(void *data) { + struct sdcardfs_vfsmount_options* opt = kmalloc(sizeof(struct sdcardfs_vfsmount_options), GFP_KERNEL); + struct sdcardfs_vfsmount_options* old = data; + if(!opt) return NULL; + opt->gid = old->gid; + opt->mask = old->mask; + return opt; +} + +static void sdcardfs_copy_mnt_data(void *data, void *newdata) { + struct sdcardfs_vfsmount_options* old = data; + struct sdcardfs_vfsmount_options* new = newdata; + old->gid = new->gid; + old->mask = new->mask; +} + +/* + * Called by iput() when the inode reference count reached zero + * and the inode is not hashed anywhere. Used to clear anything + * that needs to be, before the inode is completely destroyed and put + * on the inode free list. + */ +static void sdcardfs_evict_inode(struct inode *inode) +{ + struct inode *lower_inode; + + truncate_inode_pages(&inode->i_data, 0); + clear_inode(inode); + /* + * Decrement a reference to a lower_inode, which was incremented + * by our read_inode when it was created initially. + */ + lower_inode = sdcardfs_lower_inode(inode); + sdcardfs_set_lower_inode(inode, NULL); + set_top(SDCARDFS_I(inode), inode); + iput(lower_inode); +} + +static struct inode *sdcardfs_alloc_inode(struct super_block *sb) +{ + struct sdcardfs_inode_info *i; + + i = kmem_cache_alloc(sdcardfs_inode_cachep, GFP_KERNEL); + if (!i) + return NULL; + + /* memset everything up to the inode to 0 */ + memset(i, 0, offsetof(struct sdcardfs_inode_info, vfs_inode)); + + i->vfs_inode.i_version = 1; + return &i->vfs_inode; +} + +static void sdcardfs_destroy_inode(struct inode *inode) +{ + kmem_cache_free(sdcardfs_inode_cachep, SDCARDFS_I(inode)); +} + +/* sdcardfs inode cache constructor */ +static void init_once(void *obj) +{ + struct sdcardfs_inode_info *i = obj; + + inode_init_once(&i->vfs_inode); +} + +int sdcardfs_init_inode_cache(void) +{ + int err = 0; + + sdcardfs_inode_cachep = + kmem_cache_create("sdcardfs_inode_cache", + sizeof(struct sdcardfs_inode_info), 0, + SLAB_RECLAIM_ACCOUNT, init_once); + if (!sdcardfs_inode_cachep) + err = -ENOMEM; + return err; +} + +/* sdcardfs inode cache destructor */ +void sdcardfs_destroy_inode_cache(void) +{ + if (sdcardfs_inode_cachep) + kmem_cache_destroy(sdcardfs_inode_cachep); +} + +/* + * Used only in nfs, to kill any pending RPC tasks, so that subsequent + * code can actually succeed and won't leave tasks that need handling. + */ +static void sdcardfs_umount_begin(struct super_block *sb) +{ + struct super_block *lower_sb; + + lower_sb = sdcardfs_lower_super(sb); + if (lower_sb && lower_sb->s_op && lower_sb->s_op->umount_begin) + lower_sb->s_op->umount_begin(lower_sb); +} + +static int sdcardfs_show_options(struct vfsmount *mnt, struct seq_file *m, struct dentry *root) +{ + struct sdcardfs_sb_info *sbi = SDCARDFS_SB(root->d_sb); + struct sdcardfs_mount_options *opts = &sbi->options; + struct sdcardfs_vfsmount_options *vfsopts = mnt->data; + + if (opts->fs_low_uid != 0) + seq_printf(m, ",fsuid=%u", opts->fs_low_uid); + if (opts->fs_low_gid != 0) + seq_printf(m, ",fsgid=%u", opts->fs_low_gid); + if (vfsopts->gid != 0) + seq_printf(m, ",gid=%u", vfsopts->gid); + if (opts->multiuser) + seq_printf(m, ",multiuser"); + if (vfsopts->mask) + seq_printf(m, ",mask=%u", vfsopts->mask); + if (opts->fs_user_id) + seq_printf(m, ",userid=%u", opts->fs_user_id); + if (opts->reserved_mb != 0) + seq_printf(m, ",reserved=%uMB", opts->reserved_mb); + + return 0; +}; + +const struct super_operations sdcardfs_sops = { + .put_super = sdcardfs_put_super, + .statfs = sdcardfs_statfs, + .remount_fs = sdcardfs_remount_fs, + .remount_fs2 = sdcardfs_remount_fs2, + .clone_mnt_data = sdcardfs_clone_mnt_data, + .copy_mnt_data = sdcardfs_copy_mnt_data, + .evict_inode = sdcardfs_evict_inode, + .umount_begin = sdcardfs_umount_begin, + .show_options2 = sdcardfs_show_options, + .alloc_inode = sdcardfs_alloc_inode, + .destroy_inode = sdcardfs_destroy_inode, + .drop_inode = generic_delete_inode, +}; diff --git a/fs/splice.c b/fs/splice.c index 5a7750bd2eea76..63b8f54485dc01 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -1086,7 +1086,13 @@ EXPORT_SYMBOL(do_splice_direct); static int wait_for_space(struct pipe_inode_info *pipe, unsigned flags) { - while (pipe->nrbufs == pipe->buffers) { + for (;;) { + if (unlikely(!pipe->readers)) { + send_sig(SIGPIPE, current, 0); + return -EPIPE; + } + if (pipe->nrbufs != pipe->buffers) + return 0; if (flags & SPLICE_F_NONBLOCK) return -EAGAIN; if (signal_pending(current)) @@ -1095,7 +1101,6 @@ static int wait_for_space(struct pipe_inode_info *pipe, unsigned flags) pipe_wait(pipe); pipe->waiting_writers--; } - return 0; } static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe, diff --git a/fs/super.c b/fs/super.c index c183835566c19c..719579f5e98e25 100644 --- a/fs/super.c +++ b/fs/super.c @@ -750,7 +750,8 @@ struct super_block *user_get_super(dev_t dev) } /** - * do_remount_sb - asks filesystem to change mount options. + * do_remount_sb2 - asks filesystem to change mount options. + * @mnt: mount we are looking at * @sb: superblock in question * @flags: numeric part of options * @data: the rest of options @@ -758,7 +759,7 @@ struct super_block *user_get_super(dev_t dev) * * Alters the mount options of a mounted file system. */ -int do_remount_sb(struct super_block *sb, int flags, void *data, int force) +int do_remount_sb2(struct vfsmount *mnt, struct super_block *sb, int flags, void *data, int force) { int retval; int remount_ro; @@ -800,7 +801,16 @@ int do_remount_sb(struct super_block *sb, int flags, void *data, int force) } } - if (sb->s_op->remount_fs) { + if (mnt && sb->s_op->remount_fs2) { + retval = sb->s_op->remount_fs2(mnt, sb, &flags, data); + if (retval) { + if (!force) + goto cancel_readonly; + /* If forced remount, go ahead despite any errors */ + WARN(1, "forced remount of a %s fs returned %i\n", + sb->s_type->name, retval); + } + } else if (sb->s_op->remount_fs) { retval = sb->s_op->remount_fs(sb, &flags, data); if (retval) { if (!force) @@ -832,12 +842,17 @@ int do_remount_sb(struct super_block *sb, int flags, void *data, int force) return retval; } +int do_remount_sb(struct super_block *sb, int flags, void *data, int force) +{ + return do_remount_sb2(NULL, sb, flags, data, force); +} + static void do_emergency_remount(struct work_struct *work) { struct super_block *sb, *p = NULL; spin_lock(&sb_lock); - list_for_each_entry(sb, &super_blocks, s_list) { + list_for_each_entry_reverse(sb, &super_blocks, s_list) { if (hlist_unhashed(&sb->s_instances)) continue; sb->s_count++; @@ -1157,7 +1172,7 @@ struct dentry *mount_single(struct file_system_type *fs_type, EXPORT_SYMBOL(mount_single); struct dentry * -mount_fs(struct file_system_type *type, int flags, const char *name, void *data) +mount_fs(struct file_system_type *type, int flags, const char *name, struct vfsmount *mnt, void *data) { struct dentry *root; struct super_block *sb; @@ -1174,7 +1189,10 @@ mount_fs(struct file_system_type *type, int flags, const char *name, void *data) goto out_free_secdata; } - root = type->mount(type, flags, name, data); + if (type->mount2) + root = type->mount2(mnt, type, flags, name, data); + else + root = type->mount(type, flags, name, data); if (IS_ERR(root)) { error = PTR_ERR(root); goto out_free_secdata; diff --git a/fs/ubifs/tnc.c b/fs/ubifs/tnc.c index fa9a20cc60d674..fe5e8d4970ae1c 100644 --- a/fs/ubifs/tnc.c +++ b/fs/ubifs/tnc.c @@ -34,6 +34,11 @@ #include #include "ubifs.h" +static int try_read_node(const struct ubifs_info *c, void *buf, int type, + int len, int lnum, int offs); +static int fallible_read_node(struct ubifs_info *c, const union ubifs_key *key, + struct ubifs_zbranch *zbr, void *node); + /* * Returned codes of 'matches_name()' and 'fallible_matches_name()' functions. * @NAME_LESS: name corresponding to the first argument is less than second @@ -402,7 +407,19 @@ static int tnc_read_node_nm(struct ubifs_info *c, struct ubifs_zbranch *zbr, return 0; } - err = ubifs_tnc_read_node(c, zbr, node); + if (c->replaying) { + err = fallible_read_node(c, &zbr->key, zbr, node); + /* + * When the node was not found, return -ENOENT, 0 otherwise. + * Negative return codes stay as-is. + */ + if (err == 0) + err = -ENOENT; + else if (err == 1) + err = 0; + } else { + err = ubifs_tnc_read_node(c, zbr, node); + } if (err) return err; @@ -2766,7 +2783,11 @@ struct ubifs_dent_node *ubifs_tnc_next_ent(struct ubifs_info *c, if (nm->name) { if (err) { /* Handle collisions */ - err = resolve_collision(c, key, &znode, &n, nm); + if (c->replaying) + err = fallible_resolve_collision(c, key, &znode, &n, + nm, 0); + else + err = resolve_collision(c, key, &znode, &n, nm); dbg_tnc("rc returned %d, znode %p, n %d", err, znode, n); if (unlikely(err < 0)) diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index 85959d8324df26..69c867c01d524e 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -460,7 +460,8 @@ static int userfaultfd_release(struct inode *inode, struct file *file) new_flags, vma->anon_vma, vma->vm_file, vma->vm_pgoff, vma_policy(vma), - NULL_VM_UFFD_CTX); + NULL_VM_UFFD_CTX, + vma_get_anon_name(vma)); if (prev) vma = prev; else @@ -839,7 +840,8 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, prev = vma_merge(mm, prev, start, vma_end, new_flags, vma->anon_vma, vma->vm_file, vma->vm_pgoff, vma_policy(vma), - ((struct vm_userfaultfd_ctx){ ctx })); + ((struct vm_userfaultfd_ctx){ ctx }), + vma_get_anon_name(vma)); if (prev) { vma = prev; goto next; @@ -976,7 +978,8 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, prev = vma_merge(mm, prev, start, vma_end, new_flags, vma->anon_vma, vma->vm_file, vma->vm_pgoff, vma_policy(vma), - NULL_VM_UFFD_CTX); + NULL_VM_UFFD_CTX, + vma_get_anon_name(vma)); if (prev) { vma = prev; goto next; diff --git a/fs/utimes.c b/fs/utimes.c index 22307cdf7014ba..87ce37bcaa84e8 100644 --- a/fs/utimes.c +++ b/fs/utimes.c @@ -91,7 +91,7 @@ static int utimes_common(struct path *path, struct timespec *times) } retry_deleg: inode_lock(inode); - error = notify_change(path->dentry, &newattrs, &delegated_inode); + error = notify_change2(path->mnt, path->dentry, &newattrs, &delegated_inode); inode_unlock(inode); if (delegated_inode) { error = break_deleg_wait(&delegated_inode); diff --git a/fs/xfs/libxfs/xfs_ag_resv.c b/fs/xfs/libxfs/xfs_ag_resv.c index e5ebc37704608a..d346d42c54d159 100644 --- a/fs/xfs/libxfs/xfs_ag_resv.c +++ b/fs/xfs/libxfs/xfs_ag_resv.c @@ -256,6 +256,9 @@ xfs_ag_resv_init( goto out; } + ASSERT(xfs_perag_resv(pag, XFS_AG_RESV_METADATA)->ar_reserved + + xfs_perag_resv(pag, XFS_AG_RESV_AGFL)->ar_reserved <= + pag->pagf_freeblks + pag->pagf_flcount); out: return error; } diff --git a/fs/xfs/libxfs/xfs_alloc.c b/fs/xfs/libxfs/xfs_alloc.c index effb64cf714fee..9f06a211e15705 100644 --- a/fs/xfs/libxfs/xfs_alloc.c +++ b/fs/xfs/libxfs/xfs_alloc.c @@ -95,10 +95,7 @@ unsigned int xfs_alloc_set_aside( struct xfs_mount *mp) { - unsigned int blocks; - - blocks = 4 + (mp->m_sb.sb_agcount * XFS_ALLOC_AGFL_RESERVE); - return blocks; + return mp->m_sb.sb_agcount * (XFS_ALLOC_AGFL_RESERVE + 4); } /* @@ -365,35 +362,11 @@ xfs_alloc_fix_len( return; ASSERT(rlen >= args->minlen && rlen <= args->maxlen); ASSERT(rlen % args->prod == args->mod); + ASSERT(args->pag->pagf_freeblks + args->pag->pagf_flcount >= + rlen + args->minleft); args->len = rlen; } -/* - * Fix up length if there is too little space left in the a.g. - * Return 1 if ok, 0 if too little, should give up. - */ -STATIC int -xfs_alloc_fix_minleft( - xfs_alloc_arg_t *args) /* allocation argument structure */ -{ - xfs_agf_t *agf; /* a.g. freelist header */ - int diff; /* free space difference */ - - if (args->minleft == 0) - return 1; - agf = XFS_BUF_TO_AGF(args->agbp); - diff = be32_to_cpu(agf->agf_freeblks) - - args->len - args->minleft; - if (diff >= 0) - return 1; - args->len += diff; /* shrink the allocated space */ - /* casts to (int) catch length underflows */ - if ((int)args->len >= (int)args->minlen) - return 1; - args->agbno = NULLAGBLOCK; - return 0; -} - /* * Update the two btrees, logically removing from freespace the extent * starting at rbno, rlen blocks. The extent is contained within the @@ -689,8 +662,6 @@ xfs_alloc_ag_vextent( xfs_alloc_arg_t *args) /* argument structure for allocation */ { int error=0; - xfs_extlen_t reservation; - xfs_extlen_t oldmax; ASSERT(args->minlen > 0); ASSERT(args->maxlen > 0); @@ -698,20 +669,6 @@ xfs_alloc_ag_vextent( ASSERT(args->mod < args->prod); ASSERT(args->alignment > 0); - /* - * Clamp maxlen to the amount of free space minus any reservations - * that have been made. - */ - oldmax = args->maxlen; - reservation = xfs_ag_resv_needed(args->pag, args->resv); - if (args->maxlen > args->pag->pagf_freeblks - reservation) - args->maxlen = args->pag->pagf_freeblks - reservation; - if (args->maxlen == 0) { - args->agbno = NULLAGBLOCK; - args->maxlen = oldmax; - return 0; - } - /* * Branch to correct routine based on the type. */ @@ -731,8 +688,6 @@ xfs_alloc_ag_vextent( /* NOTREACHED */ } - args->maxlen = oldmax; - if (error || args->agbno == NULLAGBLOCK) return error; @@ -841,9 +796,6 @@ xfs_alloc_ag_vextent_exact( args->len = XFS_AGBLOCK_MIN(tend, args->agbno + args->maxlen) - args->agbno; xfs_alloc_fix_len(args); - if (!xfs_alloc_fix_minleft(args)) - goto not_found; - ASSERT(args->agbno + args->len <= tend); /* @@ -1149,12 +1101,7 @@ xfs_alloc_ag_vextent_near( XFS_WANT_CORRUPTED_GOTO(args->mp, i == 1, error0); ASSERT(ltbno + ltlen <= be32_to_cpu(XFS_BUF_TO_AGF(args->agbp)->agf_length)); args->len = blen; - if (!xfs_alloc_fix_minleft(args)) { - xfs_btree_del_cursor(cnt_cur, XFS_BTREE_NOERROR); - trace_xfs_alloc_near_nominleft(args); - return 0; - } - blen = args->len; + /* * We are allocating starting at bnew for blen blocks. */ @@ -1346,12 +1293,6 @@ xfs_alloc_ag_vextent_near( */ args->len = XFS_EXTLEN_MIN(ltlena, args->maxlen); xfs_alloc_fix_len(args); - if (!xfs_alloc_fix_minleft(args)) { - trace_xfs_alloc_near_nominleft(args); - xfs_btree_del_cursor(bno_cur_lt, XFS_BTREE_NOERROR); - xfs_btree_del_cursor(cnt_cur, XFS_BTREE_NOERROR); - return 0; - } rlen = args->len; (void)xfs_alloc_compute_diff(args->agbno, rlen, args->alignment, args->datatype, ltbnoa, ltlena, <new); @@ -1553,8 +1494,6 @@ xfs_alloc_ag_vextent_size( } xfs_alloc_fix_len(args); - if (!xfs_alloc_fix_minleft(args)) - goto out_nominleft; rlen = args->len; XFS_WANT_CORRUPTED_GOTO(args->mp, rlen <= flen, error0); /* @@ -2056,7 +1995,7 @@ xfs_alloc_space_available( int flags) { struct xfs_perag *pag = args->pag; - xfs_extlen_t longest; + xfs_extlen_t alloc_len, longest; xfs_extlen_t reservation; /* blocks that are still reserved */ int available; @@ -2066,17 +2005,28 @@ xfs_alloc_space_available( reservation = xfs_ag_resv_needed(pag, args->resv); /* do we have enough contiguous free space for the allocation? */ + alloc_len = args->minlen + (args->alignment - 1) + args->minalignslop; longest = xfs_alloc_longest_free_extent(args->mp, pag, min_free, reservation); - if ((args->minlen + args->alignment + args->minalignslop - 1) > longest) + if (longest < alloc_len) return false; /* do we have enough free space remaining for the allocation? */ available = (int)(pag->pagf_freeblks + pag->pagf_flcount - - reservation - min_free - args->total); - if (available < (int)args->minleft || available <= 0) + reservation - min_free - args->minleft); + if (available < (int)max(args->total, alloc_len)) return false; + /* + * Clamp maxlen to the amount of free space available for the actual + * extent allocation. + */ + if (available < (int)args->maxlen && !(flags & XFS_ALLOC_FLAG_CHECK)) { + args->maxlen = available; + ASSERT(args->maxlen > 0); + ASSERT(args->maxlen >= args->minlen); + } + return true; } @@ -2122,7 +2072,8 @@ xfs_alloc_fix_freelist( } need = xfs_alloc_min_freelist(mp, pag); - if (!xfs_alloc_space_available(args, need, flags)) + if (!xfs_alloc_space_available(args, need, flags | + XFS_ALLOC_FLAG_CHECK)) goto out_agbp_relse; /* @@ -2455,12 +2406,15 @@ xfs_agf_verify( be32_to_cpu(agf->agf_flcount) <= XFS_AGFL_SIZE(mp))) return false; - if (be32_to_cpu(agf->agf_levels[XFS_BTNUM_BNO]) > XFS_BTREE_MAXLEVELS || + if (be32_to_cpu(agf->agf_levels[XFS_BTNUM_BNO]) < 1 || + be32_to_cpu(agf->agf_levels[XFS_BTNUM_CNT]) < 1 || + be32_to_cpu(agf->agf_levels[XFS_BTNUM_BNO]) > XFS_BTREE_MAXLEVELS || be32_to_cpu(agf->agf_levels[XFS_BTNUM_CNT]) > XFS_BTREE_MAXLEVELS) return false; if (xfs_sb_version_hasrmapbt(&mp->m_sb) && - be32_to_cpu(agf->agf_levels[XFS_BTNUM_RMAP]) > XFS_BTREE_MAXLEVELS) + (be32_to_cpu(agf->agf_levels[XFS_BTNUM_RMAP]) < 1 || + be32_to_cpu(agf->agf_levels[XFS_BTNUM_RMAP]) > XFS_BTREE_MAXLEVELS)) return false; /* @@ -2477,7 +2431,8 @@ xfs_agf_verify( return false; if (xfs_sb_version_hasreflink(&mp->m_sb) && - be32_to_cpu(agf->agf_refcount_level) > XFS_BTREE_MAXLEVELS) + (be32_to_cpu(agf->agf_refcount_level) < 1 || + be32_to_cpu(agf->agf_refcount_level) > XFS_BTREE_MAXLEVELS)) return false; return true;; @@ -2634,12 +2589,10 @@ xfs_alloc_vextent( xfs_agblock_t agsize; /* allocation group size */ int error; int flags; /* XFS_ALLOC_FLAG_... locking flags */ - xfs_extlen_t minleft;/* minimum left value, temp copy */ xfs_mount_t *mp; /* mount structure pointer */ xfs_agnumber_t sagno; /* starting allocation group number */ xfs_alloctype_t type; /* input allocation type */ int bump_rotor = 0; - int no_min = 0; xfs_agnumber_t rotorstep = xfs_rotorstep; /* inode32 agf stepper */ mp = args->mp; @@ -2668,7 +2621,6 @@ xfs_alloc_vextent( trace_xfs_alloc_vextent_badargs(args); return 0; } - minleft = args->minleft; switch (type) { case XFS_ALLOCTYPE_THIS_AG: @@ -2679,9 +2631,7 @@ xfs_alloc_vextent( */ args->agno = XFS_FSB_TO_AGNO(mp, args->fsbno); args->pag = xfs_perag_get(mp, args->agno); - args->minleft = 0; error = xfs_alloc_fix_freelist(args, 0); - args->minleft = minleft; if (error) { trace_xfs_alloc_vextent_nofix(args); goto error0; @@ -2746,9 +2696,7 @@ xfs_alloc_vextent( */ for (;;) { args->pag = xfs_perag_get(mp, args->agno); - if (no_min) args->minleft = 0; error = xfs_alloc_fix_freelist(args, flags); - args->minleft = minleft; if (error) { trace_xfs_alloc_vextent_nofix(args); goto error0; @@ -2788,20 +2736,17 @@ xfs_alloc_vextent( * or switch to non-trylock mode. */ if (args->agno == sagno) { - if (no_min == 1) { + if (flags == 0) { args->agbno = NULLAGBLOCK; trace_xfs_alloc_vextent_allfailed(args); break; } - if (flags == 0) { - no_min = 1; - } else { - flags = 0; - if (type == XFS_ALLOCTYPE_START_BNO) { - args->agbno = XFS_FSB_TO_AGBNO(mp, - args->fsbno); - args->type = XFS_ALLOCTYPE_NEAR_BNO; - } + + flags = 0; + if (type == XFS_ALLOCTYPE_START_BNO) { + args->agbno = XFS_FSB_TO_AGBNO(mp, + args->fsbno); + args->type = XFS_ALLOCTYPE_NEAR_BNO; } } xfs_perag_put(args->pag); diff --git a/fs/xfs/libxfs/xfs_alloc.h b/fs/xfs/libxfs/xfs_alloc.h index 7c404a6b0ae322..1d0f48a501a3d6 100644 --- a/fs/xfs/libxfs/xfs_alloc.h +++ b/fs/xfs/libxfs/xfs_alloc.h @@ -56,7 +56,7 @@ typedef unsigned int xfs_alloctype_t; #define XFS_ALLOC_FLAG_FREEING 0x00000002 /* indicate caller is freeing extents*/ #define XFS_ALLOC_FLAG_NORMAP 0x00000004 /* don't modify the rmapbt */ #define XFS_ALLOC_FLAG_NOSHRINK 0x00000008 /* don't shrink the freelist */ - +#define XFS_ALLOC_FLAG_CHECK 0x00000010 /* test only, don't modify args */ /* * Argument structure for xfs_alloc routines. diff --git a/fs/xfs/libxfs/xfs_alloc_btree.c b/fs/xfs/libxfs/xfs_alloc_btree.c index 5ba2dac5e67c49..c06ec77a94182d 100644 --- a/fs/xfs/libxfs/xfs_alloc_btree.c +++ b/fs/xfs/libxfs/xfs_alloc_btree.c @@ -421,7 +421,7 @@ xfs_allocbt_init_cursor( ASSERT(btnum == XFS_BTNUM_BNO || btnum == XFS_BTNUM_CNT); - cur = kmem_zone_zalloc(xfs_btree_cur_zone, KM_SLEEP); + cur = kmem_zone_zalloc(xfs_btree_cur_zone, KM_NOFS); cur->bc_tp = tp; cur->bc_mp = mp; diff --git a/fs/xfs/libxfs/xfs_attr.c b/fs/xfs/libxfs/xfs_attr.c index af1ecb19121e9e..6622d46ddec389 100644 --- a/fs/xfs/libxfs/xfs_attr.c +++ b/fs/xfs/libxfs/xfs_attr.c @@ -131,9 +131,6 @@ xfs_attr_get( if (XFS_FORCED_SHUTDOWN(ip->i_mount)) return -EIO; - if (!xfs_inode_hasattr(ip)) - return -ENOATTR; - error = xfs_attr_args_init(&args, ip, name, flags); if (error) return error; @@ -392,9 +389,6 @@ xfs_attr_remove( if (XFS_FORCED_SHUTDOWN(dp->i_mount)) return -EIO; - if (!xfs_inode_hasattr(dp)) - return -ENOATTR; - error = xfs_attr_args_init(&args, dp, name, flags); if (error) return error; diff --git a/fs/xfs/libxfs/xfs_attr_leaf.c b/fs/xfs/libxfs/xfs_attr_leaf.c index 8ea91f3630938a..2852521fc8ecf7 100644 --- a/fs/xfs/libxfs/xfs_attr_leaf.c +++ b/fs/xfs/libxfs/xfs_attr_leaf.c @@ -253,6 +253,7 @@ xfs_attr3_leaf_verify( { struct xfs_mount *mp = bp->b_target->bt_mount; struct xfs_attr_leafblock *leaf = bp->b_addr; + struct xfs_perag *pag = bp->b_pag; struct xfs_attr3_icleaf_hdr ichdr; xfs_attr3_leaf_hdr_from_disk(mp->m_attr_geo, &ichdr, leaf); @@ -273,7 +274,12 @@ xfs_attr3_leaf_verify( if (ichdr.magic != XFS_ATTR_LEAF_MAGIC) return false; } - if (ichdr.count == 0) + /* + * In recovery there is a transient state where count == 0 is valid + * because we may have transitioned an empty shortform attr to a leaf + * if the attr didn't fit in shortform. + */ + if (pag && pag->pagf_init && ichdr.count == 0) return false; /* XXX: need to range check rest of attr header values */ diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index c6eb21940783e4..f52fd63fce1988 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -49,6 +49,8 @@ #include "xfs_rmap.h" #include "xfs_ag_resv.h" #include "xfs_refcount.h" +#include "xfs_rmap_btree.h" +#include "xfs_icache.h" kmem_zone_t *xfs_bmap_free_item_zone; @@ -190,8 +192,12 @@ xfs_bmap_worst_indlen( int maxrecs; /* maximum record count at this level */ xfs_mount_t *mp; /* mount structure */ xfs_filblks_t rval; /* return value */ + xfs_filblks_t orig_len; mp = ip->i_mount; + + /* Calculate the worst-case size of the bmbt. */ + orig_len = len; maxrecs = mp->m_bmap_dmxr[0]; for (level = 0, rval = 0; level < XFS_BM_MAXLEVELS(mp, XFS_DATA_FORK); @@ -199,12 +205,20 @@ xfs_bmap_worst_indlen( len += maxrecs - 1; do_div(len, maxrecs); rval += len; - if (len == 1) - return rval + XFS_BM_MAXLEVELS(mp, XFS_DATA_FORK) - + if (len == 1) { + rval += XFS_BM_MAXLEVELS(mp, XFS_DATA_FORK) - level - 1; + break; + } if (level == 0) maxrecs = mp->m_bmap_dmxr[1]; } + + /* Calculate the worst-case size of the rmapbt. */ + if (xfs_sb_version_hasrmapbt(&mp->m_sb)) + rval += 1 + xfs_rmapbt_calc_size(mp, orig_len) + + mp->m_rmap_maxlevels; + return rval; } @@ -504,7 +518,7 @@ void xfs_bmap_trace_exlist( xfs_inode_t *ip, /* incore inode pointer */ xfs_extnum_t cnt, /* count of entries in the list */ - int whichfork, /* data or attr fork */ + int whichfork, /* data or attr or cow fork */ unsigned long caller_ip) { xfs_extnum_t idx; /* extent record index */ @@ -513,11 +527,13 @@ xfs_bmap_trace_exlist( if (whichfork == XFS_ATTR_FORK) state |= BMAP_ATTRFORK; + else if (whichfork == XFS_COW_FORK) + state |= BMAP_COWFORK; ifp = XFS_IFORK_PTR(ip, whichfork); - ASSERT(cnt == (ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t))); + ASSERT(cnt == xfs_iext_count(ifp)); for (idx = 0; idx < cnt; idx++) - trace_xfs_extlist(ip, idx, whichfork, caller_ip); + trace_xfs_extlist(ip, idx, state, caller_ip); } /* @@ -811,7 +827,7 @@ xfs_bmap_extents_to_btree( XFS_BTREE_LONG_PTRS); arp = XFS_BMBT_REC_ADDR(mp, ablock, 1); - nextents = ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t); + nextents = xfs_iext_count(ifp); for (cnt = i = 0; i < nextents; i++) { ep = xfs_iext_get_ext(ifp, i); if (!isnullstartblock(xfs_bmbt_get_startblock(ep))) { @@ -1137,6 +1153,10 @@ xfs_bmap_add_attrfork( goto trans_cancel; if (XFS_IFORK_Q(ip)) goto trans_cancel; + if (ip->i_d.di_anextents != 0) { + error = -EFSCORRUPTED; + goto trans_cancel; + } if (ip->i_d.di_aformat != XFS_DINODE_FMT_EXTENTS) { /* * For inodes coming from pre-6.2 filesystems. @@ -1144,7 +1164,6 @@ xfs_bmap_add_attrfork( ASSERT(ip->i_d.di_aformat == 0); ip->i_d.di_aformat = XFS_DINODE_FMT_EXTENTS; } - ASSERT(ip->i_d.di_anextents == 0); xfs_trans_ijoin(tp, ip, 0); xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); @@ -1296,7 +1315,7 @@ xfs_bmap_read_extents( /* * Here with bp and block set to the leftmost leaf node in the tree. */ - room = ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t); + room = xfs_iext_count(ifp); i = 0; /* * Loop over all leaf nodes. Copy information to the extent records. @@ -1361,8 +1380,9 @@ xfs_bmap_read_extents( return error; block = XFS_BUF_TO_BLOCK(bp); } - ASSERT(i == (ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t))); - ASSERT(i == XFS_IFORK_NEXTENTS(ip, whichfork)); + if (i != XFS_IFORK_NEXTENTS(ip, whichfork)) + return -EFSCORRUPTED; + ASSERT(i == xfs_iext_count(ifp)); XFS_BMAP_TRACE_EXLIST(ip, i, whichfork); return 0; error0: @@ -1404,7 +1424,7 @@ xfs_bmap_search_multi_extents( if (lastx > 0) { xfs_bmbt_get_all(xfs_iext_get_ext(ifp, lastx - 1), prevp); } - if (lastx < (ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t))) { + if (lastx < xfs_iext_count(ifp)) { xfs_bmbt_get_all(ep, gotp); *eofp = 0; } else { @@ -1497,7 +1517,7 @@ xfs_bmap_first_unused( (error = xfs_iread_extents(tp, ip, whichfork))) return error; lowest = *first_unused; - nextents = ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t); + nextents = xfs_iext_count(ifp); for (idx = 0, lastaddr = 0, max = lowest; idx < nextents; idx++) { xfs_bmbt_rec_host_t *ep = xfs_iext_get_ext(ifp, idx); off = xfs_bmbt_get_startoff(ep); @@ -1582,7 +1602,7 @@ xfs_bmap_last_extent( return error; } - nextents = ifp->if_bytes / sizeof(xfs_bmbt_rec_t); + nextents = xfs_iext_count(ifp); if (nextents == 0) { *is_empty = 1; return 0; @@ -1735,7 +1755,7 @@ xfs_bmap_add_extent_delay_real( &bma->ip->i_d.di_nextents); ASSERT(bma->idx >= 0); - ASSERT(bma->idx <= ifp->if_bytes / sizeof(struct xfs_bmbt_rec)); + ASSERT(bma->idx <= xfs_iext_count(ifp)); ASSERT(!isnullstartblock(new->br_startblock)); ASSERT(!bma->cur || (bma->cur->bc_private.b.flags & XFS_BTCUR_BPRV_WASDEL)); @@ -1794,7 +1814,7 @@ xfs_bmap_add_extent_delay_real( * Don't set contiguous if the combined extent would be too large. * Also check for all-three-contiguous being too large. */ - if (bma->idx < ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t) - 1) { + if (bma->idx < xfs_iext_count(ifp) - 1) { state |= BMAP_RIGHT_VALID; xfs_bmbt_get_all(xfs_iext_get_ext(ifp, bma->idx + 1), &RIGHT); @@ -2300,7 +2320,7 @@ xfs_bmap_add_extent_unwritten_real( ifp = XFS_IFORK_PTR(ip, XFS_DATA_FORK); ASSERT(*idx >= 0); - ASSERT(*idx <= ifp->if_bytes / sizeof(struct xfs_bmbt_rec)); + ASSERT(*idx <= xfs_iext_count(ifp)); ASSERT(!isnullstartblock(new->br_startblock)); XFS_STATS_INC(mp, xs_add_exlist); @@ -2356,7 +2376,7 @@ xfs_bmap_add_extent_unwritten_real( * Don't set contiguous if the combined extent would be too large. * Also check for all-three-contiguous being too large. */ - if (*idx < ip->i_df.if_bytes / (uint)sizeof(xfs_bmbt_rec_t) - 1) { + if (*idx < xfs_iext_count(&ip->i_df) - 1) { state |= BMAP_RIGHT_VALID; xfs_bmbt_get_all(xfs_iext_get_ext(ifp, *idx + 1), &RIGHT); if (isnullstartblock(RIGHT.br_startblock)) @@ -2836,7 +2856,7 @@ xfs_bmap_add_extent_hole_delay( * Check and set flags if the current (right) segment exists. * If it doesn't exist, we're converting the hole at end-of-file. */ - if (*idx < ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t)) { + if (*idx < xfs_iext_count(ifp)) { state |= BMAP_RIGHT_VALID; xfs_bmbt_get_all(xfs_iext_get_ext(ifp, *idx), &right); @@ -2966,7 +2986,7 @@ xfs_bmap_add_extent_hole_real( ifp = XFS_IFORK_PTR(bma->ip, whichfork); ASSERT(bma->idx >= 0); - ASSERT(bma->idx <= ifp->if_bytes / sizeof(struct xfs_bmbt_rec)); + ASSERT(bma->idx <= xfs_iext_count(ifp)); ASSERT(!isnullstartblock(new->br_startblock)); ASSERT(!bma->cur || !(bma->cur->bc_private.b.flags & XFS_BTCUR_BPRV_WASDEL)); @@ -2992,7 +3012,7 @@ xfs_bmap_add_extent_hole_real( * Check and set flags if this segment has a current value. * Not true if we're inserting into the "hole" at eof. */ - if (bma->idx < ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t)) { + if (bma->idx < xfs_iext_count(ifp)) { state |= BMAP_RIGHT_VALID; xfs_bmbt_get_all(xfs_iext_get_ext(ifp, bma->idx), &right); if (isnullstartblock(right.br_startblock)) @@ -3700,7 +3720,7 @@ xfs_bmap_btalloc( align = xfs_get_cowextsz_hint(ap->ip); else if (xfs_alloc_is_userdata(ap->datatype)) align = xfs_get_extsz_hint(ap->ip); - if (unlikely(align)) { + if (align) { error = xfs_bmap_extsize_align(mp, &ap->got, &ap->prev, align, 0, ap->eof, 0, ap->conv, &ap->offset, &ap->length); @@ -3772,7 +3792,7 @@ xfs_bmap_btalloc( args.minlen = ap->minlen; } /* apply extent size hints if obtained earlier */ - if (unlikely(align)) { + if (align) { args.prod = align; if ((args.mod = (xfs_extlen_t)do_mod(ap->offset, args.prod))) args.mod = (xfs_extlen_t)(args.prod - args.mod); @@ -3883,7 +3903,6 @@ xfs_bmap_btalloc( args.fsbno = 0; args.type = XFS_ALLOCTYPE_FIRST_AG; args.total = ap->minlen; - args.minleft = 0; if ((error = xfs_alloc_vextent(&args))) return error; ap->dfops->dop_low = true; @@ -4221,7 +4240,7 @@ xfs_bmapi_read( break; /* Else go on to the next record. */ - if (++lastx < ifp->if_bytes / sizeof(xfs_bmbt_rec_t)) + if (++lastx < xfs_iext_count(ifp)) xfs_bmbt_get_all(xfs_iext_get_ext(ifp, lastx), &got); else eof = 1; @@ -4234,10 +4253,10 @@ int xfs_bmapi_reserve_delalloc( struct xfs_inode *ip, int whichfork, - xfs_fileoff_t aoff, + xfs_fileoff_t off, xfs_filblks_t len, + xfs_filblks_t prealloc, struct xfs_bmbt_irec *got, - struct xfs_bmbt_irec *prev, xfs_extnum_t *lastx, int eof) { @@ -4248,10 +4267,17 @@ xfs_bmapi_reserve_delalloc( char rt = XFS_IS_REALTIME_INODE(ip); xfs_extlen_t extsz; int error; + xfs_fileoff_t aoff = off; - alen = XFS_FILBLKS_MIN(len, MAXEXTLEN); + /* + * Cap the alloc length. Keep track of prealloc so we know whether to + * tag the inode before we return. + */ + alen = XFS_FILBLKS_MIN(len + prealloc, MAXEXTLEN); if (!eof) alen = XFS_FILBLKS_MIN(alen, got->br_startoff - aoff); + if (prealloc && alen >= len) + prealloc = alen - len; /* Figure out the extent size, adjust alen */ if (whichfork == XFS_COW_FORK) @@ -4259,7 +4285,12 @@ xfs_bmapi_reserve_delalloc( else extsz = xfs_get_extsz_hint(ip); if (extsz) { - error = xfs_bmap_extsize_align(mp, got, prev, extsz, rt, eof, + struct xfs_bmbt_irec prev; + + if (!xfs_iext_get_extent(ifp, *lastx - 1, &prev)) + prev.br_startoff = NULLFILEOFF; + + error = xfs_bmap_extsize_align(mp, got, &prev, extsz, rt, eof, 1, 0, &aoff, &alen); ASSERT(!error); } @@ -4312,6 +4343,16 @@ xfs_bmapi_reserve_delalloc( */ xfs_bmbt_get_all(xfs_iext_get_ext(ifp, *lastx), got); + /* + * Tag the inode if blocks were preallocated. Note that COW fork + * preallocation can occur at the start or end of the extent, even when + * prealloc == 0, so we must also check the aligned offset and length. + */ + if (whichfork == XFS_DATA_FORK && prealloc) + xfs_inode_set_eofblocks_tag(ip); + if (whichfork == XFS_COW_FORK && (prealloc || aoff < off || alen > len)) + xfs_inode_set_cowblocks_tag(ip); + ASSERT(got->br_startoff <= aoff); ASSERT(got->br_startoff + got->br_blockcount >= aoff + alen); ASSERT(isnullstartblock(got->br_startblock)); @@ -4395,8 +4436,6 @@ xfs_bmapi_allocate( if (error) return error; - if (bma->dfops->dop_low) - bma->minleft = 0; if (bma->cur) bma->cur->bc_private.b.firstblock = *bma->firstblock; if (bma->blkno == NULLFSBLOCK) @@ -4568,8 +4607,6 @@ xfs_bmapi_write( int n; /* current extent index */ xfs_fileoff_t obno; /* old block number (offset) */ int whichfork; /* data or attr fork */ - char inhole; /* current location is hole in file */ - char wasdelay; /* old extent was delayed */ #ifdef DEBUG xfs_fileoff_t orig_bno; /* original block number value */ @@ -4655,22 +4692,44 @@ xfs_bmapi_write( bma.firstblock = firstblock; while (bno < end && n < *nmap) { - inhole = eof || bma.got.br_startoff > bno; - wasdelay = !inhole && isnullstartblock(bma.got.br_startblock); + bool need_alloc = false, wasdelay = false; - /* - * Make sure we only reflink into a hole. - */ - if (flags & XFS_BMAPI_REMAP) - ASSERT(inhole); - if (flags & XFS_BMAPI_COWFORK) - ASSERT(!inhole); + /* in hole or beyoned EOF? */ + if (eof || bma.got.br_startoff > bno) { + if (flags & XFS_BMAPI_DELALLOC) { + /* + * For the COW fork we can reasonably get a + * request for converting an extent that races + * with other threads already having converted + * part of it, as there converting COW to + * regular blocks is not protected using the + * IOLOCK. + */ + ASSERT(flags & XFS_BMAPI_COWFORK); + if (!(flags & XFS_BMAPI_COWFORK)) { + error = -EIO; + goto error0; + } + + if (eof || bno >= end) + break; + } else { + need_alloc = true; + } + } else { + /* + * Make sure we only reflink into a hole. + */ + ASSERT(!(flags & XFS_BMAPI_REMAP)); + if (isnullstartblock(bma.got.br_startblock)) + wasdelay = true; + } /* * First, deal with the hole before the allocated space * that we found, if any. */ - if (inhole || wasdelay) { + if (need_alloc || wasdelay) { bma.eof = eof; bma.conv = !!(flags & XFS_BMAPI_CONVERT); bma.wasdel = wasdelay; @@ -4733,7 +4792,7 @@ xfs_bmapi_write( /* Else go on to the next record. */ bma.prev = bma.got; - if (++bma.idx < ifp->if_bytes / sizeof(xfs_bmbt_rec_t)) { + if (++bma.idx < xfs_iext_count(ifp)) { xfs_bmbt_get_all(xfs_iext_get_ext(ifp, bma.idx), &bma.got); } else @@ -4885,7 +4944,7 @@ xfs_bmap_del_extent_delay( da_new = 0; ASSERT(*idx >= 0); - ASSERT(*idx < ifp->if_bytes / sizeof(struct xfs_bmbt_rec)); + ASSERT(*idx <= xfs_iext_count(ifp)); ASSERT(del->br_blockcount > 0); ASSERT(got->br_startoff <= del->br_startoff); ASSERT(got_endoff >= del_endoff); @@ -4902,8 +4961,11 @@ xfs_bmap_del_extent_delay( * sb counters as we might have to borrow some blocks for the * indirect block accounting. */ - xfs_trans_reserve_quota_nblks(NULL, ip, -((long)del->br_blockcount), 0, + error = xfs_trans_reserve_quota_nblks(NULL, ip, + -((long)del->br_blockcount), 0, isrt ? XFS_QMOPT_RES_RTBLKS : XFS_QMOPT_RES_REGBLKS); + if (error) + return error; ip->i_delayed_blks -= del->br_blockcount; if (whichfork == XFS_COW_FORK) @@ -5013,7 +5075,7 @@ xfs_bmap_del_extent_cow( got_endoff = got->br_startoff + got->br_blockcount; ASSERT(*idx >= 0); - ASSERT(*idx < ifp->if_bytes / sizeof(struct xfs_bmbt_rec)); + ASSERT(*idx <= xfs_iext_count(ifp)); ASSERT(del->br_blockcount > 0); ASSERT(got->br_startoff <= del->br_startoff); ASSERT(got_endoff >= del_endoff); @@ -5119,8 +5181,7 @@ xfs_bmap_del_extent( state |= BMAP_COWFORK; ifp = XFS_IFORK_PTR(ip, whichfork); - ASSERT((*idx >= 0) && (*idx < ifp->if_bytes / - (uint)sizeof(xfs_bmbt_rec_t))); + ASSERT((*idx >= 0) && (*idx < xfs_iext_count(ifp))); ASSERT(del->br_blockcount > 0); ep = xfs_iext_get_ext(ifp, *idx); xfs_bmbt_get_all(ep, &got); @@ -5445,7 +5506,6 @@ __xfs_bunmapi( int logflags; /* transaction logging flags */ xfs_extlen_t mod; /* rt extent offset */ xfs_mount_t *mp; /* mount structure */ - xfs_extnum_t nextents; /* number of file extents */ xfs_bmbt_irec_t prev; /* previous extent record */ xfs_fileoff_t start; /* first file offset deleted */ int tmp_logflags; /* partial logging flags */ @@ -5477,8 +5537,7 @@ __xfs_bunmapi( if (!(ifp->if_flags & XFS_IFEXTENTS) && (error = xfs_iread_extents(tp, ip, whichfork))) return error; - nextents = ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t); - if (nextents == 0) { + if (xfs_iext_count(ifp) == 0) { *rlen = 0; return 0; } @@ -5963,7 +6022,7 @@ xfs_bmse_shift_one( mp = ip->i_mount; ifp = XFS_IFORK_PTR(ip, whichfork); - total_extents = ifp->if_bytes / sizeof(xfs_bmbt_rec_t); + total_extents = xfs_iext_count(ifp); xfs_bmbt_get_all(gotp, &got); @@ -6140,7 +6199,7 @@ xfs_bmap_shift_extents( * are collapsing out, so we cannot use the count of real extents here. * Instead we have to calculate it from the incore fork. */ - total_extents = ifp->if_bytes / sizeof(xfs_bmbt_rec_t); + total_extents = xfs_iext_count(ifp); if (total_extents == 0) { *done = 1; goto del_cursor; @@ -6200,7 +6259,7 @@ xfs_bmap_shift_extents( * count can change. Update the total and grade the next record. */ if (direction == SHIFT_LEFT) { - total_extents = ifp->if_bytes / sizeof(xfs_bmbt_rec_t); + total_extents = xfs_iext_count(ifp); stop_extent = total_extents; } diff --git a/fs/xfs/libxfs/xfs_bmap.h b/fs/xfs/libxfs/xfs_bmap.h index 7cae6ec27fa6b2..e7d40b39f18f5a 100644 --- a/fs/xfs/libxfs/xfs_bmap.h +++ b/fs/xfs/libxfs/xfs_bmap.h @@ -110,6 +110,9 @@ struct xfs_extent_free_item /* Map something in the CoW fork. */ #define XFS_BMAPI_COWFORK 0x200 +/* Only convert delalloc space, don't allocate entirely new extents */ +#define XFS_BMAPI_DELALLOC 0x400 + #define XFS_BMAPI_FLAGS \ { XFS_BMAPI_ENTIRE, "ENTIRE" }, \ { XFS_BMAPI_METADATA, "METADATA" }, \ @@ -120,7 +123,8 @@ struct xfs_extent_free_item { XFS_BMAPI_CONVERT, "CONVERT" }, \ { XFS_BMAPI_ZERO, "ZERO" }, \ { XFS_BMAPI_REMAP, "REMAP" }, \ - { XFS_BMAPI_COWFORK, "COWFORK" } + { XFS_BMAPI_COWFORK, "COWFORK" }, \ + { XFS_BMAPI_DELALLOC, "DELALLOC" } static inline int xfs_bmapi_aflag(int w) @@ -242,9 +246,8 @@ struct xfs_bmbt_rec_host * int fork, int *eofp, xfs_extnum_t *lastxp, struct xfs_bmbt_irec *gotp, struct xfs_bmbt_irec *prevp); int xfs_bmapi_reserve_delalloc(struct xfs_inode *ip, int whichfork, - xfs_fileoff_t aoff, xfs_filblks_t len, - struct xfs_bmbt_irec *got, struct xfs_bmbt_irec *prev, - xfs_extnum_t *lastx, int eof); + xfs_fileoff_t off, xfs_filblks_t len, xfs_filblks_t prealloc, + struct xfs_bmbt_irec *got, xfs_extnum_t *lastx, int eof); enum xfs_bmap_intent_type { XFS_BMAP_MAP = 1, diff --git a/fs/xfs/libxfs/xfs_bmap_btree.c b/fs/xfs/libxfs/xfs_bmap_btree.c index 8007d2ba9aef9c..f76c1693ff0106 100644 --- a/fs/xfs/libxfs/xfs_bmap_btree.c +++ b/fs/xfs/libxfs/xfs_bmap_btree.c @@ -502,12 +502,11 @@ xfs_bmbt_alloc_block( if (args.fsbno == NULLFSBLOCK && args.minleft) { /* * Could not find an AG with enough free space to satisfy - * a full btree split. Try again without minleft and if + * a full btree split. Try again and if * successful activate the lowspace algorithm. */ args.fsbno = 0; args.type = XFS_ALLOCTYPE_FIRST_AG; - args.minleft = 0; error = xfs_alloc_vextent(&args); if (error) goto error0; @@ -796,7 +795,7 @@ xfs_bmbt_init_cursor( struct xfs_btree_cur *cur; ASSERT(whichfork != XFS_COW_FORK); - cur = kmem_zone_zalloc(xfs_btree_cur_zone, KM_SLEEP); + cur = kmem_zone_zalloc(xfs_btree_cur_zone, KM_NOFS); cur->bc_tp = tp; cur->bc_mp = mp; diff --git a/fs/xfs/libxfs/xfs_btree.c b/fs/xfs/libxfs/xfs_btree.c index 0e80993c8a5914..21e6a6ab6b9a40 100644 --- a/fs/xfs/libxfs/xfs_btree.c +++ b/fs/xfs/libxfs/xfs_btree.c @@ -1769,8 +1769,28 @@ xfs_btree_lookup_get_block( if (error) return error; + /* Check the inode owner since the verifiers don't. */ + if (xfs_sb_version_hascrc(&cur->bc_mp->m_sb) && + (cur->bc_flags & XFS_BTREE_LONG_PTRS) && + be64_to_cpu((*blkp)->bb_u.l.bb_owner) != + cur->bc_private.b.ip->i_ino) + goto out_bad; + + /* Did we get the level we were looking for? */ + if (be16_to_cpu((*blkp)->bb_level) != level) + goto out_bad; + + /* Check that internal nodes have at least one record. */ + if (level != 0 && be16_to_cpu((*blkp)->bb_numrecs) == 0) + goto out_bad; + xfs_btree_setbuf(cur, level, bp); return 0; + +out_bad: + *blkp = NULL; + xfs_trans_brelse(cur->bc_tp, bp); + return -EFSCORRUPTED; } /* diff --git a/fs/xfs/libxfs/xfs_dir2.c b/fs/xfs/libxfs/xfs_dir2.c index 20a96dd5af7eb6..7825d78d4587aa 100644 --- a/fs/xfs/libxfs/xfs_dir2.c +++ b/fs/xfs/libxfs/xfs_dir2.c @@ -36,21 +36,29 @@ struct xfs_name xfs_name_dotdot = { (unsigned char *)"..", 2, XFS_DIR3_FT_DIR }; /* - * @mode, if set, indicates that the type field needs to be set up. - * This uses the transformation from file mode to DT_* as defined in linux/fs.h - * for file type specification. This will be propagated into the directory - * structure if appropriate for the given operation and filesystem config. + * Convert inode mode to directory entry filetype */ -const unsigned char xfs_mode_to_ftype[S_IFMT >> S_SHIFT] = { - [0] = XFS_DIR3_FT_UNKNOWN, - [S_IFREG >> S_SHIFT] = XFS_DIR3_FT_REG_FILE, - [S_IFDIR >> S_SHIFT] = XFS_DIR3_FT_DIR, - [S_IFCHR >> S_SHIFT] = XFS_DIR3_FT_CHRDEV, - [S_IFBLK >> S_SHIFT] = XFS_DIR3_FT_BLKDEV, - [S_IFIFO >> S_SHIFT] = XFS_DIR3_FT_FIFO, - [S_IFSOCK >> S_SHIFT] = XFS_DIR3_FT_SOCK, - [S_IFLNK >> S_SHIFT] = XFS_DIR3_FT_SYMLINK, -}; +unsigned char xfs_mode_to_ftype(int mode) +{ + switch (mode & S_IFMT) { + case S_IFREG: + return XFS_DIR3_FT_REG_FILE; + case S_IFDIR: + return XFS_DIR3_FT_DIR; + case S_IFCHR: + return XFS_DIR3_FT_CHRDEV; + case S_IFBLK: + return XFS_DIR3_FT_BLKDEV; + case S_IFIFO: + return XFS_DIR3_FT_FIFO; + case S_IFSOCK: + return XFS_DIR3_FT_SOCK; + case S_IFLNK: + return XFS_DIR3_FT_SYMLINK; + default: + return XFS_DIR3_FT_UNKNOWN; + } +} /* * ASCII case-insensitive (ie. A-Z) support for directories that was @@ -631,7 +639,8 @@ xfs_dir2_isblock( if ((rval = xfs_bmap_last_offset(args->dp, &last, XFS_DATA_FORK))) return rval; rval = XFS_FSB_TO_B(args->dp->i_mount, last) == args->geo->blksize; - ASSERT(rval == 0 || args->dp->i_d.di_size == args->geo->blksize); + if (rval != 0 && args->dp->i_d.di_size != args->geo->blksize) + return -EFSCORRUPTED; *vp = rval; return 0; } diff --git a/fs/xfs/libxfs/xfs_dir2.h b/fs/xfs/libxfs/xfs_dir2.h index becc926c3e3d90..ae0d55bf6500f5 100644 --- a/fs/xfs/libxfs/xfs_dir2.h +++ b/fs/xfs/libxfs/xfs_dir2.h @@ -18,6 +18,9 @@ #ifndef __XFS_DIR2_H__ #define __XFS_DIR2_H__ +#include "xfs_da_format.h" +#include "xfs_da_btree.h" + struct xfs_defer_ops; struct xfs_da_args; struct xfs_inode; @@ -32,10 +35,9 @@ struct xfs_dir2_data_unused; extern struct xfs_name xfs_name_dotdot; /* - * directory filetype conversion tables. + * Convert inode mode to directory entry filetype */ -#define S_SHIFT 12 -extern const unsigned char xfs_mode_to_ftype[]; +extern unsigned char xfs_mode_to_ftype(int mode); /* * directory operations vector for encode/decode routines diff --git a/fs/xfs/libxfs/xfs_dir2_data.c b/fs/xfs/libxfs/xfs_dir2_data.c index 725fc7841fdeb3..e526f5a5f0beae 100644 --- a/fs/xfs/libxfs/xfs_dir2_data.c +++ b/fs/xfs/libxfs/xfs_dir2_data.c @@ -329,7 +329,7 @@ xfs_dir3_data_read( err = xfs_da_read_buf(tp, dp, bno, mapped_bno, bpp, XFS_DATA_FORK, &xfs_dir3_data_buf_ops); - if (!err && tp) + if (!err && tp && *bpp) xfs_trans_buf_set_type(tp, *bpp, XFS_BLFT_DIR_DATA_BUF); return err; } diff --git a/fs/xfs/libxfs/xfs_ialloc.c b/fs/xfs/libxfs/xfs_ialloc.c index 51b4e0de1fdc42..d45c03779dae39 100644 --- a/fs/xfs/libxfs/xfs_ialloc.c +++ b/fs/xfs/libxfs/xfs_ialloc.c @@ -2450,8 +2450,6 @@ xfs_ialloc_log_agi( ASSERT(agi->agi_magicnum == cpu_to_be32(XFS_AGI_MAGIC)); #endif - xfs_trans_buf_set_type(tp, bp, XFS_BLFT_AGI_BUF); - /* * Compute byte offsets for the first and last fields in the first * region and log the agi buffer. This only logs up through @@ -2512,8 +2510,15 @@ xfs_agi_verify( if (!XFS_AGI_GOOD_VERSION(be32_to_cpu(agi->agi_versionnum))) return false; - if (be32_to_cpu(agi->agi_level) > XFS_BTREE_MAXLEVELS) + if (be32_to_cpu(agi->agi_level) < 1 || + be32_to_cpu(agi->agi_level) > XFS_BTREE_MAXLEVELS) + return false; + + if (xfs_sb_version_hasfinobt(&mp->m_sb) && + (be32_to_cpu(agi->agi_free_level) < 1 || + be32_to_cpu(agi->agi_free_level) > XFS_BTREE_MAXLEVELS)) return false; + /* * during growfs operations, the perag is not fully initialised, * so we can't use it for any useful checking. growfs ensures we can't @@ -2592,6 +2597,8 @@ xfs_read_agi( XFS_FSS_TO_BB(mp, 1), 0, bpp, &xfs_agi_buf_ops); if (error) return error; + if (tp) + xfs_trans_buf_set_type(tp, *bpp, XFS_BLFT_AGI_BUF); xfs_buf_set_ref(*bpp, XFS_AGI_REF); return 0; diff --git a/fs/xfs/libxfs/xfs_ialloc_btree.c b/fs/xfs/libxfs/xfs_ialloc_btree.c index eab68ae2e01184..6c6b95947e716d 100644 --- a/fs/xfs/libxfs/xfs_ialloc_btree.c +++ b/fs/xfs/libxfs/xfs_ialloc_btree.c @@ -357,7 +357,7 @@ xfs_inobt_init_cursor( struct xfs_agi *agi = XFS_BUF_TO_AGI(agbp); struct xfs_btree_cur *cur; - cur = kmem_zone_zalloc(xfs_btree_cur_zone, KM_SLEEP); + cur = kmem_zone_zalloc(xfs_btree_cur_zone, KM_NOFS); cur->bc_tp = tp; cur->bc_mp = mp; diff --git a/fs/xfs/libxfs/xfs_inode_buf.c b/fs/xfs/libxfs/xfs_inode_buf.c index 134424fac434fd..37ee7f01a35dbc 100644 --- a/fs/xfs/libxfs/xfs_inode_buf.c +++ b/fs/xfs/libxfs/xfs_inode_buf.c @@ -29,6 +29,7 @@ #include "xfs_icache.h" #include "xfs_trans.h" #include "xfs_ialloc.h" +#include "xfs_dir2.h" /* * Check that none of the inode's in the buffer have a next @@ -386,12 +387,25 @@ xfs_dinode_verify( struct xfs_inode *ip, struct xfs_dinode *dip) { + uint16_t mode; uint16_t flags; uint64_t flags2; if (dip->di_magic != cpu_to_be16(XFS_DINODE_MAGIC)) return false; + /* don't allow invalid i_size */ + if (be64_to_cpu(dip->di_size) & (1ULL << 63)) + return false; + + mode = be16_to_cpu(dip->di_mode); + if (mode && xfs_mode_to_ftype(mode) == XFS_DIR3_FT_UNKNOWN) + return false; + + /* No zero-length symlinks/dirs. */ + if ((S_ISLNK(mode) || S_ISDIR(mode)) && dip->di_size == 0) + return false; + /* only version 3 or greater inodes are extensively verified here */ if (dip->di_version < 3) return true; diff --git a/fs/xfs/libxfs/xfs_inode_fork.c b/fs/xfs/libxfs/xfs_inode_fork.c index 5dd56d3dbb3aeb..222e103356c6d8 100644 --- a/fs/xfs/libxfs/xfs_inode_fork.c +++ b/fs/xfs/libxfs/xfs_inode_fork.c @@ -775,6 +775,13 @@ xfs_idestroy_fork( } } +/* Count number of incore extents based on if_bytes */ +xfs_extnum_t +xfs_iext_count(struct xfs_ifork *ifp) +{ + return ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t); +} + /* * Convert in-core extents to on-disk form * @@ -803,7 +810,7 @@ xfs_iextents_copy( ASSERT(xfs_isilocked(ip, XFS_ILOCK_EXCL|XFS_ILOCK_SHARED)); ASSERT(ifp->if_bytes > 0); - nrecs = ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t); + nrecs = xfs_iext_count(ifp); XFS_BMAP_TRACE_EXLIST(ip, nrecs, whichfork); ASSERT(nrecs > 0); @@ -941,7 +948,7 @@ xfs_iext_get_ext( xfs_extnum_t idx) /* index of target extent */ { ASSERT(idx >= 0); - ASSERT(idx < ifp->if_bytes / sizeof(xfs_bmbt_rec_t)); + ASSERT(idx < xfs_iext_count(ifp)); if ((ifp->if_flags & XFS_IFEXTIREC) && (idx == 0)) { return ifp->if_u1.if_ext_irec->er_extbuf; @@ -1017,7 +1024,7 @@ xfs_iext_add( int new_size; /* size of extents after adding */ xfs_extnum_t nextents; /* number of extents in file */ - nextents = ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t); + nextents = xfs_iext_count(ifp); ASSERT((idx >= 0) && (idx <= nextents)); byte_diff = ext_diff * sizeof(xfs_bmbt_rec_t); new_size = ifp->if_bytes + byte_diff; @@ -1241,7 +1248,7 @@ xfs_iext_remove( trace_xfs_iext_remove(ip, idx, state, _RET_IP_); ASSERT(ext_diff > 0); - nextents = ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t); + nextents = xfs_iext_count(ifp); new_size = (nextents - ext_diff) * sizeof(xfs_bmbt_rec_t); if (new_size == 0) { @@ -1270,7 +1277,7 @@ xfs_iext_remove_inline( ASSERT(!(ifp->if_flags & XFS_IFEXTIREC)); ASSERT(idx < XFS_INLINE_EXTS); - nextents = ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t); + nextents = xfs_iext_count(ifp); ASSERT(((nextents - ext_diff) > 0) && (nextents - ext_diff) < XFS_INLINE_EXTS); @@ -1309,7 +1316,7 @@ xfs_iext_remove_direct( ASSERT(!(ifp->if_flags & XFS_IFEXTIREC)); new_size = ifp->if_bytes - (ext_diff * sizeof(xfs_bmbt_rec_t)); - nextents = ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t); + nextents = xfs_iext_count(ifp); if (new_size == 0) { xfs_iext_destroy(ifp); @@ -1546,7 +1553,7 @@ xfs_iext_indirect_to_direct( int size; /* size of file extents */ ASSERT(ifp->if_flags & XFS_IFEXTIREC); - nextents = ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t); + nextents = xfs_iext_count(ifp); ASSERT(nextents <= XFS_LINEAR_EXTS); size = nextents * sizeof(xfs_bmbt_rec_t); @@ -1620,7 +1627,7 @@ xfs_iext_bno_to_ext( xfs_extnum_t nextents; /* number of file extents */ xfs_fileoff_t startoff = 0; /* start offset of extent */ - nextents = ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t); + nextents = xfs_iext_count(ifp); if (nextents == 0) { *idxp = 0; return NULL; @@ -1733,8 +1740,8 @@ xfs_iext_idx_to_irec( ASSERT(ifp->if_flags & XFS_IFEXTIREC); ASSERT(page_idx >= 0); - ASSERT(page_idx <= ifp->if_bytes / sizeof(xfs_bmbt_rec_t)); - ASSERT(page_idx < ifp->if_bytes / sizeof(xfs_bmbt_rec_t) || realloc); + ASSERT(page_idx <= xfs_iext_count(ifp)); + ASSERT(page_idx < xfs_iext_count(ifp) || realloc); nlists = ifp->if_real_bytes / XFS_IEXT_BUFSZ; erp_idx = 0; @@ -1782,7 +1789,7 @@ xfs_iext_irec_init( xfs_extnum_t nextents; /* number of extents in file */ ASSERT(!(ifp->if_flags & XFS_IFEXTIREC)); - nextents = ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t); + nextents = xfs_iext_count(ifp); ASSERT(nextents <= XFS_LINEAR_EXTS); erp = kmem_alloc(sizeof(xfs_ext_irec_t), KM_NOFS); @@ -1906,7 +1913,7 @@ xfs_iext_irec_compact( ASSERT(ifp->if_flags & XFS_IFEXTIREC); nlists = ifp->if_real_bytes / XFS_IEXT_BUFSZ; - nextents = ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t); + nextents = xfs_iext_count(ifp); if (nextents == 0) { xfs_iext_destroy(ifp); @@ -1996,3 +2003,49 @@ xfs_ifork_init_cow( ip->i_cformat = XFS_DINODE_FMT_EXTENTS; ip->i_cnextents = 0; } + +/* + * Lookup the extent covering bno. + * + * If there is an extent covering bno return the extent index, and store the + * expanded extent structure in *gotp, and the extent index in *idx. + * If there is no extent covering bno, but there is an extent after it (e.g. + * it lies in a hole) return that extent in *gotp and its index in *idx + * instead. + * If bno is beyond the last extent return false, and return the index after + * the last valid index in *idxp. + */ +bool +xfs_iext_lookup_extent( + struct xfs_inode *ip, + struct xfs_ifork *ifp, + xfs_fileoff_t bno, + xfs_extnum_t *idxp, + struct xfs_bmbt_irec *gotp) +{ + struct xfs_bmbt_rec_host *ep; + + XFS_STATS_INC(ip->i_mount, xs_look_exlist); + + ep = xfs_iext_bno_to_ext(ifp, bno, idxp); + if (!ep) + return false; + xfs_bmbt_get_all(ep, gotp); + return true; +} + +/* + * Return true if there is an extent at index idx, and return the expanded + * extent structure at idx in that case. Else return false. + */ +bool +xfs_iext_get_extent( + struct xfs_ifork *ifp, + xfs_extnum_t idx, + struct xfs_bmbt_irec *gotp) +{ + if (idx < 0 || idx >= xfs_iext_count(ifp)) + return false; + xfs_bmbt_get_all(xfs_iext_get_ext(ifp, idx), gotp); + return true; +} diff --git a/fs/xfs/libxfs/xfs_inode_fork.h b/fs/xfs/libxfs/xfs_inode_fork.h index c9476f50e32d11..7fb8365326d1a7 100644 --- a/fs/xfs/libxfs/xfs_inode_fork.h +++ b/fs/xfs/libxfs/xfs_inode_fork.h @@ -152,6 +152,7 @@ void xfs_init_local_fork(struct xfs_inode *, int, const void *, int); struct xfs_bmbt_rec_host * xfs_iext_get_ext(struct xfs_ifork *, xfs_extnum_t); +xfs_extnum_t xfs_iext_count(struct xfs_ifork *); void xfs_iext_insert(struct xfs_inode *, xfs_extnum_t, xfs_extnum_t, struct xfs_bmbt_irec *, int); void xfs_iext_add(struct xfs_ifork *, xfs_extnum_t, int); @@ -181,6 +182,12 @@ void xfs_iext_irec_compact_pages(struct xfs_ifork *); void xfs_iext_irec_compact_full(struct xfs_ifork *); void xfs_iext_irec_update_extoffs(struct xfs_ifork *, int, int); +bool xfs_iext_lookup_extent(struct xfs_inode *ip, + struct xfs_ifork *ifp, xfs_fileoff_t bno, + xfs_extnum_t *idxp, struct xfs_bmbt_irec *gotp); +bool xfs_iext_get_extent(struct xfs_ifork *ifp, xfs_extnum_t idx, + struct xfs_bmbt_irec *gotp); + extern struct kmem_zone *xfs_ifork_zone; extern void xfs_ifork_init_cow(struct xfs_inode *ip); diff --git a/fs/xfs/libxfs/xfs_refcount_btree.c b/fs/xfs/libxfs/xfs_refcount_btree.c index 453bb2757ec23f..2ba21696600298 100644 --- a/fs/xfs/libxfs/xfs_refcount_btree.c +++ b/fs/xfs/libxfs/xfs_refcount_btree.c @@ -408,13 +408,14 @@ xfs_refcountbt_calc_size( */ xfs_extlen_t xfs_refcountbt_max_size( - struct xfs_mount *mp) + struct xfs_mount *mp, + xfs_agblock_t agblocks) { /* Bail out if we're uninitialized, which can happen in mkfs. */ if (mp->m_refc_mxr[0] == 0) return 0; - return xfs_refcountbt_calc_size(mp, mp->m_sb.sb_agblocks); + return xfs_refcountbt_calc_size(mp, agblocks); } /* @@ -429,22 +430,24 @@ xfs_refcountbt_calc_reserves( { struct xfs_buf *agbp; struct xfs_agf *agf; + xfs_agblock_t agblocks; xfs_extlen_t tree_len; int error; if (!xfs_sb_version_hasreflink(&mp->m_sb)) return 0; - *ask += xfs_refcountbt_max_size(mp); error = xfs_alloc_read_agf(mp, NULL, agno, 0, &agbp); if (error) return error; agf = XFS_BUF_TO_AGF(agbp); + agblocks = be32_to_cpu(agf->agf_length); tree_len = be32_to_cpu(agf->agf_refcount_blocks); xfs_buf_relse(agbp); + *ask += xfs_refcountbt_max_size(mp, agblocks); *used += tree_len; return error; diff --git a/fs/xfs/libxfs/xfs_refcount_btree.h b/fs/xfs/libxfs/xfs_refcount_btree.h index 3be7768bd51a1c..9db008b955b7ed 100644 --- a/fs/xfs/libxfs/xfs_refcount_btree.h +++ b/fs/xfs/libxfs/xfs_refcount_btree.h @@ -66,7 +66,8 @@ extern void xfs_refcountbt_compute_maxlevels(struct xfs_mount *mp); extern xfs_extlen_t xfs_refcountbt_calc_size(struct xfs_mount *mp, unsigned long long len); -extern xfs_extlen_t xfs_refcountbt_max_size(struct xfs_mount *mp); +extern xfs_extlen_t xfs_refcountbt_max_size(struct xfs_mount *mp, + xfs_agblock_t agblocks); extern int xfs_refcountbt_calc_reserves(struct xfs_mount *mp, xfs_agnumber_t agno, xfs_extlen_t *ask, xfs_extlen_t *used); diff --git a/fs/xfs/libxfs/xfs_rmap_btree.c b/fs/xfs/libxfs/xfs_rmap_btree.c index 83e672ff7577e9..33a28efc3085b0 100644 --- a/fs/xfs/libxfs/xfs_rmap_btree.c +++ b/fs/xfs/libxfs/xfs_rmap_btree.c @@ -549,13 +549,14 @@ xfs_rmapbt_calc_size( */ xfs_extlen_t xfs_rmapbt_max_size( - struct xfs_mount *mp) + struct xfs_mount *mp, + xfs_agblock_t agblocks) { /* Bail out if we're uninitialized, which can happen in mkfs. */ if (mp->m_rmap_mxr[0] == 0) return 0; - return xfs_rmapbt_calc_size(mp, mp->m_sb.sb_agblocks); + return xfs_rmapbt_calc_size(mp, agblocks); } /* @@ -570,25 +571,24 @@ xfs_rmapbt_calc_reserves( { struct xfs_buf *agbp; struct xfs_agf *agf; - xfs_extlen_t pool_len; + xfs_agblock_t agblocks; xfs_extlen_t tree_len; int error; if (!xfs_sb_version_hasrmapbt(&mp->m_sb)) return 0; - /* Reserve 1% of the AG or enough for 1 block per record. */ - pool_len = max(mp->m_sb.sb_agblocks / 100, xfs_rmapbt_max_size(mp)); - *ask += pool_len; - error = xfs_alloc_read_agf(mp, NULL, agno, 0, &agbp); if (error) return error; agf = XFS_BUF_TO_AGF(agbp); + agblocks = be32_to_cpu(agf->agf_length); tree_len = be32_to_cpu(agf->agf_rmap_blocks); xfs_buf_relse(agbp); + /* Reserve 1% of the AG or enough for 1 block per record. */ + *ask += max(agblocks / 100, xfs_rmapbt_max_size(mp, agblocks)); *used += tree_len; return error; diff --git a/fs/xfs/libxfs/xfs_rmap_btree.h b/fs/xfs/libxfs/xfs_rmap_btree.h index 2a9ac472fb15a2..19c08e93304954 100644 --- a/fs/xfs/libxfs/xfs_rmap_btree.h +++ b/fs/xfs/libxfs/xfs_rmap_btree.h @@ -60,7 +60,8 @@ extern void xfs_rmapbt_compute_maxlevels(struct xfs_mount *mp); extern xfs_extlen_t xfs_rmapbt_calc_size(struct xfs_mount *mp, unsigned long long len); -extern xfs_extlen_t xfs_rmapbt_max_size(struct xfs_mount *mp); +extern xfs_extlen_t xfs_rmapbt_max_size(struct xfs_mount *mp, + xfs_agblock_t agblocks); extern int xfs_rmapbt_calc_reserves(struct xfs_mount *mp, xfs_agnumber_t agno, xfs_extlen_t *ask, xfs_extlen_t *used); diff --git a/fs/xfs/libxfs/xfs_sb.c b/fs/xfs/libxfs/xfs_sb.c index a70aec9106263f..584ec896a53374 100644 --- a/fs/xfs/libxfs/xfs_sb.c +++ b/fs/xfs/libxfs/xfs_sb.c @@ -242,7 +242,7 @@ xfs_mount_validate_sb( sbp->sb_blocklog < XFS_MIN_BLOCKSIZE_LOG || sbp->sb_blocklog > XFS_MAX_BLOCKSIZE_LOG || sbp->sb_blocksize != (1 << sbp->sb_blocklog) || - sbp->sb_dirblklog > XFS_MAX_BLOCKSIZE_LOG || + sbp->sb_dirblklog + sbp->sb_blocklog > XFS_MAX_BLOCKSIZE_LOG || sbp->sb_inodesize < XFS_DINODE_MIN_SIZE || sbp->sb_inodesize > XFS_DINODE_MAX_SIZE || sbp->sb_inodelog < XFS_DINODE_MIN_LOG || @@ -262,6 +262,12 @@ xfs_mount_validate_sb( return -EFSCORRUPTED; } + if (xfs_sb_version_hascrc(&mp->m_sb) && + sbp->sb_blocksize < XFS_MIN_CRC_BLOCKSIZE) { + xfs_notice(mp, "v5 SB sanity check failed"); + return -EFSCORRUPTED; + } + /* * Until this is fixed only page-sized or smaller data blocks work. */ @@ -338,13 +344,16 @@ xfs_sb_quota_from_disk(struct xfs_sb *sbp) XFS_PQUOTA_CHKD : XFS_GQUOTA_CHKD; sbp->sb_qflags &= ~(XFS_OQUOTA_ENFD | XFS_OQUOTA_CHKD); - if (sbp->sb_qflags & XFS_PQUOTA_ACCT) { + if (sbp->sb_qflags & XFS_PQUOTA_ACCT && + sbp->sb_gquotino != NULLFSINO) { /* * In older version of superblock, on-disk superblock only * has sb_gquotino, and in-core superblock has both sb_gquotino * and sb_pquotino. But, only one of them is supported at any * point of time. So, if PQUOTA is set in disk superblock, - * copy over sb_gquotino to sb_pquotino. + * copy over sb_gquotino to sb_pquotino. The NULLFSINO test + * above is to make sure we don't do this twice and wipe them + * both out! */ sbp->sb_pquotino = sbp->sb_gquotino; sbp->sb_gquotino = NULLFSINO; diff --git a/fs/xfs/libxfs/xfs_types.h b/fs/xfs/libxfs/xfs_types.h index 8d74870468c24b..cf044c0f4d4178 100644 --- a/fs/xfs/libxfs/xfs_types.h +++ b/fs/xfs/libxfs/xfs_types.h @@ -75,11 +75,14 @@ typedef __int64_t xfs_sfiloff_t; /* signed block number in a file */ * Minimum and maximum blocksize and sectorsize. * The blocksize upper limit is pretty much arbitrary. * The sectorsize upper limit is due to sizeof(sb_sectsize). + * CRC enable filesystems use 512 byte inodes, meaning 512 byte block sizes + * cannot be used. */ #define XFS_MIN_BLOCKSIZE_LOG 9 /* i.e. 512 bytes */ #define XFS_MAX_BLOCKSIZE_LOG 16 /* i.e. 65536 bytes */ #define XFS_MIN_BLOCKSIZE (1 << XFS_MIN_BLOCKSIZE_LOG) #define XFS_MAX_BLOCKSIZE (1 << XFS_MAX_BLOCKSIZE_LOG) +#define XFS_MIN_CRC_BLOCKSIZE (1 << (XFS_MIN_BLOCKSIZE_LOG + 1)) #define XFS_MIN_SECTORSIZE_LOG 9 /* i.e. 512 bytes */ #define XFS_MAX_SECTORSIZE_LOG 15 /* i.e. 32768 bytes */ #define XFS_MIN_SECTORSIZE (1 << XFS_MIN_SECTORSIZE_LOG) diff --git a/fs/xfs/xfs_aops.c b/fs/xfs/xfs_aops.c index 3e57a56cf82947..06763f5cc70186 100644 --- a/fs/xfs/xfs_aops.c +++ b/fs/xfs/xfs_aops.c @@ -1158,19 +1158,22 @@ xfs_vm_releasepage( * block_invalidatepage() can send pages that are still marked dirty * but otherwise have invalidated buffers. * - * We've historically freed buffers on the latter. Instead, quietly - * filter out all dirty pages to avoid spurious buffer state warnings. - * This can likely be removed once shrink_active_list() is fixed. + * We want to release the latter to avoid unnecessary buildup of the + * LRU, skip the former and warn if we've left any lingering + * delalloc/unwritten buffers on clean pages. Skip pages with delalloc + * or unwritten buffers and warn if the page is not dirty. Otherwise + * try to release the buffers. */ - if (PageDirty(page)) - return 0; - xfs_count_page_state(page, &delalloc, &unwritten); - if (WARN_ON_ONCE(delalloc)) + if (delalloc) { + WARN_ON_ONCE(!PageDirty(page)); return 0; - if (WARN_ON_ONCE(unwritten)) + } + if (unwritten) { + WARN_ON_ONCE(!PageDirty(page)); return 0; + } return try_to_free_buffers(page); } @@ -1361,6 +1364,26 @@ __xfs_get_blocks( if (error) goto out_unlock; + /* + * The only time we can ever safely find delalloc blocks on direct I/O + * is a dio write to post-eof speculative preallocation. All other + * scenarios are indicative of a problem or misuse (such as mixing + * direct and mapped I/O). + * + * The file may be unmapped by the time we get here so we cannot + * reliably fail the I/O based on mapping. Instead, fail the I/O if this + * is a read or a write within eof. Otherwise, carry on but warn as a + * precuation if the file happens to be mapped. + */ + if (direct && imap.br_startblock == DELAYSTARTBLOCK) { + if (!create || offset < i_size_read(VFS_I(ip))) { + WARN_ON_ONCE(1); + error = -EIO; + goto out_unlock; + } + WARN_ON_ONCE(mapping_mapped(VFS_I(ip)->i_mapping)); + } + /* for DAX, we convert unwritten extents directly */ if (create && (!nimaps || @@ -1450,8 +1473,6 @@ __xfs_get_blocks( (new || ISUNWRITTEN(&imap)))) set_buffer_new(bh_result); - BUG_ON(direct && imap.br_startblock == DELAYSTARTBLOCK); - return 0; out_unlock: diff --git a/fs/xfs/xfs_bmap_util.c b/fs/xfs/xfs_bmap_util.c index 552465e011ecb0..efb8ccd6bbf2c3 100644 --- a/fs/xfs/xfs_bmap_util.c +++ b/fs/xfs/xfs_bmap_util.c @@ -359,9 +359,7 @@ xfs_bmap_count_blocks( mp = ip->i_mount; ifp = XFS_IFORK_PTR(ip, whichfork); if ( XFS_IFORK_FORMAT(ip, whichfork) == XFS_DINODE_FMT_EXTENTS ) { - xfs_bmap_count_leaves(ifp, 0, - ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t), - count); + xfs_bmap_count_leaves(ifp, 0, xfs_iext_count(ifp), count); return 0; } @@ -426,7 +424,7 @@ xfs_getbmapx_fix_eof_hole( ifp = XFS_IFORK_PTR(ip, whichfork); if (!moretocome && xfs_iext_bno_to_ext(ifp, fileblock, &lastx) && - (lastx == (ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t))-1)) + (lastx == xfs_iext_count(ifp) - 1)) out->bmv_oflags |= BMV_OF_LAST; } @@ -530,7 +528,6 @@ xfs_getbmap( xfs_bmbt_irec_t *map; /* buffer for user's data */ xfs_mount_t *mp; /* file system mount point */ int nex; /* # of user extents can do */ - int nexleft; /* # of user extents left */ int subnex; /* # of bmapi's can do */ int nmap; /* number of map entries */ struct getbmapx *out; /* output structure */ @@ -688,10 +685,8 @@ xfs_getbmap( goto out_free_map; } - nexleft = nex; - do { - nmap = (nexleft > subnex) ? subnex : nexleft; + nmap = (nex> subnex) ? subnex : nex; error = xfs_bmapi_read(ip, XFS_BB_TO_FSBT(mp, bmv->bmv_offset), XFS_BB_TO_FSB(mp, bmv->bmv_length), map, &nmap, bmapi_flags); @@ -699,8 +694,8 @@ xfs_getbmap( goto out_free_map; ASSERT(nmap <= subnex); - for (i = 0; i < nmap && nexleft && bmv->bmv_length && - cur_ext < bmv->bmv_count; i++) { + for (i = 0; i < nmap && bmv->bmv_length && + cur_ext < bmv->bmv_count - 1; i++) { out[cur_ext].bmv_oflags = 0; if (map[i].br_state == XFS_EXT_UNWRITTEN) out[cur_ext].bmv_oflags |= BMV_OF_PREALLOC; @@ -762,16 +757,27 @@ xfs_getbmap( continue; } + /* + * In order to report shared extents accurately, + * we report each distinct shared/unshared part + * of a single bmbt record using multiple bmap + * extents. To make that happen, we iterate the + * same map array item multiple times, each + * time trimming out the subextent that we just + * reported. + * + * Because of this, we must check the out array + * index (cur_ext) directly against bmv_count-1 + * to avoid overflows. + */ if (inject_map.br_startblock != NULLFSBLOCK) { map[i] = inject_map; i--; - } else - nexleft--; + } bmv->bmv_entries++; cur_ext++; } - } while (nmap && nexleft && bmv->bmv_length && - cur_ext < bmv->bmv_count); + } while (nmap && bmv->bmv_length && cur_ext < bmv->bmv_count - 1); out_free_map: kmem_free(map); @@ -1792,6 +1798,7 @@ xfs_swap_extent_forks( struct xfs_ifork tempifp, *ifp, *tifp; int aforkblks = 0; int taforkblks = 0; + xfs_extnum_t nextents; __uint64_t tmp; int error; @@ -1877,14 +1884,13 @@ xfs_swap_extent_forks( switch (ip->i_d.di_format) { case XFS_DINODE_FMT_EXTENTS: - /* If the extents fit in the inode, fix the - * pointer. Otherwise it's already NULL or - * pointing to the extent. + /* + * If the extents fit in the inode, fix the pointer. Otherwise + * it's already NULL or pointing to the extent. */ - if (ip->i_d.di_nextents <= XFS_INLINE_EXTS) { - ifp->if_u1.if_extents = - ifp->if_u2.if_inline_ext; - } + nextents = xfs_iext_count(&ip->i_df); + if (nextents <= XFS_INLINE_EXTS) + ifp->if_u1.if_extents = ifp->if_u2.if_inline_ext; (*src_log_flags) |= XFS_ILOG_DEXT; break; case XFS_DINODE_FMT_BTREE: @@ -1896,14 +1902,13 @@ xfs_swap_extent_forks( switch (tip->i_d.di_format) { case XFS_DINODE_FMT_EXTENTS: - /* If the extents fit in the inode, fix the - * pointer. Otherwise it's already NULL or - * pointing to the extent. + /* + * If the extents fit in the inode, fix the pointer. Otherwise + * it's already NULL or pointing to the extent. */ - if (tip->i_d.di_nextents <= XFS_INLINE_EXTS) { - tifp->if_u1.if_extents = - tifp->if_u2.if_inline_ext; - } + nextents = xfs_iext_count(&tip->i_df); + if (nextents <= XFS_INLINE_EXTS) + tifp->if_u1.if_extents = tifp->if_u2.if_inline_ext; (*target_log_flags) |= XFS_ILOG_DEXT; break; case XFS_DINODE_FMT_BTREE: diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c index b5b9bffe352074..d7a67d7fbc7f9c 100644 --- a/fs/xfs/xfs_buf.c +++ b/fs/xfs/xfs_buf.c @@ -423,6 +423,7 @@ xfs_buf_allocate_memory( out_free_pages: for (i = 0; i < bp->b_page_count; i++) __free_page(bp->b_pages[i]); + bp->b_flags &= ~_XBF_PAGES; return error; } diff --git a/fs/xfs/xfs_dquot.c b/fs/xfs/xfs_dquot.c index 7a30b8f11db7a2..9d06cc30e875e1 100644 --- a/fs/xfs/xfs_dquot.c +++ b/fs/xfs/xfs_dquot.c @@ -710,6 +710,10 @@ xfs_dq_get_next_id( /* Simple advance */ next_id = *id + 1; + /* If we'd wrap past the max ID, stop */ + if (next_id < *id) + return -ENOENT; + /* If new ID is within the current chunk, advancing it sufficed */ if (next_id % mp->m_quotainfo->qi_dqperchunk) { *id = next_id; diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index 6e4f7f900fea4c..9a5d64b5f35a1d 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -939,7 +939,6 @@ xfs_file_clone_range( len, false); } -#define XFS_MAX_DEDUPE_LEN (16 * 1024 * 1024) STATIC ssize_t xfs_file_dedupe_range( struct file *src_file, @@ -950,14 +949,6 @@ xfs_file_dedupe_range( { int error; - /* - * Limit the total length we will dedupe for each operation. - * This is intended to bound the total time spent in this - * ioctl to something sane. - */ - if (len > XFS_MAX_DEDUPE_LEN) - len = XFS_MAX_DEDUPE_LEN; - error = xfs_reflink_remap_range(src_file, loff, dst_file, dst_loff, len, true); if (error) diff --git a/fs/xfs/xfs_fsops.c b/fs/xfs/xfs_fsops.c index 93d12fa2670d53..242e8091296daf 100644 --- a/fs/xfs/xfs_fsops.c +++ b/fs/xfs/xfs_fsops.c @@ -631,6 +631,20 @@ xfs_growfs_data_private( xfs_set_low_space_thresholds(mp); mp->m_alloc_set_aside = xfs_alloc_set_aside(mp); + /* + * If we expanded the last AG, free the per-AG reservation + * so we can reinitialize it with the new size. + */ + if (new) { + struct xfs_perag *pag; + + pag = xfs_perag_get(mp, agno); + error = xfs_ag_resv_free(pag); + xfs_perag_put(pag); + if (error) + goto out; + } + /* Reserve AG metadata blocks. */ error = xfs_fs_reserve_ag_blocks(mp); if (error && error != -ENOSPC) diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index f295049db68159..29cc9886a3cb6e 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -123,7 +123,6 @@ __xfs_inode_free( { /* asserts to verify all state is correct here */ ASSERT(atomic_read(&ip->i_pincount) == 0); - ASSERT(!xfs_isiflocked(ip)); XFS_STATS_DEC(ip->i_mount, vn_active); call_rcu(&VFS_I(ip)->i_rcu, xfs_inode_free_callback); @@ -133,6 +132,8 @@ void xfs_inode_free( struct xfs_inode *ip) { + ASSERT(!xfs_isiflocked(ip)); + /* * Because we use RCU freeing we need to ensure the inode always * appears to be reclaimed with an invalid inode number when in the @@ -981,6 +982,7 @@ xfs_reclaim_inode( if (XFS_FORCED_SHUTDOWN(ip->i_mount)) { xfs_iunpin_wait(ip); + /* xfs_iflush_abort() drops the flush lock */ xfs_iflush_abort(ip, false); goto reclaim; } @@ -989,10 +991,10 @@ xfs_reclaim_inode( goto out_ifunlock; xfs_iunpin_wait(ip); } - if (xfs_iflags_test(ip, XFS_ISTALE)) - goto reclaim; - if (xfs_inode_clean(ip)) + if (xfs_iflags_test(ip, XFS_ISTALE) || xfs_inode_clean(ip)) { + xfs_ifunlock(ip); goto reclaim; + } /* * Never flush out dirty data during non-blocking reclaim, as it would @@ -1030,25 +1032,24 @@ xfs_reclaim_inode( xfs_buf_relse(bp); } - xfs_iflock(ip); reclaim: + ASSERT(!xfs_isiflocked(ip)); + /* * Because we use RCU freeing we need to ensure the inode always appears * to be reclaimed with an invalid inode number when in the free state. - * We do this as early as possible under the ILOCK and flush lock so - * that xfs_iflush_cluster() can be guaranteed to detect races with us - * here. By doing this, we guarantee that once xfs_iflush_cluster has - * locked both the XFS_ILOCK and the flush lock that it will see either - * a valid, flushable inode that will serialise correctly against the - * locks below, or it will see a clean (and invalid) inode that it can - * skip. + * We do this as early as possible under the ILOCK so that + * xfs_iflush_cluster() can be guaranteed to detect races with us here. + * By doing this, we guarantee that once xfs_iflush_cluster has locked + * XFS_ILOCK that it will see either a valid, flushable inode that will + * serialise correctly, or it will see a clean (and invalid) inode that + * it can skip. */ spin_lock(&ip->i_flags_lock); ip->i_flags = XFS_IRECLAIM; ip->i_ino = 0; spin_unlock(&ip->i_flags_lock); - xfs_ifunlock(ip); xfs_iunlock(ip, XFS_ILOCK_EXCL); XFS_STATS_INC(ip->i_mount, xs_ig_reclaims); @@ -1580,10 +1581,15 @@ xfs_inode_free_cowblocks( struct xfs_eofblocks *eofb = args; bool need_iolock = true; int match; + struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK); ASSERT(!eofb || (eofb && eofb->eof_scan_owner != 0)); - if (!xfs_reflink_has_real_cow_blocks(ip)) { + /* + * Just clear the tag if we have an empty cow fork or none at all. It's + * possible the inode was fully unshared since it was originally tagged. + */ + if (!xfs_is_reflink_inode(ip) || !ifp->if_bytes) { trace_xfs_inode_free_cowblocks_invalid(ip); xfs_inode_clear_cowblocks_tag(ip); return 0; @@ -1593,7 +1599,8 @@ xfs_inode_free_cowblocks( * If the mapping is dirty or under writeback we cannot touch the * CoW fork. Leave it alone if we're in the midst of a directio. */ - if (mapping_tagged(VFS_I(ip)->i_mapping, PAGECACHE_TAG_DIRTY) || + if ((VFS_I(ip)->i_state & I_DIRTY_PAGES) || + mapping_tagged(VFS_I(ip)->i_mapping, PAGECACHE_TAG_DIRTY) || mapping_tagged(VFS_I(ip)->i_mapping, PAGECACHE_TAG_WRITEBACK) || atomic_read(&VFS_I(ip)->i_dio_count)) return 0; diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index 4e560e6a12c1cc..512ff13ed66a95 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -2041,7 +2041,6 @@ xfs_iunlink( agi->agi_unlinked[bucket_index] = cpu_to_be32(agino); offset = offsetof(xfs_agi_t, agi_unlinked) + (sizeof(xfs_agino_t) * bucket_index); - xfs_trans_buf_set_type(tp, agibp, XFS_BLFT_AGI_BUF); xfs_trans_log_buf(tp, agibp, offset, (offset + sizeof(xfs_agino_t) - 1)); return 0; @@ -2133,7 +2132,6 @@ xfs_iunlink_remove( agi->agi_unlinked[bucket_index] = cpu_to_be32(next_agino); offset = offsetof(xfs_agi_t, agi_unlinked) + (sizeof(xfs_agino_t) * bucket_index); - xfs_trans_buf_set_type(tp, agibp, XFS_BLFT_AGI_BUF); xfs_trans_log_buf(tp, agibp, offset, (offset + sizeof(xfs_agino_t) - 1)); } else { diff --git a/fs/xfs/xfs_inode.h b/fs/xfs/xfs_inode.h index f14c1de2549db7..71e8a81c91a371 100644 --- a/fs/xfs/xfs_inode.h +++ b/fs/xfs/xfs_inode.h @@ -246,6 +246,11 @@ static inline bool xfs_is_reflink_inode(struct xfs_inode *ip) * Synchronize processes attempting to flush the in-core inode back to disk. */ +static inline int xfs_isiflocked(struct xfs_inode *ip) +{ + return xfs_iflags_test(ip, XFS_IFLOCK); +} + extern void __xfs_iflock(struct xfs_inode *ip); static inline int xfs_iflock_nowait(struct xfs_inode *ip) @@ -261,16 +266,12 @@ static inline void xfs_iflock(struct xfs_inode *ip) static inline void xfs_ifunlock(struct xfs_inode *ip) { + ASSERT(xfs_isiflocked(ip)); xfs_iflags_clear(ip, XFS_IFLOCK); smp_mb(); wake_up_bit(&ip->i_flags, __XFS_IFLOCK_BIT); } -static inline int xfs_isiflocked(struct xfs_inode *ip) -{ - return xfs_iflags_test(ip, XFS_IFLOCK); -} - /* * Flags for inode locking. * Bit ranges: 1<<1 - 1<<16-1 -- iolock/ilock modes (bitfield) diff --git a/fs/xfs/xfs_inode_item.c b/fs/xfs/xfs_inode_item.c index 9610e9c0095277..d90e7811ccdd9c 100644 --- a/fs/xfs/xfs_inode_item.c +++ b/fs/xfs/xfs_inode_item.c @@ -164,7 +164,7 @@ xfs_inode_item_format_data_fork( struct xfs_bmbt_rec *p; ASSERT(ip->i_df.if_u1.if_extents != NULL); - ASSERT(ip->i_df.if_bytes / sizeof(xfs_bmbt_rec_t) > 0); + ASSERT(xfs_iext_count(&ip->i_df) > 0); p = xlog_prepare_iovec(lv, vecp, XLOG_REG_TYPE_IEXT); data_bytes = xfs_iextents_copy(ip, p, XFS_DATA_FORK); @@ -261,7 +261,7 @@ xfs_inode_item_format_attr_fork( ip->i_afp->if_bytes > 0) { struct xfs_bmbt_rec *p; - ASSERT(ip->i_afp->if_bytes / sizeof(xfs_bmbt_rec_t) == + ASSERT(xfs_iext_count(ip->i_afp) == ip->i_d.di_anextents); ASSERT(ip->i_afp->if_u1.if_extents != NULL); diff --git a/fs/xfs/xfs_ioctl.c b/fs/xfs/xfs_ioctl.c index c245bed3249bf1..a39197501a7cd8 100644 --- a/fs/xfs/xfs_ioctl.c +++ b/fs/xfs/xfs_ioctl.c @@ -910,16 +910,14 @@ xfs_ioc_fsgetxattr( if (attr) { if (ip->i_afp) { if (ip->i_afp->if_flags & XFS_IFEXTENTS) - fa.fsx_nextents = ip->i_afp->if_bytes / - sizeof(xfs_bmbt_rec_t); + fa.fsx_nextents = xfs_iext_count(ip->i_afp); else fa.fsx_nextents = ip->i_d.di_anextents; } else fa.fsx_nextents = 0; } else { if (ip->i_df.if_flags & XFS_IFEXTENTS) - fa.fsx_nextents = ip->i_df.if_bytes / - sizeof(xfs_bmbt_rec_t); + fa.fsx_nextents = xfs_iext_count(&ip->i_df); else fa.fsx_nextents = ip->i_d.di_nextents; } diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c index 436e109bb01e59..cdc6bdd495be25 100644 --- a/fs/xfs/xfs_iomap.c +++ b/fs/xfs/xfs_iomap.c @@ -395,11 +395,12 @@ xfs_iomap_prealloc_size( struct xfs_inode *ip, loff_t offset, loff_t count, - xfs_extnum_t idx, - struct xfs_bmbt_irec *prev) + xfs_extnum_t idx) { struct xfs_mount *mp = ip->i_mount; + struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_DATA_FORK); xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset); + struct xfs_bmbt_irec prev; int shift = 0; int64_t freesp; xfs_fsblock_t qblocks; @@ -419,8 +420,8 @@ xfs_iomap_prealloc_size( */ if ((mp->m_flags & XFS_MOUNT_DFLT_IOSIZE) || XFS_ISIZE(ip) < XFS_FSB_TO_B(mp, mp->m_dalign) || - idx == 0 || - prev->br_startoff + prev->br_blockcount < offset_fsb) + !xfs_iext_get_extent(ifp, idx - 1, &prev) || + prev.br_startoff + prev.br_blockcount < offset_fsb) return mp->m_writeio_blocks; /* @@ -439,8 +440,8 @@ xfs_iomap_prealloc_size( * always extends to MAXEXTLEN rather than falling short due to things * like stripe unit/width alignment of real extents. */ - if (prev->br_blockcount <= (MAXEXTLEN >> 1)) - alloc_blocks = prev->br_blockcount << 1; + if (prev.br_blockcount <= (MAXEXTLEN >> 1)) + alloc_blocks = prev.br_blockcount << 1; else alloc_blocks = XFS_B_TO_FSB(mp, offset); if (!alloc_blocks) @@ -535,11 +536,11 @@ xfs_file_iomap_begin_delay( xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset); xfs_fileoff_t maxbytes_fsb = XFS_B_TO_FSB(mp, mp->m_super->s_maxbytes); - xfs_fileoff_t end_fsb, orig_end_fsb; + xfs_fileoff_t end_fsb; int error = 0, eof = 0; struct xfs_bmbt_irec got; - struct xfs_bmbt_irec prev; xfs_extnum_t idx; + xfs_fsblock_t prealloc_blocks = 0; ASSERT(!XFS_IS_REALTIME_INODE(ip)); ASSERT(!xfs_get_extsz_hint(ip)); @@ -563,8 +564,7 @@ xfs_file_iomap_begin_delay( goto out_unlock; } - xfs_bmap_search_extents(ip, offset_fsb, XFS_DATA_FORK, &eof, &idx, - &got, &prev); + eof = !xfs_iext_lookup_extent(ip, ifp, offset_fsb, &idx, &got); if (!eof && got.br_startoff <= offset_fsb) { if (xfs_is_reflink_inode(ip)) { bool shared; @@ -595,35 +595,32 @@ xfs_file_iomap_begin_delay( * the lower level functions are updated. */ count = min_t(loff_t, count, 1024 * PAGE_SIZE); - end_fsb = orig_end_fsb = - min(XFS_B_TO_FSB(mp, offset + count), maxbytes_fsb); + end_fsb = min(XFS_B_TO_FSB(mp, offset + count), maxbytes_fsb); if (eof) { - xfs_fsblock_t prealloc_blocks; - - prealloc_blocks = - xfs_iomap_prealloc_size(ip, offset, count, idx, &prev); + prealloc_blocks = xfs_iomap_prealloc_size(ip, offset, count, idx); if (prealloc_blocks) { xfs_extlen_t align; xfs_off_t end_offset; + xfs_fileoff_t p_end_fsb; end_offset = XFS_WRITEIO_ALIGN(mp, offset + count - 1); - end_fsb = XFS_B_TO_FSBT(mp, end_offset) + - prealloc_blocks; + p_end_fsb = XFS_B_TO_FSBT(mp, end_offset) + + prealloc_blocks; align = xfs_eof_alignment(ip, 0); if (align) - end_fsb = roundup_64(end_fsb, align); + p_end_fsb = roundup_64(p_end_fsb, align); - end_fsb = min(end_fsb, maxbytes_fsb); - ASSERT(end_fsb > offset_fsb); + p_end_fsb = min(p_end_fsb, maxbytes_fsb); + ASSERT(p_end_fsb > offset_fsb); + prealloc_blocks = p_end_fsb - end_fsb; } } retry: error = xfs_bmapi_reserve_delalloc(ip, XFS_DATA_FORK, offset_fsb, - end_fsb - offset_fsb, &got, - &prev, &idx, eof); + end_fsb - offset_fsb, prealloc_blocks, &got, &idx, eof); switch (error) { case 0: break; @@ -631,8 +628,8 @@ xfs_file_iomap_begin_delay( case -EDQUOT: /* retry without any preallocation */ trace_xfs_delalloc_enospc(ip, offset, count); - if (end_fsb != orig_end_fsb) { - end_fsb = orig_end_fsb; + if (prealloc_blocks) { + prealloc_blocks = 0; goto retry; } /*FALLTHRU*/ @@ -640,13 +637,6 @@ xfs_file_iomap_begin_delay( goto out_unlock; } - /* - * Tag the inode as speculatively preallocated so we can reclaim this - * space on demand, if necessary. - */ - if (end_fsb != orig_end_fsb) - xfs_inode_set_eofblocks_tag(ip); - trace_xfs_iomap_alloc(ip, offset, count, 0, &got); done: if (isnullstartblock(got.br_startblock)) @@ -691,7 +681,7 @@ xfs_iomap_write_allocate( xfs_trans_t *tp; int nimaps; int error = 0; - int flags = 0; + int flags = XFS_BMAPI_DELALLOC; int nres; if (whichfork == XFS_COW_FORK) diff --git a/fs/xfs/xfs_iops.c b/fs/xfs/xfs_iops.c index 405a65cd9d6bcf..f5e0f608e2455a 100644 --- a/fs/xfs/xfs_iops.c +++ b/fs/xfs/xfs_iops.c @@ -97,13 +97,28 @@ xfs_init_security( static void xfs_dentry_to_name( + struct xfs_name *namep, + struct dentry *dentry) +{ + namep->name = dentry->d_name.name; + namep->len = dentry->d_name.len; + namep->type = XFS_DIR3_FT_UNKNOWN; +} + +static int +xfs_dentry_mode_to_name( struct xfs_name *namep, struct dentry *dentry, int mode) { namep->name = dentry->d_name.name; namep->len = dentry->d_name.len; - namep->type = xfs_mode_to_ftype[(mode & S_IFMT) >> S_SHIFT]; + namep->type = xfs_mode_to_ftype(mode); + + if (unlikely(namep->type == XFS_DIR3_FT_UNKNOWN)) + return -EFSCORRUPTED; + + return 0; } STATIC void @@ -119,7 +134,7 @@ xfs_cleanup_inode( * xfs_init_security we must back out. * ENOSPC can hit here, among other things. */ - xfs_dentry_to_name(&teardown, dentry, 0); + xfs_dentry_to_name(&teardown, dentry); xfs_remove(XFS_I(dir), &teardown, XFS_I(inode)); } @@ -154,8 +169,12 @@ xfs_generic_create( if (error) return error; + /* Verify mode is valid also for tmpfile case */ + error = xfs_dentry_mode_to_name(&name, dentry, mode); + if (unlikely(error)) + goto out_free_acl; + if (!tmpfile) { - xfs_dentry_to_name(&name, dentry, mode); error = xfs_create(XFS_I(dir), &name, mode, rdev, &ip); } else { error = xfs_create_tmpfile(XFS_I(dir), dentry, mode, &ip); @@ -248,7 +267,7 @@ xfs_vn_lookup( if (dentry->d_name.len >= MAXNAMELEN) return ERR_PTR(-ENAMETOOLONG); - xfs_dentry_to_name(&name, dentry, 0); + xfs_dentry_to_name(&name, dentry); error = xfs_lookup(XFS_I(dir), &name, &cip, NULL); if (unlikely(error)) { if (unlikely(error != -ENOENT)) @@ -275,7 +294,7 @@ xfs_vn_ci_lookup( if (dentry->d_name.len >= MAXNAMELEN) return ERR_PTR(-ENAMETOOLONG); - xfs_dentry_to_name(&xname, dentry, 0); + xfs_dentry_to_name(&xname, dentry); error = xfs_lookup(XFS_I(dir), &xname, &ip, &ci_name); if (unlikely(error)) { if (unlikely(error != -ENOENT)) @@ -310,7 +329,9 @@ xfs_vn_link( struct xfs_name name; int error; - xfs_dentry_to_name(&name, dentry, inode->i_mode); + error = xfs_dentry_mode_to_name(&name, dentry, inode->i_mode); + if (unlikely(error)) + return error; error = xfs_link(XFS_I(dir), XFS_I(inode), &name); if (unlikely(error)) @@ -329,7 +350,7 @@ xfs_vn_unlink( struct xfs_name name; int error; - xfs_dentry_to_name(&name, dentry, 0); + xfs_dentry_to_name(&name, dentry); error = xfs_remove(XFS_I(dir), &name, XFS_I(d_inode(dentry))); if (error) @@ -359,7 +380,9 @@ xfs_vn_symlink( mode = S_IFLNK | (irix_symlink_mode ? 0777 & ~current_umask() : S_IRWXUGO); - xfs_dentry_to_name(&name, dentry, mode); + error = xfs_dentry_mode_to_name(&name, dentry, mode); + if (unlikely(error)) + goto out; error = xfs_symlink(XFS_I(dir), &name, symname, mode, &cip); if (unlikely(error)) @@ -395,6 +418,7 @@ xfs_vn_rename( { struct inode *new_inode = d_inode(ndentry); int omode = 0; + int error; struct xfs_name oname; struct xfs_name nname; @@ -405,8 +429,14 @@ xfs_vn_rename( if (flags & RENAME_EXCHANGE) omode = d_inode(ndentry)->i_mode; - xfs_dentry_to_name(&oname, odentry, omode); - xfs_dentry_to_name(&nname, ndentry, d_inode(odentry)->i_mode); + error = xfs_dentry_mode_to_name(&oname, odentry, omode); + if (omode && unlikely(error)) + return error; + + error = xfs_dentry_mode_to_name(&nname, ndentry, + d_inode(odentry)->i_mode); + if (unlikely(error)) + return error; return xfs_rename(XFS_I(odir), &oname, XFS_I(d_inode(odentry)), XFS_I(ndir), &nname, diff --git a/fs/xfs/xfs_linux.h b/fs/xfs/xfs_linux.h index 68640fb63a5423..1455b25205a8ea 100644 --- a/fs/xfs/xfs_linux.h +++ b/fs/xfs/xfs_linux.h @@ -330,11 +330,11 @@ static inline __uint64_t howmany_64(__uint64_t x, __uint32_t y) } #define ASSERT_ALWAYS(expr) \ - (unlikely(expr) ? (void)0 : assfail(#expr, __FILE__, __LINE__)) + (likely(expr) ? (void)0 : assfail(#expr, __FILE__, __LINE__)) #ifdef DEBUG #define ASSERT(expr) \ - (unlikely(expr) ? (void)0 : assfail(#expr, __FILE__, __LINE__)) + (likely(expr) ? (void)0 : assfail(#expr, __FILE__, __LINE__)) #ifndef STATIC # define STATIC noinline @@ -345,7 +345,7 @@ static inline __uint64_t howmany_64(__uint64_t x, __uint32_t y) #ifdef XFS_WARN #define ASSERT(expr) \ - (unlikely(expr) ? (void)0 : asswarn(#expr, __FILE__, __LINE__)) + (likely(expr) ? (void)0 : asswarn(#expr, __FILE__, __LINE__)) #ifndef STATIC # define STATIC static noinline diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c index 3b74fa011bb156..4017aa96733176 100644 --- a/fs/xfs/xfs_log.c +++ b/fs/xfs/xfs_log.c @@ -3324,12 +3324,8 @@ xfs_log_force( xfs_mount_t *mp, uint flags) { - int error; - trace_xfs_log_force(mp, 0, _RET_IP_); - error = _xfs_log_force(mp, flags, NULL); - if (error) - xfs_warn(mp, "%s: error %d returned.", __func__, error); + _xfs_log_force(mp, flags, NULL); } /* @@ -3473,12 +3469,8 @@ xfs_log_force_lsn( xfs_lsn_t lsn, uint flags) { - int error; - trace_xfs_log_force(mp, lsn, _RET_IP_); - error = _xfs_log_force_lsn(mp, lsn, flags, NULL); - if (error) - xfs_warn(mp, "%s: error %d returned.", __func__, error); + _xfs_log_force_lsn(mp, lsn, flags, NULL); } /* diff --git a/fs/xfs/xfs_qm.c b/fs/xfs/xfs_qm.c index a60d9e2739d14a..b669b123287bb1 100644 --- a/fs/xfs/xfs_qm.c +++ b/fs/xfs/xfs_qm.c @@ -1135,7 +1135,7 @@ xfs_qm_get_rtblks( return error; } rtblks = 0; - nextents = ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t); + nextents = xfs_iext_count(ifp); for (idx = 0; idx < nextents; idx++) rtblks += xfs_bmbt_get_blockcount(xfs_iext_get_ext(ifp, idx)); *O_rtblks = (xfs_qcnt_t)rtblks; @@ -1177,7 +1177,8 @@ xfs_qm_dqusage_adjust( * the case in all other instances. It's OK that we do this because * quotacheck is done only at mount time. */ - error = xfs_iget(mp, NULL, ino, 0, XFS_ILOCK_EXCL, &ip); + error = xfs_iget(mp, NULL, ino, XFS_IGET_DONTCACHE, XFS_ILOCK_EXCL, + &ip); if (error) { *res = BULKSTAT_RV_NOTHING; return error; diff --git a/fs/xfs/xfs_refcount_item.c b/fs/xfs/xfs_refcount_item.c index fe86a668a57e70..6e4c7446c3d456 100644 --- a/fs/xfs/xfs_refcount_item.c +++ b/fs/xfs/xfs_refcount_item.c @@ -526,13 +526,14 @@ xfs_cui_recover( xfs_refcount_finish_one_cleanup(tp, rcur, error); error = xfs_defer_finish(&tp, &dfops, NULL); if (error) - goto abort_error; + goto abort_defer; set_bit(XFS_CUI_RECOVERED, &cuip->cui_flags); error = xfs_trans_commit(tp); return error; abort_error: xfs_refcount_finish_one_cleanup(tp, rcur, error); +abort_defer: xfs_defer_cancel(&dfops); xfs_trans_cancel(tp); return error; diff --git a/fs/xfs/xfs_reflink.c b/fs/xfs/xfs_reflink.c index a279b4e7f5feaa..4d3f74e3c5e1d0 100644 --- a/fs/xfs/xfs_reflink.c +++ b/fs/xfs/xfs_reflink.c @@ -243,12 +243,11 @@ xfs_reflink_reserve_cow( struct xfs_bmbt_irec *imap, bool *shared) { - struct xfs_bmbt_irec got, prev; - xfs_fileoff_t end_fsb, orig_end_fsb; - int eof = 0, error = 0; - bool trimmed; + struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK); + struct xfs_bmbt_irec got; + int error = 0; + bool eof = false, trimmed; xfs_extnum_t idx; - xfs_extlen_t align; /* * Search the COW fork extent list first. This serves two purposes: @@ -258,8 +257,9 @@ xfs_reflink_reserve_cow( * extent list is generally faster than going out to the shared extent * tree. */ - xfs_bmap_search_extents(ip, imap->br_startoff, XFS_COW_FORK, &eof, &idx, - &got, &prev); + + if (!xfs_iext_lookup_extent(ip, ifp, imap->br_startoff, &idx, &got)) + eof = true; if (!eof && got.br_startoff <= imap->br_startoff) { trace_xfs_reflink_cow_found(ip, imap); xfs_trim_extent(imap, got.br_startoff, got.br_blockcount); @@ -285,33 +285,12 @@ xfs_reflink_reserve_cow( if (error) return error; - end_fsb = orig_end_fsb = imap->br_startoff + imap->br_blockcount; - - align = xfs_eof_alignment(ip, xfs_get_cowextsz_hint(ip)); - if (align) - end_fsb = roundup_64(end_fsb, align); - -retry: error = xfs_bmapi_reserve_delalloc(ip, XFS_COW_FORK, imap->br_startoff, - end_fsb - imap->br_startoff, &got, &prev, &idx, eof); - switch (error) { - case 0: - break; - case -ENOSPC: - case -EDQUOT: - /* retry without any preallocation */ + imap->br_blockcount, 0, &got, &idx, eof); + if (error == -ENOSPC || error == -EDQUOT) trace_xfs_reflink_cow_enospc(ip, imap); - if (end_fsb != orig_end_fsb) { - end_fsb = orig_end_fsb; - goto retry; - } - /*FALLTHRU*/ - default: + if (error) return error; - } - - if (end_fsb != orig_end_fsb) - xfs_inode_set_cowblocks_tag(ip); trace_xfs_reflink_cow_alloc(ip, &got); return 0; @@ -486,7 +465,7 @@ xfs_reflink_trim_irec_to_next_cow( /* This is the extent before; try sliding up one. */ if (irec.br_startoff < offset_fsb) { idx++; - if (idx >= ifp->if_bytes / sizeof(xfs_bmbt_rec_t)) + if (idx >= xfs_iext_count(ifp)) return 0; gotp = xfs_iext_get_ext(ifp, idx); xfs_bmbt_get_all(gotp, &irec); @@ -566,7 +545,7 @@ xfs_reflink_cancel_cow_blocks( xfs_bmap_del_extent_cow(ip, &idx, &got, &del); } - if (++idx >= ifp->if_bytes / sizeof(struct xfs_bmbt_rec)) + if (++idx >= xfs_iext_count(ifp)) break; xfs_bmbt_get_all(xfs_iext_get_ext(ifp, idx), &got); } @@ -1345,8 +1324,14 @@ xfs_reflink_remap_range( goto out_unlock; } - if (len == 0) + /* Zero length dedupe exits immediately; reflink goes to EOF. */ + if (len == 0) { + if (is_dedupe) { + ret = 0; + goto out_unlock; + } len = isize - pos_in; + } /* Ensure offsets don't wrap and the input is inside i_size */ if (pos_in + len < pos_in || pos_out + len < pos_out || @@ -1697,37 +1682,3 @@ xfs_reflink_unshare( trace_xfs_reflink_unshare_error(ip, error, _RET_IP_); return error; } - -/* - * Does this inode have any real CoW reservations? - */ -bool -xfs_reflink_has_real_cow_blocks( - struct xfs_inode *ip) -{ - struct xfs_bmbt_irec irec; - struct xfs_ifork *ifp; - struct xfs_bmbt_rec_host *gotp; - xfs_extnum_t idx; - - if (!xfs_is_reflink_inode(ip)) - return false; - - /* Go find the old extent in the CoW fork. */ - ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK); - gotp = xfs_iext_bno_to_ext(ifp, 0, &idx); - while (gotp) { - xfs_bmbt_get_all(gotp, &irec); - - if (!isnullstartblock(irec.br_startblock)) - return true; - - /* Roll on... */ - idx++; - if (idx >= ifp->if_bytes / sizeof(xfs_bmbt_rec_t)) - break; - gotp = xfs_iext_get_ext(ifp, idx); - } - - return false; -} diff --git a/fs/xfs/xfs_reflink.h b/fs/xfs/xfs_reflink.h index fad11607c9adf3..97ea9b487884e3 100644 --- a/fs/xfs/xfs_reflink.h +++ b/fs/xfs/xfs_reflink.h @@ -50,6 +50,4 @@ extern int xfs_reflink_clear_inode_flag(struct xfs_inode *ip, extern int xfs_reflink_unshare(struct xfs_inode *ip, xfs_off_t offset, xfs_off_t len); -extern bool xfs_reflink_has_real_cow_blocks(struct xfs_inode *ip); - #endif /* __XFS_REFLINK_H */ diff --git a/fs/xfs/xfs_sysfs.c b/fs/xfs/xfs_sysfs.c index 276d3023d60f82..de6195e3891096 100644 --- a/fs/xfs/xfs_sysfs.c +++ b/fs/xfs/xfs_sysfs.c @@ -396,7 +396,7 @@ max_retries_show( int retries; struct xfs_error_cfg *cfg = to_error_cfg(kobject); - if (cfg->retry_timeout == XFS_ERR_RETRY_FOREVER) + if (cfg->max_retries == XFS_ERR_RETRY_FOREVER) retries = -1; else retries = cfg->max_retries; @@ -422,7 +422,7 @@ max_retries_store( return -EINVAL; if (val == -1) - cfg->retry_timeout = XFS_ERR_RETRY_FOREVER; + cfg->max_retries = XFS_ERR_RETRY_FOREVER; else cfg->max_retries = val; return count; diff --git a/include/asm-generic/asm-prototypes.h b/include/asm-generic/asm-prototypes.h new file mode 100644 index 00000000000000..939869c772b17a --- /dev/null +++ b/include/asm-generic/asm-prototypes.h @@ -0,0 +1,13 @@ +#include +#undef __memset +extern void *__memset(void *, int, __kernel_size_t); +#undef __memcpy +extern void *__memcpy(void *, const void *, __kernel_size_t); +#undef __memmove +extern void *__memmove(void *, const void *, __kernel_size_t); +#undef memset +extern void *memset(void *, int, __kernel_size_t); +#undef memcpy +extern void *memcpy(void *, const void *, __kernel_size_t); +#undef memmove +extern void *memmove(void *, const void *, __kernel_size_t); diff --git a/include/dt-bindings/clock/r8a7794-clock.h b/include/dt-bindings/clock/r8a7794-clock.h index 9d02f5317c7c31..88e64846cf370f 100644 --- a/include/dt-bindings/clock/r8a7794-clock.h +++ b/include/dt-bindings/clock/r8a7794-clock.h @@ -20,8 +20,7 @@ #define R8A7794_CLK_QSPI 5 #define R8A7794_CLK_SDH 6 #define R8A7794_CLK_SD0 7 -#define R8A7794_CLK_Z 8 -#define R8A7794_CLK_RCAN 9 +#define R8A7794_CLK_RCAN 8 /* MSTP0 */ #define R8A7794_CLK_MSIOF0 0 diff --git a/include/linux/Kbuild b/include/linux/Kbuild new file mode 100644 index 00000000000000..a4608897a5f327 --- /dev/null +++ b/include/linux/Kbuild @@ -0,0 +1,2 @@ +header-y += if_pppolac.h +header-y += if_pppopns.h diff --git a/include/linux/amba/mmci.h b/include/linux/amba/mmci.h index 8c98113069ce92..eff56cb0016a7c 100644 --- a/include/linux/amba/mmci.h +++ b/include/linux/amba/mmci.h @@ -5,6 +5,15 @@ #define AMBA_MMCI_H #include +#include +#include + +struct embedded_sdio_data { + struct sdio_cis cis; + struct sdio_cccr cccr; + struct sdio_embedded_func *funcs; + int num_funcs; +}; /** * struct mmci_platform_data - platform configuration for the MMCI @@ -31,6 +40,7 @@ struct mmci_platform_data { int gpio_wp; int gpio_cd; bool cd_invert; + struct embedded_sdio_data *embedded_sdio; }; #endif diff --git a/include/linux/android_aid.h b/include/linux/android_aid.h new file mode 100644 index 00000000000000..6f1fa1792dfc22 --- /dev/null +++ b/include/linux/android_aid.h @@ -0,0 +1,28 @@ +/* include/linux/android_aid.h + * + * Copyright (C) 2008 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#ifndef _LINUX_ANDROID_AID_H +#define _LINUX_ANDROID_AID_H + +/* AIDs that the kernel treats differently */ +#define AID_OBSOLETE_000 KGIDT_INIT(3001) /* was NET_BT_ADMIN */ +#define AID_OBSOLETE_001 KGIDT_INIT(3002) /* was NET_BT */ +#define AID_INET KGIDT_INIT(3003) +#define AID_NET_RAW KGIDT_INIT(3004) +#define AID_NET_ADMIN KGIDT_INIT(3005) +#define AID_NET_BW_STATS KGIDT_INIT(3006) /* read bandwidth statistics */ +#define AID_NET_BW_ACCT KGIDT_INIT(3007) /* change bandwidth statistics accounting */ + +#endif diff --git a/include/linux/arm-smccc.h b/include/linux/arm-smccc.h index b5abfda80465a1..699c0d879922b5 100644 --- a/include/linux/arm-smccc.h +++ b/include/linux/arm-smccc.h @@ -23,8 +23,9 @@ * http://infocenter.arm.com/help/topic/com.arm.doc.den0028a/index.html */ -#define ARM_SMCCC_STD_CALL 0 -#define ARM_SMCCC_FAST_CALL 1 +/* This constant is shifted by 31, make sure it's of an unsigned type */ +#define ARM_SMCCC_STD_CALL 0UL +#define ARM_SMCCC_FAST_CALL 1UL #define ARM_SMCCC_TYPE_SHIFT 31 #define ARM_SMCCC_SMC_32 0 diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index c47c358ba0529c..e47a7f7025a090 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -196,6 +196,9 @@ struct request { /* for bidi */ struct request *next_rq; + + ktime_t lat_hist_io_start; + int lat_hist_enabled; }; #define REQ_OP_SHIFT (8 * sizeof(u64) - REQ_OP_BITS) @@ -1057,7 +1060,7 @@ static inline int blk_pre_runtime_suspend(struct request_queue *q) static inline void blk_post_runtime_suspend(struct request_queue *q, int err) {} static inline void blk_pre_runtime_resume(struct request_queue *q) {} static inline void blk_post_runtime_resume(struct request_queue *q, int err) {} -extern inline void blk_set_runtime_active(struct request_queue *q) {} +static inline void blk_set_runtime_active(struct request_queue *q) {} #endif /* @@ -1700,6 +1703,79 @@ extern int bdev_write_page(struct block_device *, sector_t, struct page *, extern long bdev_direct_access(struct block_device *, struct blk_dax_ctl *); extern int bdev_dax_supported(struct super_block *, int); extern bool bdev_dax_capable(struct block_device *); + +/* + * X-axis for IO latency histogram support. + */ +static const u_int64_t latency_x_axis_us[] = { + 100, + 200, + 300, + 400, + 500, + 600, + 700, + 800, + 900, + 1000, + 1200, + 1400, + 1600, + 1800, + 2000, + 2500, + 3000, + 4000, + 5000, + 6000, + 7000, + 9000, + 10000 +}; + +#define BLK_IO_LAT_HIST_DISABLE 0 +#define BLK_IO_LAT_HIST_ENABLE 1 +#define BLK_IO_LAT_HIST_ZERO 2 + +struct io_latency_state { + u_int64_t latency_y_axis_read[ARRAY_SIZE(latency_x_axis_us) + 1]; + u_int64_t latency_reads_elems; + u_int64_t latency_y_axis_write[ARRAY_SIZE(latency_x_axis_us) + 1]; + u_int64_t latency_writes_elems; +}; + +static inline void +blk_update_latency_hist(struct io_latency_state *s, + int read, + u_int64_t delta_us) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(latency_x_axis_us); i++) { + if (delta_us < (u_int64_t)latency_x_axis_us[i]) { + if (read) + s->latency_y_axis_read[i]++; + else + s->latency_y_axis_write[i]++; + break; + } + } + if (i == ARRAY_SIZE(latency_x_axis_us)) { + /* Overflowed the histogram */ + if (read) + s->latency_y_axis_read[i]++; + else + s->latency_y_axis_write[i]++; + } + if (read) + s->latency_reads_elems++; + else + s->latency_writes_elems++; +} + +void blk_zero_latency_hist(struct io_latency_state *s); +ssize_t blk_latency_hist_show(struct io_latency_state *s, char *buf); + #else /* CONFIG_BLOCK */ struct block_device; diff --git a/include/linux/capability.h b/include/linux/capability.h index dbc21c719ce61b..6ffb67e10c060a 100644 --- a/include/linux/capability.h +++ b/include/linux/capability.h @@ -240,8 +240,10 @@ static inline bool ns_capable_noaudit(struct user_namespace *ns, int cap) return true; } #endif /* CONFIG_MULTIUSER */ +extern bool privileged_wrt_inode_uidgid(struct user_namespace *ns, const struct inode *inode); extern bool capable_wrt_inode_uidgid(const struct inode *inode, int cap); extern bool file_ns_capable(const struct file *file, struct user_namespace *ns, int cap); +extern bool ptracer_capable(struct task_struct *tsk, struct user_namespace *ns); /* audit system wants to get cap info from files as well */ extern int get_vfs_caps_from_disk(const struct dentry *dentry, struct cpu_vfs_cap_data *cpu_caps); diff --git a/include/linux/cgroup_subsys.h b/include/linux/cgroup_subsys.h index 0df0336acee9ec..7f4a2a5a2a7718 100644 --- a/include/linux/cgroup_subsys.h +++ b/include/linux/cgroup_subsys.h @@ -20,6 +20,10 @@ SUBSYS(cpu) SUBSYS(cpuacct) #endif +#if IS_ENABLED(CONFIG_CGROUP_SCHEDTUNE) +SUBSYS(schedtune) +#endif + #if IS_ENABLED(CONFIG_BLK_CGROUP) SUBSYS(io) #endif diff --git a/include/linux/cpu.h b/include/linux/cpu.h index b886dc17f2f345..dfd2b7c14ef70a 100644 --- a/include/linux/cpu.h +++ b/include/linux/cpu.h @@ -93,22 +93,16 @@ extern bool cpuhp_tasks_frozen; { .notifier_call = fn, .priority = pri }; \ __register_cpu_notifier(&fn##_nb); \ } -#else /* #if defined(CONFIG_HOTPLUG_CPU) || !defined(MODULE) */ -#define cpu_notifier(fn, pri) do { (void)(fn); } while (0) -#define __cpu_notifier(fn, pri) do { (void)(fn); } while (0) -#endif /* #else #if defined(CONFIG_HOTPLUG_CPU) || !defined(MODULE) */ -#ifdef CONFIG_HOTPLUG_CPU extern int register_cpu_notifier(struct notifier_block *nb); extern int __register_cpu_notifier(struct notifier_block *nb); extern void unregister_cpu_notifier(struct notifier_block *nb); extern void __unregister_cpu_notifier(struct notifier_block *nb); -#else -#ifndef MODULE -extern int register_cpu_notifier(struct notifier_block *nb); -extern int __register_cpu_notifier(struct notifier_block *nb); -#else +#else /* #if defined(CONFIG_HOTPLUG_CPU) || !defined(MODULE) */ +#define cpu_notifier(fn, pri) do { (void)(fn); } while (0) +#define __cpu_notifier(fn, pri) do { (void)(fn); } while (0) + static inline int register_cpu_notifier(struct notifier_block *nb) { return 0; @@ -118,7 +112,6 @@ static inline int __register_cpu_notifier(struct notifier_block *nb) { return 0; } -#endif static inline void unregister_cpu_notifier(struct notifier_block *nb) { @@ -253,4 +246,11 @@ void cpuhp_report_idle_dead(void); static inline void cpuhp_report_idle_dead(void) { } #endif /* #ifdef CONFIG_HOTPLUG_CPU */ +#define IDLE_START 1 +#define IDLE_END 2 + +void idle_notifier_register(struct notifier_block *n); +void idle_notifier_unregister(struct notifier_block *n); +void idle_notifier_call_chain(unsigned long val); + #endif /* _LINUX_CPU_H_ */ diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index 32dc0cbd51ca37..cc57986d3bfeeb 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -177,6 +177,7 @@ u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy); int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu); int cpufreq_update_policy(unsigned int cpu); bool have_governor_per_policy(void); +bool cpufreq_driver_is_slow(void); struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy); void cpufreq_enable_fast_switch(struct cpufreq_policy *policy); void cpufreq_disable_fast_switch(struct cpufreq_policy *policy); @@ -359,6 +360,14 @@ struct cpufreq_driver { */ #define CPUFREQ_NEED_INITIAL_FREQ_CHECK (1 << 5) +/* + * Indicates that it is safe to call cpufreq_driver_target from + * non-interruptable context in scheduler hot paths. Drivers must + * opt-in to this flag, as the safe default is that they might sleep + * or be too slow for hot path use. + */ +#define CPUFREQ_DRIVER_FAST (1 << 6) + int cpufreq_register_driver(struct cpufreq_driver *driver_data); int cpufreq_unregister_driver(struct cpufreq_driver *driver_data); @@ -553,6 +562,32 @@ struct governor_attr { ssize_t (*store)(struct gov_attr_set *attr_set, const char *buf, size_t count); }; +/* CPUFREQ DEFAULT GOVERNOR */ +/* + * Performance governor is fallback governor if any other gov failed to auto + * load due latency restrictions + */ +#ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE +extern struct cpufreq_governor cpufreq_gov_performance; +#endif +#ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE +#define CPUFREQ_DEFAULT_GOVERNOR (&cpufreq_gov_performance) +#elif defined(CONFIG_CPU_FREQ_DEFAULT_GOV_POWERSAVE) +extern struct cpufreq_governor cpufreq_gov_powersave; +#define CPUFREQ_DEFAULT_GOVERNOR (&cpufreq_gov_powersave) +#elif defined(CONFIG_CPU_FREQ_DEFAULT_GOV_USERSPACE) +extern struct cpufreq_governor cpufreq_gov_userspace; +#define CPUFREQ_DEFAULT_GOVERNOR (&cpufreq_gov_userspace) +#elif defined(CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND) +extern struct cpufreq_governor cpufreq_gov_ondemand; +#define CPUFREQ_DEFAULT_GOVERNOR (&cpufreq_gov_ondemand) +#elif defined(CONFIG_CPU_FREQ_DEFAULT_GOV_CONSERVATIVE) +extern struct cpufreq_governor cpufreq_gov_conservative; +#define CPUFREQ_DEFAULT_GOVERNOR (&cpufreq_gov_conservative) +#elif defined(CONFIG_CPU_FREQ_DEFAULT_GOV_SCHED) +extern struct cpufreq_governor cpufreq_gov_sched; +#define CPUFREQ_DEFAULT_GOVERNOR (&cpufreq_gov_sched) +#endif /********************************************************************* * FREQUENCY TABLE HELPERS * @@ -886,4 +921,8 @@ unsigned int cpufreq_generic_get(unsigned int cpu); int cpufreq_generic_init(struct cpufreq_policy *policy, struct cpufreq_frequency_table *table, unsigned int transition_latency); + +struct sched_domain; +unsigned long cpufreq_scale_freq_capacity(struct sched_domain *sd, int cpu); +unsigned long cpufreq_scale_max_freq_capacity(int cpu); #endif /* _LINUX_CPUFREQ_H */ diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index afe641c02dca33..ba1cad7b97cfa1 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -80,7 +80,6 @@ enum cpuhp_state { CPUHP_AP_ARM_L2X0_STARTING, CPUHP_AP_ARM_ARCH_TIMER_STARTING, CPUHP_AP_ARM_GLOBAL_TIMER_STARTING, - CPUHP_AP_DUMMY_TIMER_STARTING, CPUHP_AP_JCORE_TIMER_STARTING, CPUHP_AP_EXYNOS4_MCT_TIMER_STARTING, CPUHP_AP_ARM_TWD_STARTING, @@ -94,6 +93,8 @@ enum cpuhp_state { CPUHP_AP_KVM_ARM_VGIC_INIT_STARTING, CPUHP_AP_KVM_ARM_VGIC_STARTING, CPUHP_AP_KVM_ARM_TIMER_STARTING, + /* Must be the last timer callback */ + CPUHP_AP_DUMMY_TIMER_STARTING, CPUHP_AP_ARM_XEN_STARTING, CPUHP_AP_ARM_CORESIGHT_STARTING, CPUHP_AP_ARM_CORESIGHT4_STARTING, diff --git a/include/linux/cpuidle.h b/include/linux/cpuidle.h index bb31373c3478bd..9a8eec9e59b209 100644 --- a/include/linux/cpuidle.h +++ b/include/linux/cpuidle.h @@ -207,7 +207,7 @@ static inline int cpuidle_enter_freeze(struct cpuidle_driver *drv, #endif /* kernel/sched/idle.c */ -extern void sched_idle_set_state(struct cpuidle_state *idle_state); +extern void sched_idle_set_state(struct cpuidle_state *idle_state, int index); extern void default_idle_call(void); #ifdef CONFIG_ARCH_NEEDS_CPU_IDLE_COUPLED diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index da7fbf1cdd5613..fa3b155ce7e1ab 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -560,7 +560,7 @@ static inline void cpumask_copy(struct cpumask *dstp, static inline int cpumask_parse_user(const char __user *buf, int len, struct cpumask *dstp) { - return bitmap_parse_user(buf, len, cpumask_bits(dstp), nr_cpu_ids); + return bitmap_parse_user(buf, len, cpumask_bits(dstp), nr_cpumask_bits); } /** @@ -575,7 +575,7 @@ static inline int cpumask_parselist_user(const char __user *buf, int len, struct cpumask *dstp) { return bitmap_parselist_user(buf, len, cpumask_bits(dstp), - nr_cpu_ids); + nr_cpumask_bits); } /** @@ -590,7 +590,7 @@ static inline int cpumask_parse(const char *buf, struct cpumask *dstp) char *nl = strchr(buf, '\n'); unsigned int len = nl ? (unsigned int)(nl - buf) : strlen(buf); - return bitmap_parse(buf, len, cpumask_bits(dstp), nr_cpu_ids); + return bitmap_parse(buf, len, cpumask_bits(dstp), nr_cpumask_bits); } /** @@ -602,7 +602,7 @@ static inline int cpumask_parse(const char *buf, struct cpumask *dstp) */ static inline int cpulist_parse(const char *buf, struct cpumask *dstp) { - return bitmap_parselist(buf, cpumask_bits(dstp), nr_cpu_ids); + return bitmap_parselist(buf, cpumask_bits(dstp), nr_cpumask_bits); } /** diff --git a/include/linux/dcache.h b/include/linux/dcache.h index 5beed7b3056111..9b0477ea5dd908 100644 --- a/include/linux/dcache.h +++ b/include/linux/dcache.h @@ -142,6 +142,7 @@ struct dentry_operations { int (*d_manage)(struct dentry *, bool); struct dentry *(*d_real)(struct dentry *, const struct inode *, unsigned int); + void (*d_canonical_path)(const struct path *, struct path *); } ____cacheline_aligned; /* diff --git a/include/linux/debugfs.h b/include/linux/debugfs.h index 4d3f0d1aec73b8..1b413a9aab81c6 100644 --- a/include/linux/debugfs.h +++ b/include/linux/debugfs.h @@ -62,6 +62,21 @@ static inline const struct file_operations *debugfs_real_fops(struct file *filp) return filp->f_path.dentry->d_fsdata; } +#define DEFINE_DEBUGFS_ATTRIBUTE(__fops, __get, __set, __fmt) \ +static int __fops ## _open(struct inode *inode, struct file *file) \ +{ \ + __simple_attr_check_format(__fmt, 0ull); \ + return simple_attr_open(inode, file, __get, __set, __fmt); \ +} \ +static const struct file_operations __fops = { \ + .owner = THIS_MODULE, \ + .open = __fops ## _open, \ + .release = simple_attr_release, \ + .read = debugfs_attr_read, \ + .write = debugfs_attr_write, \ + .llseek = generic_file_llseek, \ +} + #if defined(CONFIG_DEBUG_FS) struct dentry *debugfs_create_file(const char *name, umode_t mode, @@ -99,21 +114,6 @@ ssize_t debugfs_attr_read(struct file *file, char __user *buf, ssize_t debugfs_attr_write(struct file *file, const char __user *buf, size_t len, loff_t *ppos); -#define DEFINE_DEBUGFS_ATTRIBUTE(__fops, __get, __set, __fmt) \ -static int __fops ## _open(struct inode *inode, struct file *file) \ -{ \ - __simple_attr_check_format(__fmt, 0ull); \ - return simple_attr_open(inode, file, __get, __set, __fmt); \ -} \ -static const struct file_operations __fops = { \ - .owner = THIS_MODULE, \ - .open = __fops ## _open, \ - .release = simple_attr_release, \ - .read = debugfs_attr_read, \ - .write = debugfs_attr_write, \ - .llseek = generic_file_llseek, \ -} - struct dentry *debugfs_rename(struct dentry *old_dir, struct dentry *old_dentry, struct dentry *new_dir, const char *new_name); @@ -233,8 +233,18 @@ static inline void debugfs_use_file_finish(int srcu_idx) __releases(&debugfs_srcu) { } -#define DEFINE_DEBUGFS_ATTRIBUTE(__fops, __get, __set, __fmt) \ - static const struct file_operations __fops = { 0 } +static inline ssize_t debugfs_attr_read(struct file *file, char __user *buf, + size_t len, loff_t *ppos) +{ + return -ENODEV; +} + +static inline ssize_t debugfs_attr_write(struct file *file, + const char __user *buf, + size_t len, loff_t *ppos) +{ + return -ENODEV; +} static inline struct dentry *debugfs_rename(struct dentry *old_dir, struct dentry *old_dentry, struct dentry *new_dir, char *new_name) diff --git a/include/linux/device-mapper.h b/include/linux/device-mapper.h index ef7962e84444d9..0e1e050bc5741b 100644 --- a/include/linux/device-mapper.h +++ b/include/linux/device-mapper.h @@ -407,6 +407,12 @@ void dm_put(struct mapped_device *md); void dm_set_mdptr(struct mapped_device *md, void *ptr); void *dm_get_mdptr(struct mapped_device *md); +/* + * Export the device via the ioctl interface (uses mdptr). + */ +int dm_ioctl_export(struct mapped_device *md, const char *name, + const char *uuid); + /* * A device can still be used while suspended, but I/O is deferred. */ diff --git a/include/linux/efi.h b/include/linux/efi.h index 2d089487d2dacc..cba7177cbec74e 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -103,6 +103,7 @@ typedef struct { #define EFI_PAGE_SHIFT 12 #define EFI_PAGE_SIZE (1UL << EFI_PAGE_SHIFT) +#define EFI_PAGES_MAX (U64_MAX >> EFI_PAGE_SHIFT) typedef struct { u32 type; @@ -930,6 +931,7 @@ static inline efi_status_t efi_query_variable_store(u32 attributes, #endif extern void __iomem *efi_lookup_mapped_addr(u64 phys_addr); +extern phys_addr_t __init efi_memmap_alloc(unsigned int num_entries); extern int __init efi_memmap_init_early(struct efi_memory_map_data *data); extern int __init efi_memmap_init_late(phys_addr_t addr, unsigned long size); extern void __init efi_memmap_unmap(void); diff --git a/include/linux/fence.h b/include/linux/fence.h index 0d763053f97a26..8d7265fe049ec1 100644 --- a/include/linux/fence.h +++ b/include/linux/fence.h @@ -108,6 +108,7 @@ struct fence_cb { * @get_driver_name: returns the driver name. * @get_timeline_name: return the name of the context this fence belongs to. * @enable_signaling: enable software signaling of fence. + * @disable_signaling: disable software signaling of fence (optional). * @signaled: [optional] peek whether the fence is signaled, can be null. * @wait: custom wait implementation, or fence_default_wait. * @release: [optional] called on destruction of fence, can be null @@ -167,6 +168,7 @@ struct fence_ops { const char * (*get_driver_name)(struct fence *fence); const char * (*get_timeline_name)(struct fence *fence); bool (*enable_signaling)(struct fence *fence); + void (*disable_signaling)(struct fence *fence); bool (*signaled)(struct fence *fence); signed long (*wait)(struct fence *fence, bool intr, signed long timeout); void (*release)(struct fence *fence); diff --git a/include/linux/fs.h b/include/linux/fs.h index dc0478c07b2abd..bed7a849371d61 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1603,13 +1603,21 @@ extern bool inode_owner_or_capable(const struct inode *inode); * VFS helper functions.. */ extern int vfs_create(struct inode *, struct dentry *, umode_t, bool); +extern int vfs_create2(struct vfsmount *, struct inode *, struct dentry *, umode_t, bool); extern int vfs_mkdir(struct inode *, struct dentry *, umode_t); +extern int vfs_mkdir2(struct vfsmount *, struct inode *, struct dentry *, umode_t); extern int vfs_mknod(struct inode *, struct dentry *, umode_t, dev_t); +extern int vfs_mknod2(struct vfsmount *, struct inode *, struct dentry *, umode_t, dev_t); extern int vfs_symlink(struct inode *, struct dentry *, const char *); +extern int vfs_symlink2(struct vfsmount *, struct inode *, struct dentry *, const char *); extern int vfs_link(struct dentry *, struct inode *, struct dentry *, struct inode **); +extern int vfs_link2(struct vfsmount *, struct dentry *, struct inode *, struct dentry *, struct inode **); extern int vfs_rmdir(struct inode *, struct dentry *); +extern int vfs_rmdir2(struct vfsmount *, struct inode *, struct dentry *); extern int vfs_unlink(struct inode *, struct dentry *, struct inode **); +extern int vfs_unlink2(struct vfsmount *, struct inode *, struct dentry *, struct inode **); extern int vfs_rename(struct inode *, struct dentry *, struct inode *, struct dentry *, struct inode **, unsigned int); +extern int vfs_rename2(struct vfsmount *, struct inode *, struct dentry *, struct inode *, struct dentry *, struct inode **, unsigned int); extern int vfs_whiteout(struct inode *, struct dentry *); /* @@ -1737,6 +1745,7 @@ struct inode_operations { struct dentry * (*lookup) (struct inode *,struct dentry *, unsigned int); const char * (*get_link) (struct dentry *, struct inode *, struct delayed_call *); int (*permission) (struct inode *, int); + int (*permission2) (struct vfsmount *, struct inode *, int); struct posix_acl * (*get_acl)(struct inode *, int); int (*readlink) (struct dentry *, char __user *,int); @@ -1751,6 +1760,7 @@ struct inode_operations { int (*rename) (struct inode *, struct dentry *, struct inode *, struct dentry *, unsigned int); int (*setattr) (struct dentry *, struct iattr *); + int (*setattr2) (struct vfsmount *, struct dentry *, struct iattr *); int (*getattr) (struct vfsmount *mnt, struct dentry *, struct kstat *); ssize_t (*listxattr) (struct dentry *, char *, size_t); int (*fiemap)(struct inode *, struct fiemap_extent_info *, u64 start, @@ -1799,9 +1809,13 @@ struct super_operations { int (*unfreeze_fs) (struct super_block *); int (*statfs) (struct dentry *, struct kstatfs *); int (*remount_fs) (struct super_block *, int *, char *); + int (*remount_fs2) (struct vfsmount *, struct super_block *, int *, char *); + void *(*clone_mnt_data) (void *); + void (*copy_mnt_data) (void *, void *); void (*umount_begin) (struct super_block *); int (*show_options)(struct seq_file *, struct dentry *); + int (*show_options2)(struct vfsmount *,struct seq_file *, struct dentry *); int (*show_devname)(struct seq_file *, struct dentry *); int (*show_path)(struct seq_file *, struct dentry *); int (*show_stats)(struct seq_file *, struct dentry *); @@ -2035,6 +2049,9 @@ struct file_system_type { #define FS_RENAME_DOES_D_MOVE 32768 /* FS will handle d_move() during rename() internally. */ struct dentry *(*mount) (struct file_system_type *, int, const char *, void *); + struct dentry *(*mount2) (struct vfsmount *, struct file_system_type *, int, + const char *, void *); + void *(*alloc_mnt_data) (void); void (*kill_sb) (struct super_block *); struct module *owner; struct file_system_type * next; @@ -2333,6 +2350,8 @@ struct filename { extern long vfs_truncate(const struct path *, loff_t); extern int do_truncate(struct dentry *, loff_t start, unsigned int time_attrs, struct file *filp); +extern int do_truncate2(struct vfsmount *, struct dentry *, loff_t start, + unsigned int time_attrs, struct file *filp); extern int vfs_fallocate(struct file *file, int mode, loff_t offset, loff_t len); extern long do_sys_open(int dfd, const char __user *filename, int flags, @@ -2575,8 +2594,11 @@ extern void emergency_remount(void); extern sector_t bmap(struct inode *, sector_t); #endif extern int notify_change(struct dentry *, struct iattr *, struct inode **); +extern int notify_change2(struct vfsmount *, struct dentry *, struct iattr *, struct inode **); extern int inode_permission(struct inode *, int); +extern int inode_permission2(struct vfsmount *, struct inode *, int); extern int __inode_permission(struct inode *, int); +extern int __inode_permission2(struct vfsmount *, struct inode *, int); extern int generic_permission(struct inode *, int); extern int __check_sticky(struct inode *dir, struct inode *inode); diff --git a/include/linux/gpio_event.h b/include/linux/gpio_event.h new file mode 100644 index 00000000000000..2613fc5e4a93c0 --- /dev/null +++ b/include/linux/gpio_event.h @@ -0,0 +1,170 @@ +/* include/linux/gpio_event.h + * + * Copyright (C) 2007 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#ifndef _LINUX_GPIO_EVENT_H +#define _LINUX_GPIO_EVENT_H + +#include + +struct gpio_event_input_devs { + int count; + struct input_dev *dev[]; +}; +enum { + GPIO_EVENT_FUNC_UNINIT = 0x0, + GPIO_EVENT_FUNC_INIT = 0x1, + GPIO_EVENT_FUNC_SUSPEND = 0x2, + GPIO_EVENT_FUNC_RESUME = 0x3, +}; +struct gpio_event_info { + int (*func)(struct gpio_event_input_devs *input_devs, + struct gpio_event_info *info, + void **data, int func); + int (*event)(struct gpio_event_input_devs *input_devs, + struct gpio_event_info *info, + void **data, unsigned int dev, unsigned int type, + unsigned int code, int value); /* out events */ + bool no_suspend; +}; + +struct gpio_event_platform_data { + const char *name; + struct gpio_event_info **info; + size_t info_count; + int (*power)(const struct gpio_event_platform_data *pdata, bool on); + const char *names[]; /* If name is NULL, names contain a NULL */ + /* terminated list of input devices to create */ +}; + +#define GPIO_EVENT_DEV_NAME "gpio-event" + +/* Key matrix */ + +enum gpio_event_matrix_flags { + /* unset: drive active output low, set: drive active output high */ + GPIOKPF_ACTIVE_HIGH = 1U << 0, + GPIOKPF_DEBOUNCE = 1U << 1, + GPIOKPF_REMOVE_SOME_PHANTOM_KEYS = 1U << 2, + GPIOKPF_REMOVE_PHANTOM_KEYS = GPIOKPF_REMOVE_SOME_PHANTOM_KEYS | + GPIOKPF_DEBOUNCE, + GPIOKPF_DRIVE_INACTIVE = 1U << 3, + GPIOKPF_LEVEL_TRIGGERED_IRQ = 1U << 4, + GPIOKPF_PRINT_UNMAPPED_KEYS = 1U << 16, + GPIOKPF_PRINT_MAPPED_KEYS = 1U << 17, + GPIOKPF_PRINT_PHANTOM_KEYS = 1U << 18, +}; + +#define MATRIX_CODE_BITS (10) +#define MATRIX_KEY_MASK ((1U << MATRIX_CODE_BITS) - 1) +#define MATRIX_KEY(dev, code) \ + (((dev) << MATRIX_CODE_BITS) | (code & MATRIX_KEY_MASK)) + +extern int gpio_event_matrix_func(struct gpio_event_input_devs *input_devs, + struct gpio_event_info *info, void **data, int func); +struct gpio_event_matrix_info { + /* initialize to gpio_event_matrix_func */ + struct gpio_event_info info; + /* size must be ninputs * noutputs */ + const unsigned short *keymap; + unsigned int *input_gpios; + unsigned int *output_gpios; + unsigned int ninputs; + unsigned int noutputs; + /* time to wait before reading inputs after driving each output */ + ktime_t settle_time; + /* time to wait before scanning the keypad a second time */ + ktime_t debounce_delay; + ktime_t poll_time; + unsigned flags; +}; + +/* Directly connected inputs and outputs */ + +enum gpio_event_direct_flags { + GPIOEDF_ACTIVE_HIGH = 1U << 0, +/* GPIOEDF_USE_DOWN_IRQ = 1U << 1, */ +/* GPIOEDF_USE_IRQ = (1U << 2) | GPIOIDF_USE_DOWN_IRQ, */ + GPIOEDF_PRINT_KEYS = 1U << 8, + GPIOEDF_PRINT_KEY_DEBOUNCE = 1U << 9, + GPIOEDF_PRINT_KEY_UNSTABLE = 1U << 10, +}; + +struct gpio_event_direct_entry { + uint32_t gpio:16; + uint32_t code:10; + uint32_t dev:6; +}; + +/* inputs */ +extern int gpio_event_input_func(struct gpio_event_input_devs *input_devs, + struct gpio_event_info *info, void **data, int func); +struct gpio_event_input_info { + /* initialize to gpio_event_input_func */ + struct gpio_event_info info; + ktime_t debounce_time; + ktime_t poll_time; + uint16_t flags; + uint16_t type; + const struct gpio_event_direct_entry *keymap; + size_t keymap_size; +}; + +/* outputs */ +extern int gpio_event_output_func(struct gpio_event_input_devs *input_devs, + struct gpio_event_info *info, void **data, int func); +extern int gpio_event_output_event(struct gpio_event_input_devs *input_devs, + struct gpio_event_info *info, void **data, + unsigned int dev, unsigned int type, + unsigned int code, int value); +struct gpio_event_output_info { + /* initialize to gpio_event_output_func and gpio_event_output_event */ + struct gpio_event_info info; + uint16_t flags; + uint16_t type; + const struct gpio_event_direct_entry *keymap; + size_t keymap_size; +}; + + +/* axes */ + +enum gpio_event_axis_flags { + GPIOEAF_PRINT_UNKNOWN_DIRECTION = 1U << 16, + GPIOEAF_PRINT_RAW = 1U << 17, + GPIOEAF_PRINT_EVENT = 1U << 18, +}; + +extern int gpio_event_axis_func(struct gpio_event_input_devs *input_devs, + struct gpio_event_info *info, void **data, int func); +struct gpio_event_axis_info { + /* initialize to gpio_event_axis_func */ + struct gpio_event_info info; + uint8_t count; /* number of gpios for this axis */ + uint8_t dev; /* device index when using multiple input devices */ + uint8_t type; /* EV_REL or EV_ABS */ + uint16_t code; + uint16_t decoded_size; + uint16_t (*map)(struct gpio_event_axis_info *info, uint16_t in); + uint32_t *gpio; + uint32_t flags; +}; +#define gpio_axis_2bit_gray_map gpio_axis_4bit_gray_map +#define gpio_axis_3bit_gray_map gpio_axis_4bit_gray_map +uint16_t gpio_axis_4bit_gray_map( + struct gpio_event_axis_info *info, uint16_t in); +uint16_t gpio_axis_5bit_singletrack_map( + struct gpio_event_axis_info *info, uint16_t in); + +#endif diff --git a/include/linux/hyperv.h b/include/linux/hyperv.h index cd184bdca58fdc..c92a083bcf1669 100644 --- a/include/linux/hyperv.h +++ b/include/linux/hyperv.h @@ -128,6 +128,7 @@ struct hv_ring_buffer_info { u32 ring_data_startoffset; u32 priv_write_index; u32 priv_read_index; + u32 cached_read_index; }; /* @@ -180,6 +181,19 @@ static inline u32 hv_get_bytes_to_write(struct hv_ring_buffer_info *rbi) return write; } +static inline u32 hv_get_cached_bytes_to_write( + const struct hv_ring_buffer_info *rbi) +{ + u32 read_loc, write_loc, dsize, write; + + dsize = rbi->ring_datasize; + read_loc = rbi->cached_read_index; + write_loc = rbi->ring_buffer->write_index; + + write = write_loc >= read_loc ? dsize - (write_loc - read_loc) : + read_loc - write_loc; + return write; +} /* * VMBUS version is 32 bit entity broken up into * two 16 bit quantities: major_number. minor_number. @@ -1447,6 +1461,7 @@ void hv_event_tasklet_enable(struct vmbus_channel *channel); void hv_process_channel_removal(struct vmbus_channel *channel, u32 relid); +void vmbus_setevent(struct vmbus_channel *channel); /* * Negotiated version with the Host. */ @@ -1479,10 +1494,11 @@ hv_get_ring_buffer(struct hv_ring_buffer_info *ring_info) * there is room for the producer to send the pending packet. */ -static inline bool hv_need_to_signal_on_read(struct hv_ring_buffer_info *rbi) +static inline void hv_signal_on_read(struct vmbus_channel *channel) { - u32 cur_write_sz; + u32 cur_write_sz, cached_write_sz; u32 pending_sz; + struct hv_ring_buffer_info *rbi = &channel->inbound; /* * Issue a full memory barrier before making the signaling decision. @@ -1500,14 +1516,26 @@ static inline bool hv_need_to_signal_on_read(struct hv_ring_buffer_info *rbi) pending_sz = READ_ONCE(rbi->ring_buffer->pending_send_sz); /* If the other end is not blocked on write don't bother. */ if (pending_sz == 0) - return false; + return; cur_write_sz = hv_get_bytes_to_write(rbi); - if (cur_write_sz >= pending_sz) - return true; + if (cur_write_sz < pending_sz) + return; + + cached_write_sz = hv_get_cached_bytes_to_write(rbi); + if (cached_write_sz < pending_sz) + vmbus_setevent(channel); + + return; +} + +static inline void +init_cached_read_index(struct vmbus_channel *channel) +{ + struct hv_ring_buffer_info *rbi = &channel->inbound; - return false; + rbi->cached_read_index = rbi->ring_buffer->read_index; } /* @@ -1571,6 +1599,8 @@ static inline void put_pkt_raw(struct vmbus_channel *channel, * This call commits the read index and potentially signals the host. * Here is the pattern for using the "in-place" consumption APIs: * + * init_cached_read_index(); + * * while (get_next_pkt_raw() { * process the packet "in-place"; * put_pkt_raw(); @@ -1589,8 +1619,7 @@ static inline void commit_rd_index(struct vmbus_channel *channel) virt_rmb(); ring_info->ring_buffer->read_index = ring_info->priv_read_index; - if (hv_need_to_signal_on_read(ring_info)) - vmbus_set_event(channel); + hv_signal_on_read(channel); } diff --git a/include/linux/if_pppolac.h b/include/linux/if_pppolac.h new file mode 100644 index 00000000000000..e40aa1075a30e3 --- /dev/null +++ b/include/linux/if_pppolac.h @@ -0,0 +1,23 @@ +/* include/linux/if_pppolac.h + * + * Header for PPP on L2TP Access Concentrator / PPPoLAC Socket (RFC 2661) + * + * Copyright (C) 2009 Google, Inc. + * Author: Chia-chi Yeh + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef __LINUX_IF_PPPOLAC_H +#define __LINUX_IF_PPPOLAC_H + +#include + +#endif /* __LINUX_IF_PPPOLAC_H */ diff --git a/include/linux/if_pppopns.h b/include/linux/if_pppopns.h new file mode 100644 index 00000000000000..4ac621a9ce7c1d --- /dev/null +++ b/include/linux/if_pppopns.h @@ -0,0 +1,23 @@ +/* include/linux/if_pppopns.h + * + * Header for PPP on PPTP Network Server / PPPoPNS Socket (RFC 2637) + * + * Copyright (C) 2009 Google, Inc. + * Author: Chia-chi Yeh + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef __LINUX_IF_PPPOPNS_H +#define __LINUX_IF_PPPOPNS_H + +#include + +#endif /* __LINUX_IF_PPPOPNS_H */ diff --git a/include/linux/if_pppox.h b/include/linux/if_pppox.h index ba7a9b0c7c57e6..325727a7096ae0 100644 --- a/include/linux/if_pppox.h +++ b/include/linux/if_pppox.h @@ -43,6 +43,25 @@ struct pptp_opt { u32 seq_sent, seq_recv; int ppp_flags; }; + +struct pppolac_opt { + __u32 local; + __u32 remote; + __u32 recv_sequence; + __u32 xmit_sequence; + atomic_t sequencing; + int (*backlog_rcv)(struct sock *sk_udp, struct sk_buff *skb); +}; + +struct pppopns_opt { + __u16 local; + __u16 remote; + __u32 recv_sequence; + __u32 xmit_sequence; + void (*data_ready)(struct sock *sk_raw); + int (*backlog_rcv)(struct sock *sk_raw, struct sk_buff *skb); +}; + #include struct pppox_sock { @@ -53,6 +72,8 @@ struct pppox_sock { union { struct pppoe_opt pppoe; struct pptp_opt pptp; + struct pppolac_opt lac; + struct pppopns_opt pns; } proto; __be16 num; }; diff --git a/include/linux/iio/common/st_sensors.h b/include/linux/iio/common/st_sensors.h index 228bd44efa4c6e..497f2b3a5a62c8 100644 --- a/include/linux/iio/common/st_sensors.h +++ b/include/linux/iio/common/st_sensors.h @@ -115,6 +115,16 @@ struct st_sensor_bdu { u8 mask; }; +/** + * struct st_sensor_das - ST sensor device data alignment selection + * @addr: address of the register. + * @mask: mask to write the das flag for left alignment. + */ +struct st_sensor_das { + u8 addr; + u8 mask; +}; + /** * struct st_sensor_data_ready_irq - ST sensor device data-ready interrupt * @addr: address of the register. @@ -185,6 +195,7 @@ struct st_sensor_transfer_function { * @enable_axis: Enable one or more axis of the sensor. * @fs: Full scale register and full scale list available. * @bdu: Block data update register. + * @das: Data Alignment Selection register. * @drdy_irq: Data ready register of the sensor. * @multi_read_bit: Use or not particular bit for [I2C/SPI] multi-read. * @bootime: samples to discard when sensor passing from power-down to power-up. @@ -200,6 +211,7 @@ struct st_sensor_settings { struct st_sensor_axis enable_axis; struct st_sensor_fullscale fs; struct st_sensor_bdu bdu; + struct st_sensor_das das; struct st_sensor_data_ready_irq drdy_irq; bool multi_read_bit; unsigned int bootime; diff --git a/include/linux/initramfs.h b/include/linux/initramfs.h new file mode 100644 index 00000000000000..fc7da63b125b46 --- /dev/null +++ b/include/linux/initramfs.h @@ -0,0 +1,32 @@ +/* + * include/linux/initramfs.h + * + * Copyright (C) 2015, Google + * Rom Lemarchand + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef _LINUX_INITRAMFS_H +#define _LINUX_INITRAMFS_H + +#include + +#if IS_BUILTIN(CONFIG_BLK_DEV_INITRD) + +int __init default_rootfs(void); + +#endif + +#endif /* _LINUX_INITRAMFS_H */ diff --git a/include/linux/ipv6.h b/include/linux/ipv6.h index a0649973ee5b39..7aebe23acb55ea 100644 --- a/include/linux/ipv6.h +++ b/include/linux/ipv6.h @@ -40,6 +40,7 @@ struct ipv6_devconf { __s32 accept_ra_rt_info_max_plen; #endif #endif + __s32 accept_ra_rt_table; __s32 proxy_ndp; __s32 accept_source_route; __s32 accept_ra_from_local; diff --git a/include/linux/irq.h b/include/linux/irq.h index e79875574b393f..39e3254e5769d7 100644 --- a/include/linux/irq.h +++ b/include/linux/irq.h @@ -184,6 +184,7 @@ struct irq_data { * * IRQD_TRIGGER_MASK - Mask for the trigger type bits * IRQD_SETAFFINITY_PENDING - Affinity setting is pending + * IRQD_ACTIVATED - Interrupt has already been activated * IRQD_NO_BALANCING - Balancing disabled for this IRQ * IRQD_PER_CPU - Interrupt is per cpu * IRQD_AFFINITY_SET - Interrupt affinity was set @@ -202,6 +203,7 @@ struct irq_data { enum { IRQD_TRIGGER_MASK = 0xf, IRQD_SETAFFINITY_PENDING = (1 << 8), + IRQD_ACTIVATED = (1 << 9), IRQD_NO_BALANCING = (1 << 10), IRQD_PER_CPU = (1 << 11), IRQD_AFFINITY_SET = (1 << 12), @@ -312,6 +314,21 @@ static inline bool irqd_affinity_is_managed(struct irq_data *d) return __irqd_to_state(d) & IRQD_AFFINITY_MANAGED; } +static inline bool irqd_is_activated(struct irq_data *d) +{ + return __irqd_to_state(d) & IRQD_ACTIVATED; +} + +static inline void irqd_set_activated(struct irq_data *d) +{ + __irqd_to_state(d) |= IRQD_ACTIVATED; +} + +static inline void irqd_clr_activated(struct irq_data *d) +{ + __irqd_to_state(d) &= ~IRQD_ACTIVATED; +} + #undef __irqd_to_state static inline irq_hw_number_t irqd_to_hwirq(struct irq_data *d) diff --git a/include/linux/jump_label_ratelimit.h b/include/linux/jump_label_ratelimit.h index 089f70f83e97c9..23da3af459fe7a 100644 --- a/include/linux/jump_label_ratelimit.h +++ b/include/linux/jump_label_ratelimit.h @@ -14,6 +14,7 @@ struct static_key_deferred { #ifdef HAVE_JUMP_LABEL extern void static_key_slow_dec_deferred(struct static_key_deferred *key); +extern void static_key_deferred_flush(struct static_key_deferred *key); extern void jump_label_rate_limit(struct static_key_deferred *key, unsigned long rl); @@ -26,6 +27,10 @@ static inline void static_key_slow_dec_deferred(struct static_key_deferred *key) STATIC_KEY_CHECK_USE(); static_key_slow_dec(&key->key); } +static inline void static_key_deferred_flush(struct static_key_deferred *key) +{ + STATIC_KEY_CHECK_USE(); +} static inline void jump_label_rate_limit(struct static_key_deferred *key, unsigned long rl) diff --git a/include/linux/keychord.h b/include/linux/keychord.h new file mode 100644 index 00000000000000..08cf5402102ca2 --- /dev/null +++ b/include/linux/keychord.h @@ -0,0 +1,23 @@ +/* + * Key chord input driver + * + * Copyright (C) 2008 Google, Inc. + * Author: Mike Lockwood + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * +*/ + +#ifndef __LINUX_KEYCHORD_H_ +#define __LINUX_KEYCHORD_H_ + +#include + +#endif /* __LINUX_KEYCHORD_H_ */ diff --git a/include/linux/keycombo.h b/include/linux/keycombo.h new file mode 100644 index 00000000000000..c6db2626b0d3bb --- /dev/null +++ b/include/linux/keycombo.h @@ -0,0 +1,36 @@ +/* + * include/linux/keycombo.h - platform data structure for keycombo driver + * + * Copyright (C) 2014 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#ifndef _LINUX_KEYCOMBO_H +#define _LINUX_KEYCOMBO_H + +#define KEYCOMBO_NAME "keycombo" + +/* + * if key_down_fn and key_up_fn are both present, you are guaranteed that + * key_down_fn will return before key_up_fn is called, and that key_up_fn + * is called iff key_down_fn is called. + */ +struct keycombo_platform_data { + void (*key_down_fn)(void *); + void (*key_up_fn)(void *); + void *priv; + int key_down_delay; /* Time in ms */ + int *keys_up; + int keys_down[]; /* 0 terminated */ +}; + +#endif /* _LINUX_KEYCOMBO_H */ diff --git a/include/linux/keyreset.h b/include/linux/keyreset.h new file mode 100644 index 00000000000000..2e34afab65e4ae --- /dev/null +++ b/include/linux/keyreset.h @@ -0,0 +1,29 @@ +/* + * include/linux/keyreset.h - platform data structure for resetkeys driver + * + * Copyright (C) 2014 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#ifndef _LINUX_KEYRESET_H +#define _LINUX_KEYRESET_H + +#define KEYRESET_NAME "keyreset" + +struct keyreset_platform_data { + int (*reset_fn)(void); + int key_down_delay; + int *keys_up; + int keys_down[]; /* 0 terminated */ +}; + +#endif /* _LINUX_KEYRESET_H */ diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 61d20c17f3b7e4..254698856b8fc2 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -120,7 +120,7 @@ struct mem_cgroup_reclaim_iter { */ struct mem_cgroup_per_node { struct lruvec lruvec; - unsigned long lru_size[NR_LRU_LISTS]; + unsigned long lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS]; struct mem_cgroup_reclaim_iter iter[DEF_PRIORITY + 1]; @@ -432,7 +432,7 @@ static inline bool mem_cgroup_online(struct mem_cgroup *memcg) int mem_cgroup_select_victim_node(struct mem_cgroup *memcg); void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru, - int nr_pages); + int zid, int nr_pages); unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg, int nid, unsigned int lru_mask); @@ -441,9 +441,23 @@ static inline unsigned long mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru) { struct mem_cgroup_per_node *mz; + unsigned long nr_pages = 0; + int zid; mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec); - return mz->lru_size[lru]; + for (zid = 0; zid < MAX_NR_ZONES; zid++) + nr_pages += mz->lru_zone_size[zid][lru]; + return nr_pages; +} + +static inline +unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec, + enum lru_list lru, int zone_idx) +{ + struct mem_cgroup_per_node *mz; + + mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec); + return mz->lru_zone_size[zone_idx][lru]; } void mem_cgroup_handle_over_high(void); @@ -671,6 +685,12 @@ mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru) { return 0; } +static inline +unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec, + enum lru_list lru, int zone_idx) +{ + return 0; +} static inline unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg, diff --git a/include/linux/memory-state-time.h b/include/linux/memory-state-time.h new file mode 100644 index 00000000000000..d2212b02786627 --- /dev/null +++ b/include/linux/memory-state-time.h @@ -0,0 +1,42 @@ +/* include/linux/memory-state-time.h + * + * Copyright (C) 2016 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include + +#define UPDATE_MEMORY_STATE(BLOCK, VALUE) BLOCK->update_call(BLOCK, VALUE) + +struct memory_state_update_block; + +typedef void (*memory_state_update_fn_t)(struct memory_state_update_block *ub, + int value); + +/* This struct is populated when you pass it to a memory_state_register* + * function. The update_call function is used for an update and defined in the + * typedef memory_state_update_fn_t + */ +struct memory_state_update_block { + memory_state_update_fn_t update_call; + int id; +}; + +/* Register a frequency struct memory_state_update_block to provide updates to + * memory_state_time about frequency changes using its update_call function. + */ +struct memory_state_update_block *memory_state_register_frequency_source(void); + +/* Register a bandwidth struct memory_state_update_block to provide updates to + * memory_state_time about bandwidth changes using its update_call function. + */ +struct memory_state_update_block *memory_state_register_bandwidth_source(void); diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h index 01033fadea4766..134a2f69c21abf 100644 --- a/include/linux/memory_hotplug.h +++ b/include/linux/memory_hotplug.h @@ -85,7 +85,8 @@ extern int zone_grow_waitqueues(struct zone *zone, unsigned long nr_pages); extern int add_one_highpage(struct page *page, int pfn, int bad_ppro); /* VM interface that may be used by firmware interface */ extern int online_pages(unsigned long, unsigned long, int); -extern int test_pages_in_a_zone(unsigned long, unsigned long); +extern int test_pages_in_a_zone(unsigned long start_pfn, unsigned long end_pfn, + unsigned long *valid_start, unsigned long *valid_end); extern void __offline_isolated_pages(unsigned long, unsigned long); typedef void (*online_page_callback_t)(struct page *page); @@ -284,7 +285,7 @@ extern void sparse_remove_one_section(struct zone *zone, struct mem_section *ms, unsigned long map_offset); extern struct page *sparse_decode_mem_map(unsigned long coded_mem_map, unsigned long pnum); -extern int zone_can_shift(unsigned long pfn, unsigned long nr_pages, - enum zone_type target); +extern bool zone_can_shift(unsigned long pfn, unsigned long nr_pages, + enum zone_type target, int *zone_shift); #endif /* __LINUX_MEMORY_HOTPLUG_H */ diff --git a/include/linux/mm.h b/include/linux/mm.h index a92c8d73aeafc5..2d191bf833907c 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1138,6 +1138,7 @@ extern void pagefault_out_of_memory(void); extern void show_free_areas(unsigned int flags); extern bool skip_free_areas_node(unsigned int flags, int nid); +void shmem_set_file(struct vm_area_struct *vma, struct file *file); int shmem_zero_setup(struct vm_area_struct *); #ifdef CONFIG_SHMEM bool shmem_mapping(struct address_space *mapping); @@ -1270,6 +1271,8 @@ extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void * unsigned int gup_flags); extern int access_remote_vm(struct mm_struct *mm, unsigned long addr, void *buf, int len, unsigned int gup_flags); +extern int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm, + unsigned long addr, void *buf, int len, unsigned int gup_flags); long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm, unsigned long start, unsigned long nr_pages, @@ -1974,7 +1977,7 @@ static inline int vma_adjust(struct vm_area_struct *vma, unsigned long start, extern struct vm_area_struct *vma_merge(struct mm_struct *, struct vm_area_struct *prev, unsigned long addr, unsigned long end, unsigned long vm_flags, struct anon_vma *, struct file *, pgoff_t, - struct mempolicy *, struct vm_userfaultfd_ctx); + struct mempolicy *, struct vm_userfaultfd_ctx, const char __user *); extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *); extern int split_vma(struct mm_struct *, struct vm_area_struct *, unsigned long addr, int new_below); diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index 71613e8a720f99..41d376e7116dcc 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -39,7 +39,7 @@ static __always_inline void update_lru_size(struct lruvec *lruvec, { __update_lru_size(lruvec, lru, zid, nr_pages); #ifdef CONFIG_MEMCG - mem_cgroup_update_lru_size(lruvec, lru, nr_pages); + mem_cgroup_update_lru_size(lruvec, lru, zid, nr_pages); #endif } diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 4a8acedf4b7d91..3139ea45b8f45e 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -326,11 +326,18 @@ struct vm_area_struct { /* * For areas with an address space and backing store, * linkage into the address_space->i_mmap interval tree. + * + * For private anonymous mappings, a pointer to a null terminated string + * in the user process containing the name given to the vma, or NULL + * if unnamed. */ - struct { - struct rb_node rb; - unsigned long rb_subtree_last; - } shared; + union { + struct { + struct rb_node rb; + unsigned long rb_subtree_last; + } shared; + const char __user *anon_name; + }; /* * A file's MAP_PRIVATE vma can be in both i_mmap tree and anon_vma @@ -473,6 +480,7 @@ struct mm_struct { */ struct task_struct __rcu *owner; #endif + struct user_namespace *user_ns; /* store ref to file /proc//exe symlink points to */ struct file __rcu *exe_file; @@ -615,4 +623,13 @@ typedef struct { unsigned long val; } swp_entry_t; +/* Return the name for an anonymous mapping or NULL for a file-backed mapping */ +static inline const char __user *vma_get_anon_name(struct vm_area_struct *vma) +{ + if (vma->vm_file) + return NULL; + + return vma->anon_name; +} + #endif /* _LINUX_MM_TYPES_H */ diff --git a/include/linux/mmc/core.h b/include/linux/mmc/core.h index 2b953eb8ceae19..46a4b798c7cf2c 100644 --- a/include/linux/mmc/core.h +++ b/include/linux/mmc/core.h @@ -142,6 +142,10 @@ struct mmc_request { /* Allow other commands during this ongoing data transfer or busy wait */ bool cap_cmd_during_tfr; + ktime_t io_start; +#ifdef CONFIG_BLOCK + int lat_hist_enabled; +#endif }; struct mmc_card; diff --git a/include/linux/mmc/host.h b/include/linux/mmc/host.h index 0b2439441cc8cc..fac3b5c27f4ff8 100644 --- a/include/linux/mmc/host.h +++ b/include/linux/mmc/host.h @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -397,6 +398,20 @@ struct mmc_host { int dsr_req; /* DSR value is valid */ u32 dsr; /* optional driver stage (DSR) value */ +#ifdef CONFIG_MMC_EMBEDDED_SDIO + struct { + struct sdio_cis *cis; + struct sdio_cccr *cccr; + struct sdio_embedded_func *funcs; + int num_funcs; + } embedded_sdio_data; +#endif + +#ifdef CONFIG_BLOCK + int latency_hist_enabled; + struct io_latency_state io_lat_s; +#endif + unsigned long private[0] ____cacheline_aligned; }; @@ -406,6 +421,14 @@ void mmc_remove_host(struct mmc_host *); void mmc_free_host(struct mmc_host *); int mmc_of_parse(struct mmc_host *host); +#ifdef CONFIG_MMC_EMBEDDED_SDIO +extern void mmc_set_embedded_sdio_data(struct mmc_host *host, + struct sdio_cis *cis, + struct sdio_cccr *cccr, + struct sdio_embedded_func *funcs, + int num_funcs); +#endif + static inline void *mmc_priv(struct mmc_host *host) { return (void *)host->private; diff --git a/include/linux/mmc/pm.h b/include/linux/mmc/pm.h index 4a139204c20c0b..6e2d6a135c7e0d 100644 --- a/include/linux/mmc/pm.h +++ b/include/linux/mmc/pm.h @@ -26,5 +26,6 @@ typedef unsigned int mmc_pm_flag_t; #define MMC_PM_KEEP_POWER (1 << 0) /* preserve card power during suspend */ #define MMC_PM_WAKE_SDIO_IRQ (1 << 1) /* wake up host system on SDIO IRQ assertion */ +#define MMC_PM_IGNORE_PM_NOTIFY (1 << 2) /* ignore mmc pm notify */ #endif /* LINUX_MMC_PM_H */ diff --git a/include/linux/mmc/sdio_func.h b/include/linux/mmc/sdio_func.h index aab032a6ae6124..d0a69e71b8abac 100644 --- a/include/linux/mmc/sdio_func.h +++ b/include/linux/mmc/sdio_func.h @@ -22,6 +22,14 @@ struct sdio_func; typedef void (sdio_irq_handler_t)(struct sdio_func *); +/* + * Structure used to hold embedded SDIO device data from platform layer + */ +struct sdio_embedded_func { + uint8_t f_class; + uint32_t f_maxblksize; +}; + /* * SDIO function CIS tuple (unknown to the core) */ @@ -128,6 +136,8 @@ extern int sdio_release_irq(struct sdio_func *func); extern unsigned int sdio_align_size(struct sdio_func *func, unsigned int sz); extern u8 sdio_readb(struct sdio_func *func, unsigned int addr, int *err_ret); +extern u8 sdio_readb_ext(struct sdio_func *func, unsigned int addr, int *err_ret, + unsigned in); extern u16 sdio_readw(struct sdio_func *func, unsigned int addr, int *err_ret); extern u32 sdio_readl(struct sdio_func *func, unsigned int addr, int *err_ret); diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 0f088f3a2fed8d..f99c993dd50083 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -972,12 +972,16 @@ static __always_inline struct zoneref *next_zones_zonelist(struct zoneref *z, * @zonelist - The zonelist to search for a suitable zone * @highest_zoneidx - The zone index of the highest zone to return * @nodes - An optional nodemask to filter the zonelist with - * @zone - The first suitable zone found is returned via this parameter + * @return - Zoneref pointer for the first suitable zone found (see below) * * This function returns the first zone at or below a given zone index that is * within the allowed nodemask. The zoneref returned is a cursor that can be * used to iterate the zonelist with next_zones_zonelist by advancing it by * one before calling. + * + * When no eligible zone is found, zoneref->zone is NULL (zoneref itself is + * never NULL). This may happen either genuinely, or due to concurrent nodemask + * update due to cpuset modification. */ static inline struct zoneref *first_zones_zonelist(struct zonelist *zonelist, enum zone_type highest_zoneidx, diff --git a/include/linux/mount.h b/include/linux/mount.h index 1172cce949a4c3..4f333413934a15 100644 --- a/include/linux/mount.h +++ b/include/linux/mount.h @@ -67,6 +67,7 @@ struct vfsmount { struct dentry *mnt_root; /* root of the mounted tree */ struct super_block *mnt_sb; /* pointer to superblock */ int mnt_flags; + void *data; }; struct file; /* forward dec */ diff --git a/include/linux/namei.h b/include/linux/namei.h index f29abda31e6dc8..cf437f56baf4aa 100644 --- a/include/linux/namei.h +++ b/include/linux/namei.h @@ -78,8 +78,11 @@ extern struct dentry *user_path_create(int, const char __user *, struct path *, extern void done_path_create(struct path *, struct dentry *); extern struct dentry *kern_path_locked(const char *, struct path *); extern int kern_path_mountpoint(int, const char *, struct path *, unsigned int); +extern int vfs_path_lookup(struct dentry *, struct vfsmount *, + const char *, unsigned int, struct path *); extern struct dentry *lookup_one_len(const char *, struct dentry *, int); +extern struct dentry *lookup_one_len2(const char *, struct vfsmount *mnt, struct dentry *, int); extern struct dentry *lookup_one_len_unlocked(const char *, struct dentry *, int); extern int follow_down_one(struct path *); diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index e16a2a980ea8d8..d83590ef74a15e 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -2502,14 +2502,19 @@ static inline int skb_gro_header_hard(struct sk_buff *skb, unsigned int hlen) return NAPI_GRO_CB(skb)->frag0_len < hlen; } +static inline void skb_gro_frag0_invalidate(struct sk_buff *skb) +{ + NAPI_GRO_CB(skb)->frag0 = NULL; + NAPI_GRO_CB(skb)->frag0_len = 0; +} + static inline void *skb_gro_header_slow(struct sk_buff *skb, unsigned int hlen, unsigned int offset) { if (!pskb_may_pull(skb, hlen)) return NULL; - NAPI_GRO_CB(skb)->frag0 = NULL; - NAPI_GRO_CB(skb)->frag0_len = 0; + skb_gro_frag0_invalidate(skb); return skb->data + offset; } diff --git a/include/linux/netfilter/xt_qtaguid.h b/include/linux/netfilter/xt_qtaguid.h new file mode 100644 index 00000000000000..ca60fbdec2f3b0 --- /dev/null +++ b/include/linux/netfilter/xt_qtaguid.h @@ -0,0 +1,13 @@ +#ifndef _XT_QTAGUID_MATCH_H +#define _XT_QTAGUID_MATCH_H + +/* For now we just replace the xt_owner. + * FIXME: make iptables aware of qtaguid. */ +#include + +#define XT_QTAGUID_UID XT_OWNER_UID +#define XT_QTAGUID_GID XT_OWNER_GID +#define XT_QTAGUID_SOCKET XT_OWNER_SOCKET +#define xt_qtaguid_match_info xt_owner_match_info + +#endif /* _XT_QTAGUID_MATCH_H */ diff --git a/include/linux/netfilter/xt_quota2.h b/include/linux/netfilter/xt_quota2.h new file mode 100644 index 00000000000000..eadc6903314e77 --- /dev/null +++ b/include/linux/netfilter/xt_quota2.h @@ -0,0 +1,25 @@ +#ifndef _XT_QUOTA_H +#define _XT_QUOTA_H + +enum xt_quota_flags { + XT_QUOTA_INVERT = 1 << 0, + XT_QUOTA_GROW = 1 << 1, + XT_QUOTA_PACKET = 1 << 2, + XT_QUOTA_NO_CHANGE = 1 << 3, + XT_QUOTA_MASK = 0x0F, +}; + +struct xt_quota_counter; + +struct xt_quota_mtinfo2 { + char name[15]; + u_int8_t flags; + + /* Comparison-invariant */ + aligned_u64 quota; + + /* Used internally by the kernel */ + struct xt_quota_counter *master __attribute__((aligned(8))); +}; + +#endif /* _XT_QUOTA_H */ diff --git a/include/linux/nfs4.h b/include/linux/nfs4.h index 9094faf0699d61..039e76e918965e 100644 --- a/include/linux/nfs4.h +++ b/include/linux/nfs4.h @@ -282,7 +282,7 @@ enum nfsstat4 { static inline bool seqid_mutating_err(u32 err) { - /* rfc 3530 section 8.1.5: */ + /* See RFC 7530, section 9.1.7 */ switch (err) { case NFS4ERR_STALE_CLIENTID: case NFS4ERR_STALE_STATEID: @@ -291,6 +291,7 @@ static inline bool seqid_mutating_err(u32 err) case NFS4ERR_BADXDR: case NFS4ERR_RESOURCE: case NFS4ERR_NOFILEHANDLE: + case NFS4ERR_MOVED: return false; }; return true; diff --git a/include/linux/nmi.h b/include/linux/nmi.h index a78c35cff1ae34..780949dff0edbf 100644 --- a/include/linux/nmi.h +++ b/include/linux/nmi.h @@ -14,8 +14,11 @@ * may be used to reset the timeout - for code which intentionally * disables interrupts for a long time. This call is stateless. */ -#if defined(CONFIG_HAVE_NMI_WATCHDOG) || defined(CONFIG_HARDLOCKUP_DETECTOR) +#if defined(CONFIG_HAVE_NMI_WATCHDOG) || defined(CONFIG_HARDLOCKUP_DETECTOR_NMI) #include +#endif + +#if defined(CONFIG_HAVE_NMI_WATCHDOG) || defined(CONFIG_HARDLOCKUP_DETECTOR) extern void touch_nmi_watchdog(void); #else static inline void touch_nmi_watchdog(void) diff --git a/include/linux/of_fdt.h b/include/linux/of_fdt.h index 4341f32516d8a1..501d461a6a1d4e 100644 --- a/include/linux/of_fdt.h +++ b/include/linux/of_fdt.h @@ -63,6 +63,27 @@ extern int of_flat_dt_match(unsigned long node, const char *const *matches); extern unsigned long of_get_flat_dt_root(void); extern int of_get_flat_dt_size(void); +/* + * early_init_dt_scan_chosen - scan the device tree for ramdisk and bootargs + * + * The boot arguments will be placed into the memory pointed to by @data. + * That memory should be COMMAND_LINE_SIZE big and initialized to be a valid + * (possibly empty) string. Logic for what will be in @data after this + * function finishes: + * + * - CONFIG_CMDLINE_FORCE=true + * CONFIG_CMDLINE + * - CONFIG_CMDLINE_EXTEND=true, @data is non-empty string + * @data + dt bootargs (even if dt bootargs are empty) + * - CONFIG_CMDLINE_EXTEND=true, @data is empty string + * CONFIG_CMDLINE + dt bootargs (even if dt bootargs are empty) + * - CMDLINE_FROM_BOOTLOADER=true, dt bootargs=non-empty: + * dt bootargs + * - CMDLINE_FROM_BOOTLOADER=true, dt bootargs=empty, @data is non-empty string + * @data is left unchanged + * - CMDLINE_FROM_BOOTLOADER=true, dt bootargs=empty, @data is empty string + * CONFIG_CMDLINE (or "" if that's not defined) + */ extern int early_init_dt_scan_chosen(unsigned long node, const char *uname, int depth, void *data); extern int early_init_dt_scan_memory(unsigned long node, const char *uname, diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h index c58752fe16c431..f020ab4079d3e4 100644 --- a/include/linux/pci_ids.h +++ b/include/linux/pci_ids.h @@ -2256,12 +2256,29 @@ #define PCI_DEVICE_ID_ZOLTRIX_2BD0 0x2bd0 #define PCI_VENDOR_ID_MELLANOX 0x15b3 -#define PCI_DEVICE_ID_MELLANOX_TAVOR 0x5a44 +#define PCI_DEVICE_ID_MELLANOX_CONNECTX3 0x1003 +#define PCI_DEVICE_ID_MELLANOX_CONNECTX3_PRO 0x1007 +#define PCI_DEVICE_ID_MELLANOX_CONNECTIB 0x1011 +#define PCI_DEVICE_ID_MELLANOX_CONNECTX4 0x1013 +#define PCI_DEVICE_ID_MELLANOX_CONNECTX4_LX 0x1015 +#define PCI_DEVICE_ID_MELLANOX_TAVOR 0x5a44 #define PCI_DEVICE_ID_MELLANOX_TAVOR_BRIDGE 0x5a46 -#define PCI_DEVICE_ID_MELLANOX_ARBEL_COMPAT 0x6278 -#define PCI_DEVICE_ID_MELLANOX_ARBEL 0x6282 -#define PCI_DEVICE_ID_MELLANOX_SINAI_OLD 0x5e8c -#define PCI_DEVICE_ID_MELLANOX_SINAI 0x6274 +#define PCI_DEVICE_ID_MELLANOX_SINAI_OLD 0x5e8c +#define PCI_DEVICE_ID_MELLANOX_SINAI 0x6274 +#define PCI_DEVICE_ID_MELLANOX_ARBEL_COMPAT 0x6278 +#define PCI_DEVICE_ID_MELLANOX_ARBEL 0x6282 +#define PCI_DEVICE_ID_MELLANOX_HERMON_SDR 0x6340 +#define PCI_DEVICE_ID_MELLANOX_HERMON_DDR 0x634a +#define PCI_DEVICE_ID_MELLANOX_HERMON_QDR 0x6354 +#define PCI_DEVICE_ID_MELLANOX_HERMON_EN 0x6368 +#define PCI_DEVICE_ID_MELLANOX_CONNECTX_EN 0x6372 +#define PCI_DEVICE_ID_MELLANOX_HERMON_DDR_GEN2 0x6732 +#define PCI_DEVICE_ID_MELLANOX_HERMON_QDR_GEN2 0x673c +#define PCI_DEVICE_ID_MELLANOX_CONNECTX_EN_5_GEN2 0x6746 +#define PCI_DEVICE_ID_MELLANOX_HERMON_EN_GEN2 0x6750 +#define PCI_DEVICE_ID_MELLANOX_CONNECTX_EN_T_GEN2 0x675a +#define PCI_DEVICE_ID_MELLANOX_CONNECTX_EN_GEN2 0x6764 +#define PCI_DEVICE_ID_MELLANOX_CONNECTX2 0x676e #define PCI_VENDOR_ID_DFI 0x15bd diff --git a/include/linux/percpu-refcount.h b/include/linux/percpu-refcount.h index 1c7eec09e5eba7..3a481a49546ef1 100644 --- a/include/linux/percpu-refcount.h +++ b/include/linux/percpu-refcount.h @@ -204,7 +204,7 @@ static inline void percpu_ref_get(struct percpu_ref *ref) static inline bool percpu_ref_tryget(struct percpu_ref *ref) { unsigned long __percpu *percpu_count; - int ret; + bool ret; rcu_read_lock_sched(); @@ -238,7 +238,7 @@ static inline bool percpu_ref_tryget(struct percpu_ref *ref) static inline bool percpu_ref_tryget_live(struct percpu_ref *ref) { unsigned long __percpu *percpu_count; - int ret = false; + bool ret = false; rcu_read_lock_sched(); diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 4741ecdb981743..531b8b10fcdaff 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -1168,6 +1168,11 @@ extern int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write, int perf_event_max_stack_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos); +static inline bool perf_paranoid_any(void) +{ + return sysctl_perf_event_paranoid > 2; +} + static inline bool perf_paranoid_tracepoint_raw(void) { return sysctl_perf_event_paranoid > -1; diff --git a/include/linux/platform_data/ds2482.h b/include/linux/platform_data/ds2482.h new file mode 100644 index 00000000000000..5a6879e2a09ab3 --- /dev/null +++ b/include/linux/platform_data/ds2482.h @@ -0,0 +1,21 @@ +/* + * Copyright (C) 2012 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef __PLATFORM_DATA_DS2482__ +#define __PLATFORM_DATA_DS2482__ + +struct ds2482_platform_data { + int slpz_gpio; +}; + +#endif /* __PLATFORM_DATA_DS2482__ */ diff --git a/include/linux/pm_opp.h b/include/linux/pm_opp.h index bca26157f5b695..f6bc7650191281 100644 --- a/include/linux/pm_opp.h +++ b/include/linux/pm_opp.h @@ -19,6 +19,7 @@ struct dev_pm_opp; struct device; +struct opp_table; enum dev_pm_opp_event { OPP_EVENT_ADD, OPP_EVENT_REMOVE, OPP_EVENT_ENABLE, OPP_EVENT_DISABLE, @@ -62,8 +63,8 @@ int dev_pm_opp_set_supported_hw(struct device *dev, const u32 *versions, void dev_pm_opp_put_supported_hw(struct device *dev); int dev_pm_opp_set_prop_name(struct device *dev, const char *name); void dev_pm_opp_put_prop_name(struct device *dev); -int dev_pm_opp_set_regulator(struct device *dev, const char *name); -void dev_pm_opp_put_regulator(struct device *dev); +struct opp_table *dev_pm_opp_set_regulator(struct device *dev, const char *name); +void dev_pm_opp_put_regulator(struct opp_table *opp_table); int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq); int dev_pm_opp_set_sharing_cpus(struct device *cpu_dev, const struct cpumask *cpumask); int dev_pm_opp_get_sharing_cpus(struct device *cpu_dev, struct cpumask *cpumask); @@ -170,12 +171,12 @@ static inline int dev_pm_opp_set_prop_name(struct device *dev, const char *name) static inline void dev_pm_opp_put_prop_name(struct device *dev) {} -static inline int dev_pm_opp_set_regulator(struct device *dev, const char *name) +static inline struct opp_table *dev_pm_opp_set_regulator(struct device *dev, const char *name) { - return -ENOTSUPP; + return ERR_PTR(-ENOTSUPP); } -static inline void dev_pm_opp_put_regulator(struct device *dev) {} +static inline void dev_pm_opp_put_regulator(struct opp_table *opp_table) {} static inline int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq) { diff --git a/include/linux/power/bq27xxx_battery.h b/include/linux/power/bq27xxx_battery.h index e30deb04615618..bed9557b69e741 100644 --- a/include/linux/power/bq27xxx_battery.h +++ b/include/linux/power/bq27xxx_battery.h @@ -4,7 +4,8 @@ enum bq27xxx_chip { BQ27000 = 1, /* bq27000, bq27200 */ BQ27010, /* bq27010, bq27210 */ - BQ27500, /* bq27500, bq27510, bq27520 */ + BQ27500, /* bq27500 */ + BQ27510, /* bq27510, bq27520 */ BQ27530, /* bq27530, bq27531 */ BQ27541, /* bq27541, bq27542, bq27546, bq27742 */ BQ27545, /* bq27545 */ diff --git a/include/linux/power_supply.h b/include/linux/power_supply.h index 3965503315ef48..ecfb4cac973cd2 100644 --- a/include/linux/power_supply.h +++ b/include/linux/power_supply.h @@ -18,6 +18,7 @@ #include #include #include +#include /* * All voltages, currents, charges, energies, time and temperatures in uV, @@ -148,6 +149,12 @@ enum power_supply_property { POWER_SUPPLY_PROP_SCOPE, POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT, POWER_SUPPLY_PROP_CALIBRATE, + /* Local extensions */ + POWER_SUPPLY_PROP_USB_HC, + POWER_SUPPLY_PROP_USB_OTG, + POWER_SUPPLY_PROP_CHARGE_ENABLED, + /* Local extensions of type int64_t */ + POWER_SUPPLY_PROP_CHARGE_COUNTER_EXT, /* Properties of type `const char *' */ POWER_SUPPLY_PROP_MODEL_NAME, POWER_SUPPLY_PROP_MANUFACTURER, @@ -175,6 +182,7 @@ enum power_supply_notifier_events { union power_supply_propval { int intval; const char *strval; + int64_t int64val; }; struct device_node; diff --git a/include/linux/pstore_ram.h b/include/linux/pstore_ram.h index c668c861c96ced..485cc8e3f7dd4e 100644 --- a/include/linux/pstore_ram.h +++ b/include/linux/pstore_ram.h @@ -71,6 +71,8 @@ void persistent_ram_free_old(struct persistent_ram_zone *prz); ssize_t persistent_ram_ecc_string(struct persistent_ram_zone *prz, char *str, size_t len); +void ramoops_console_write_buf(const char *buf, size_t size); + /* * Ramoops platform data * @mem_size memory size for ramoops diff --git a/include/linux/ptrace.h b/include/linux/ptrace.h index 504c98a278d466..e0e539321ab95c 100644 --- a/include/linux/ptrace.h +++ b/include/linux/ptrace.h @@ -8,6 +8,9 @@ #include /* For task_active_pid_ns. */ #include +extern int ptrace_access_vm(struct task_struct *tsk, unsigned long addr, + void *buf, int len, unsigned int gup_flags); + /* * Ptrace flags * @@ -19,7 +22,6 @@ #define PT_SEIZED 0x00010000 /* SEIZE used, enable new behavior */ #define PT_PTRACED 0x00000001 #define PT_DTRACE 0x00000002 /* delayed trace (used on m68k, i386) */ -#define PT_PTRACE_CAP 0x00000004 /* ptracer can follow suid-exec */ #define PT_OPT_FLAG_SHIFT 3 /* PT_TRACE_* event enable flags */ diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 321f9ed552a995..01f71e1d2e941e 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -444,6 +444,10 @@ bool __rcu_is_watching(void); #error "Unknown RCU implementation specified to kernel configuration" #endif +#define RCU_SCHEDULER_INACTIVE 0 +#define RCU_SCHEDULER_INIT 1 +#define RCU_SCHEDULER_RUNNING 2 + /* * init_rcu_head_on_stack()/destroy_rcu_head_on_stack() are needed for dynamic * initialization and destruction of rcu_head on the stack. rcu_head structures diff --git a/include/linux/sched.h b/include/linux/sched.h index e9c009dc3a4a35..4b5a26dfa54e3e 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -173,6 +173,9 @@ extern bool single_task_running(void); extern unsigned long nr_iowait(void); extern unsigned long nr_iowait_cpu(int cpu); extern void get_iowait_load(unsigned long *nr_waiters, unsigned long *load); +#ifdef CONFIG_CPU_QUIET +extern u64 nr_running_integral(unsigned int cpu); +#endif extern void calc_global_load(unsigned long ticks); @@ -315,6 +318,15 @@ extern char ___assert_task_state[1 - 2*!!( /* Task command name length */ #define TASK_COMM_LEN 16 +enum task_event { + PUT_PREV_TASK = 0, + PICK_NEXT_TASK = 1, + TASK_WAKE = 2, + TASK_MIGRATE = 3, + TASK_UPDATE = 4, + IRQ_UPDATE = 5, +}; + #include /* @@ -972,6 +984,14 @@ enum cpu_idle_type { #define SCHED_CAPACITY_SHIFT SCHED_FIXEDPOINT_SHIFT #define SCHED_CAPACITY_SCALE (1L << SCHED_CAPACITY_SHIFT) +struct sched_capacity_reqs { + unsigned long cfs; + unsigned long rt; + unsigned long dl; + + unsigned long total; +}; + /* * Wake-queues are lists of tasks with a pending wakeup, whose * callers have already marked the task as woken internally, @@ -1035,6 +1055,7 @@ extern void wake_up_q(struct wake_q_head *head); #define SD_PREFER_SIBLING 0x1000 /* Prefer to place tasks in a sibling domain */ #define SD_OVERLAP 0x2000 /* sched_domains of this level overlap */ #define SD_NUMA 0x4000 /* cross-node balancing */ +#define SD_SHARE_CAP_STATES 0x8000 /* Domain members share capacity state */ #ifdef CONFIG_SCHED_SMT static inline int cpu_smt_flags(void) @@ -1067,6 +1088,24 @@ struct sched_domain_attr { extern int sched_domain_level_max; +struct capacity_state { + unsigned long cap; /* compute capacity */ + unsigned long power; /* power consumption at this compute capacity */ +}; + +struct idle_state { + unsigned long power; /* power consumption in this idle state */ +}; + +struct sched_group_energy { + unsigned int nr_idle_states; /* number of idle states */ + struct idle_state *idle_states; /* ptr to idle state array */ + unsigned int nr_cap_states; /* number of capacity states */ + struct capacity_state *cap_states; /* ptr to capacity state array */ +}; + +unsigned long capacity_curr_of(int cpu); + struct sched_group; struct sched_domain_shared { @@ -1174,6 +1213,8 @@ bool cpus_share_cache(int this_cpu, int that_cpu); typedef const struct cpumask *(*sched_domain_mask_f)(int cpu); typedef int (*sched_domain_flags_f)(void); +typedef +const struct sched_group_energy * const(*sched_domain_energy_f)(int cpu); #define SDTL_OVERLAP 0x01 @@ -1187,6 +1228,7 @@ struct sd_data { struct sched_domain_topology_level { sched_domain_mask_f mask; sched_domain_flags_f sd_flags; + sched_domain_energy_f energy; int flags; int numa_level; struct sd_data data; @@ -1335,6 +1377,41 @@ struct sched_statistics { }; #endif +#ifdef CONFIG_SCHED_WALT +#define RAVG_HIST_SIZE_MAX 5 + +/* ravg represents frequency scaled cpu-demand of tasks */ +struct ravg { + /* + * 'mark_start' marks the beginning of an event (task waking up, task + * starting to execute, task being preempted) within a window + * + * 'sum' represents how runnable a task has been within current + * window. It incorporates both running time and wait time and is + * frequency scaled. + * + * 'sum_history' keeps track of history of 'sum' seen over previous + * RAVG_HIST_SIZE windows. Windows where task was entirely sleeping are + * ignored. + * + * 'demand' represents maximum sum seen over previous + * sysctl_sched_ravg_hist_size windows. 'demand' could drive frequency + * demand for tasks. + * + * 'curr_window' represents task's contribution to cpu busy time + * statistics (rq->curr_runnable_sum) in current window + * + * 'prev_window' represents task's contribution to cpu busy time + * statistics (rq->prev_runnable_sum) in previous window + */ + u64 mark_start; + u32 sum, demand; + u32 sum_history[RAVG_HIST_SIZE_MAX]; + u32 curr_window, prev_window; + u16 active_windows; +}; +#endif + struct sched_entity { struct load_weight load; /* for load-balancing */ struct rb_node run_node; @@ -1505,6 +1582,15 @@ struct task_struct { const struct sched_class *sched_class; struct sched_entity se; struct sched_rt_entity rt; +#ifdef CONFIG_SCHED_WALT + struct ravg ravg; + /* + * 'init_load_pct' represents the initial task load assigned to children + * of this task + */ + u32 init_load_pct; +#endif + #ifdef CONFIG_CGROUP_SCHED struct task_group *sched_task_group; #endif @@ -1656,6 +1742,7 @@ struct task_struct { struct list_head cpu_timers[3]; /* process credentials */ + const struct cred __rcu *ptracer_cred; /* Tracer's credentials at attach */ const struct cred __rcu *real_cred; /* objective and real subjective task * credentials (COW) */ const struct cred __rcu *cred; /* effective (overridable) subjective task diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index 22db1e63707ec2..264a2208d73792 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h @@ -18,6 +18,16 @@ extern unsigned int sysctl_sched_latency; extern unsigned int sysctl_sched_min_granularity; extern unsigned int sysctl_sched_wakeup_granularity; extern unsigned int sysctl_sched_child_runs_first; +extern unsigned int sysctl_sched_is_big_little; +extern unsigned int sysctl_sched_sync_hint_enable; +extern unsigned int sysctl_sched_initial_task_util; +extern unsigned int sysctl_sched_cstate_aware; +#ifdef CONFIG_SCHED_WALT +extern unsigned int sysctl_sched_use_walt_cpu_util; +extern unsigned int sysctl_sched_use_walt_task_util; +extern unsigned int sysctl_sched_walt_init_task_load_pct; +extern unsigned int sysctl_sched_walt_cpu_high_irqload; +#endif enum sched_tunable_scaling { SCHED_TUNABLESCALING_NONE, @@ -56,6 +66,22 @@ extern int sysctl_sched_rt_runtime; extern unsigned int sysctl_sched_cfs_bandwidth_slice; #endif +#ifdef CONFIG_SCHED_TUNE +extern unsigned int sysctl_sched_cfs_boost; +int sysctl_sched_cfs_boost_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *length, + loff_t *ppos); +static inline unsigned int get_sysctl_sched_cfs_boost(void) +{ + return sysctl_sched_cfs_boost; +} +#else +static inline unsigned int get_sysctl_sched_cfs_boost(void) +{ + return 0; +} +#endif + #ifdef CONFIG_SCHED_AUTOGROUP extern unsigned int sysctl_sched_autogroup_enabled; #endif diff --git a/include/linux/sched_energy.h b/include/linux/sched_energy.h new file mode 100644 index 00000000000000..1daf3e1f98a75c --- /dev/null +++ b/include/linux/sched_energy.h @@ -0,0 +1,44 @@ +#ifndef _LINUX_SCHED_ENERGY_H +#define _LINUX_SCHED_ENERGY_H + +#include +#include + +/* + * There doesn't seem to be an NR_CPUS style max number of sched domain + * levels so here's an arbitrary constant one for the moment. + * + * The levels alluded to here correspond to entries in struct + * sched_domain_topology_level that are meant to be populated by arch + * specific code (topology.c). + */ +#define NR_SD_LEVELS 8 + +#define SD_LEVEL0 0 +#define SD_LEVEL1 1 +#define SD_LEVEL2 2 +#define SD_LEVEL3 3 +#define SD_LEVEL4 4 +#define SD_LEVEL5 5 +#define SD_LEVEL6 6 +#define SD_LEVEL7 7 + +/* + * Convenience macro for iterating through said sd levels. + */ +#define for_each_possible_sd_level(level) \ + for (level = 0; level < NR_SD_LEVELS; level++) + +#ifdef CONFIG_SMP + +extern struct sched_group_energy *sge_array[NR_CPUS][NR_SD_LEVELS]; + +void init_sched_energy_costs(void); + +#else + +#define init_sched_energy_costs() do { } while (0) + +#endif /* CONFIG_SMP */ + +#endif diff --git a/include/linux/serial_core.h b/include/linux/serial_core.h index 344201437017e9..1a943975f09ce7 100644 --- a/include/linux/serial_core.h +++ b/include/linux/serial_core.h @@ -66,6 +66,7 @@ struct uart_ops { void (*set_ldisc)(struct uart_port *, struct ktermios *); void (*pm)(struct uart_port *, unsigned int state, unsigned int oldstate); + void (*wake_peer)(struct uart_port *); /* * Return a string describing the type of the port diff --git a/include/linux/sunrpc/clnt.h b/include/linux/sunrpc/clnt.h index 85cc819676e81d..333ad11b3dd9cf 100644 --- a/include/linux/sunrpc/clnt.h +++ b/include/linux/sunrpc/clnt.h @@ -216,5 +216,6 @@ void rpc_clnt_xprt_switch_put(struct rpc_clnt *); void rpc_clnt_xprt_switch_add_xprt(struct rpc_clnt *, struct rpc_xprt *); bool rpc_clnt_xprt_switch_has_addr(struct rpc_clnt *clnt, const struct sockaddr *sap); +void rpc_cleanup_clids(void); #endif /* __KERNEL__ */ #endif /* _LINUX_SUNRPC_CLNT_H */ diff --git a/include/linux/sunrpc/svc_xprt.h b/include/linux/sunrpc/svc_xprt.h index e5d19344037491..7440290f64acd3 100644 --- a/include/linux/sunrpc/svc_xprt.h +++ b/include/linux/sunrpc/svc_xprt.h @@ -66,6 +66,7 @@ struct svc_xprt { #define XPT_LISTENER 10 /* listening endpoint */ #define XPT_CACHE_AUTH 11 /* cache auth info */ #define XPT_LOCAL 12 /* connection from loopback interface */ +#define XPT_KILL_TEMP 13 /* call xpo_kill_temp_xprt before closing */ struct svc_serv *xpt_server; /* service for transport */ atomic_t xpt_reserved; /* space on outq that is rsvd */ diff --git a/include/linux/suspend.h b/include/linux/suspend.h index d9718378a8bee0..448321bb8769f9 100644 --- a/include/linux/suspend.h +++ b/include/linux/suspend.h @@ -434,6 +434,7 @@ extern bool pm_get_wakeup_count(unsigned int *count, bool block); extern bool pm_save_wakeup_count(unsigned int count); extern void pm_wakep_autosleep_enabled(bool set); extern void pm_print_active_wakeup_sources(void); +extern void pm_get_active_wakeup_sources(char *pending_sources, size_t max); static inline void lock_system_sleep(void) { diff --git a/include/linux/swap.h b/include/linux/swap.h index a56523cefb9b7b..55ff5593c193a3 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -150,8 +150,9 @@ enum { SWP_FILE = (1 << 7), /* set after swap_activate success */ SWP_AREA_DISCARD = (1 << 8), /* single-time swap area discards */ SWP_PAGE_DISCARD = (1 << 9), /* freed swap page-cluster discards */ + SWP_STABLE_WRITES = (1 << 10), /* no overwrite PG_writeback pages */ /* add others here before... */ - SWP_SCANNING = (1 << 10), /* refcount in scan_swap_map */ + SWP_SCANNING = (1 << 11), /* refcount in scan_swap_map */ }; #define SWAP_CLUSTER_MAX 32UL diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h index 5f81f8a187f2ad..d2613536fd036e 100644 --- a/include/linux/swiotlb.h +++ b/include/linux/swiotlb.h @@ -9,7 +9,13 @@ struct device; struct page; struct scatterlist; -extern int swiotlb_force; +enum swiotlb_force { + SWIOTLB_NORMAL, /* Default - depending on HW DMA mask etc. */ + SWIOTLB_FORCE, /* swiotlb=force */ + SWIOTLB_NO_FORCE, /* swiotlb=noforce */ +}; + +extern enum swiotlb_force swiotlb_force; /* * Maximum allowable number of contiguous slabs to map, diff --git a/include/linux/tcp.h b/include/linux/tcp.h index a17ae7b8521805..647532b0eb0314 100644 --- a/include/linux/tcp.h +++ b/include/linux/tcp.h @@ -62,8 +62,13 @@ static inline unsigned int tcp_optlen(const struct sk_buff *skb) /* TCP Fast Open Cookie as stored in memory */ struct tcp_fastopen_cookie { + union { + u8 val[TCP_FASTOPEN_COOKIE_MAX]; +#if IS_ENABLED(CONFIG_IPV6) + struct in6_addr addr; +#endif + }; s8 len; - u8 val[TCP_FASTOPEN_COOKIE_MAX]; bool exp; /* In RFC6994 experimental option format */ }; diff --git a/include/linux/tee_drv.h b/include/linux/tee_drv.h new file mode 100644 index 00000000000000..3657c18acef14c --- /dev/null +++ b/include/linux/tee_drv.h @@ -0,0 +1,321 @@ +/* + * Copyright (c) 2015-2016, Linaro Limited + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#ifndef __TEE_DRV_H +#define __TEE_DRV_H + +#include +#include +#include +#include + +/* + * The file describes the API provided by the generic TEE driver to the + * specific TEE driver. + */ + +#define TEE_SHM_MAPPED BIT(0) /* Memory mapped by the kernel */ +#define TEE_SHM_DMA_BUF BIT(1) /* Memory with dma-buf handle */ +#define TEE_SHM_EXT_DMA_BUF BIT(2) /* Memory with dma-buf handle */ + +struct tee_device; +struct tee_shm; +struct tee_shm_pool; + +/** + * struct tee_context - driver specific context on file pointer data + * @teedev: pointer to this drivers struct tee_device + * @list_shm: List of shared memory object owned by this context + * @data: driver specific context data, managed by the driver + */ +struct tee_context { + struct tee_device *teedev; + struct list_head list_shm; + void *data; +}; + +struct tee_param_memref { + size_t shm_offs; + size_t size; + struct tee_shm *shm; +}; + +struct tee_param_value { + u64 a; + u64 b; + u64 c; +}; + +struct tee_param { + u64 attr; + union { + struct tee_param_memref memref; + struct tee_param_value value; + } u; +}; + +/** + * struct tee_driver_ops - driver operations vtable + * @get_version: returns version of driver + * @open: called when the device file is opened + * @release: release this open file + * @open_session: open a new session + * @close_session: close a session + * @invoke_func: invoke a trusted function + * @cancel_req: request cancel of an ongoing invoke or open + * @supp_revc: called for supplicant to get a command + * @supp_send: called for supplicant to send a response + */ +struct tee_driver_ops { + void (*get_version)(struct tee_device *teedev, + struct tee_ioctl_version_data *vers); + int (*open)(struct tee_context *ctx); + void (*release)(struct tee_context *ctx); + int (*open_session)(struct tee_context *ctx, + struct tee_ioctl_open_session_arg *arg, + struct tee_param *param); + int (*close_session)(struct tee_context *ctx, u32 session); + int (*invoke_func)(struct tee_context *ctx, + struct tee_ioctl_invoke_arg *arg, + struct tee_param *param); + int (*cancel_req)(struct tee_context *ctx, u32 cancel_id, u32 session); + int (*supp_recv)(struct tee_context *ctx, u32 *func, u32 *num_params, + struct tee_param *param); + int (*supp_send)(struct tee_context *ctx, u32 ret, u32 num_params, + struct tee_param *param); +}; + +/** + * struct tee_desc - Describes the TEE driver to the subsystem + * @name: name of driver + * @ops: driver operations vtable + * @owner: module providing the driver + * @flags: Extra properties of driver, defined by TEE_DESC_* below + */ +#define TEE_DESC_PRIVILEGED 0x1 +struct tee_desc { + const char *name; + const struct tee_driver_ops *ops; + struct module *owner; + u32 flags; +}; + +/** + * tee_device_alloc() - Allocate a new struct tee_device instance + * @teedesc: Descriptor for this driver + * @dev: Parent device for this device + * @pool: Shared memory pool, NULL if not used + * @driver_data: Private driver data for this device + * + * Allocates a new struct tee_device instance. The device is + * removed by tee_device_unregister(). + * + * @returns a pointer to a 'struct tee_device' or an ERR_PTR on failure + */ +struct tee_device *tee_device_alloc(const struct tee_desc *teedesc, + struct device *dev, + struct tee_shm_pool *pool, + void *driver_data); + +/** + * tee_device_register() - Registers a TEE device + * @teedev: Device to register + * + * tee_device_unregister() need to be called to remove the @teedev if + * this function fails. + * + * @returns < 0 on failure + */ +int tee_device_register(struct tee_device *teedev); + +/** + * tee_device_unregister() - Removes a TEE device + * @teedev: Device to unregister + * + * This function should be called to remove the @teedev even if + * tee_device_register() hasn't been called yet. Does nothing if + * @teedev is NULL. + */ +void tee_device_unregister(struct tee_device *teedev); + +/** + * struct tee_shm_pool_mem_info - holds information needed to create a shared + * memory pool + * @vaddr: Virtual address of start of pool + * @paddr: Physical address of start of pool + * @size: Size in bytes of the pool + */ +struct tee_shm_pool_mem_info { + unsigned long vaddr; + phys_addr_t paddr; + size_t size; +}; + +/** + * tee_shm_pool_alloc_res_mem() - Create a shared memory pool from reserved + * memory range + * @dev: Device allocating the pool + * @priv_info: Information for driver private shared memory pool + * @dmabuf_info: Information for dma-buf shared memory pool + * + * Start and end of pools will must be page aligned. + * + * Allocation with the flag TEE_SHM_DMA_BUF set will use the range supplied + * in @dmabuf, others will use the range provided by @priv. + * + * @returns pointer to a 'struct tee_shm_pool' or an ERR_PTR on failure. + */ +struct tee_shm_pool * +tee_shm_pool_alloc_res_mem(struct device *dev, + struct tee_shm_pool_mem_info *priv_info, + struct tee_shm_pool_mem_info *dmabuf_info); + +/** + * tee_shm_pool_free() - Free a shared memory pool + * @pool: The shared memory pool to free + * + * The must be no remaining shared memory allocated from this pool when + * this function is called. + */ +void tee_shm_pool_free(struct tee_shm_pool *pool); + +/** + * tee_get_drvdata() - Return driver_data pointer + * @returns the driver_data pointer supplied to tee_register(). + */ +void *tee_get_drvdata(struct tee_device *teedev); + +/** + * tee_shm_alloc() - Allocate shared memory + * @ctx: Context that allocates the shared memory + * @size: Requested size of shared memory + * @flags: Flags setting properties for the requested shared memory. + * + * Memory allocated as global shared memory is automatically freed when the + * TEE file pointer is closed. The @flags field uses the bits defined by + * TEE_SHM_* above. TEE_SHM_MAPPED must currently always be set. If + * TEE_SHM_DMA_BUF global shared memory will be allocated and associated + * with a dma-buf handle, else driver private memory. + * + * @returns a pointer to 'struct tee_shm' + */ +struct tee_shm *tee_shm_alloc(struct tee_context *ctx, size_t size, u32 flags); + +/** + * tee_shm_register_fd() - Register shared memory from file descriptor + * + * @ctx: Context that allocates the shared memory + * @fd: shared memory file descriptor reference. + * + * @returns a pointer to 'struct tee_shm' + */ +struct tee_shm *tee_shm_register_fd(struct tee_context *ctx, int fd); + +/** + * tee_shm_free() - Free shared memory + * @shm: Handle to shared memory to free + */ +void tee_shm_free(struct tee_shm *shm); + +/** + * tee_shm_put() - Decrease reference count on a shared memory handle + * @shm: Shared memory handle + */ +void tee_shm_put(struct tee_shm *shm); + +/** + * tee_shm_va2pa() - Get physical address of a virtual address + * @shm: Shared memory handle + * @va: Virtual address to tranlsate + * @pa: Returned physical address + * @returns 0 on success and < 0 on failure + */ +int tee_shm_va2pa(struct tee_shm *shm, void *va, phys_addr_t *pa); + +/** + * tee_shm_pa2va() - Get virtual address of a physical address + * @shm: Shared memory handle + * @pa: Physical address to tranlsate + * @va: Returned virtual address + * @returns 0 on success and < 0 on failure + */ +int tee_shm_pa2va(struct tee_shm *shm, phys_addr_t pa, void **va); + +/** + * tee_shm_get_va() - Get virtual address of a shared memory plus an offset + * @shm: Shared memory handle + * @offs: Offset from start of this shared memory + * @returns virtual address of the shared memory + offs if offs is within + * the bounds of this shared memory, else an ERR_PTR + */ +void *tee_shm_get_va(struct tee_shm *shm, size_t offs); + +/** + * tee_shm_get_pa() - Get physical address of a shared memory plus an offset + * @shm: Shared memory handle + * @offs: Offset from start of this shared memory + * @pa: Physical address to return + * @returns 0 if offs is within the bounds of this shared memory, else an + * error code. + */ +int tee_shm_get_pa(struct tee_shm *shm, size_t offs, phys_addr_t *pa); + +/** + * tee_shm_get_id() - Get id of a shared memory object + * @shm: Shared memory handle + * @returns id + */ +int tee_shm_get_id(struct tee_shm *shm); + +/** + * tee_shm_get_from_id() - Find shared memory object and increase referece count + * @ctx: Context owning the shared memory + * @id: Id of shared memory object + * @returns a pointer to 'struct tee_shm' on success or an ERR_PTR on failure + */ +struct tee_shm *tee_shm_get_from_id(struct tee_context *ctx, int id); + +static inline bool tee_param_is_memref(struct tee_param *param) +{ + switch (param->attr & TEE_IOCTL_PARAM_ATTR_TYPE_MASK) { + case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT: + case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_OUTPUT: + case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INOUT: + return true; + default: + return false; + } +} + +struct tee_context *tee_client_open_context(struct tee_context *start, + int (*match)(struct tee_ioctl_version_data *, + const void *), + const void *data, struct tee_ioctl_version_data *vers); + +void tee_client_close_context(struct tee_context *ctx); + +void tee_client_get_version(struct tee_context *ctx, + struct tee_ioctl_version_data *vers); + +int tee_client_open_session(struct tee_context *ctx, + struct tee_ioctl_open_session_arg *arg, + struct tee_param *param); + +int tee_client_close_session(struct tee_context *ctx, u32 session); + +int tee_client_invoke_func(struct tee_context *ctx, + struct tee_ioctl_invoke_arg *arg, + struct tee_param *param); + +#endif /*__TEE_DRV_H*/ diff --git a/include/linux/ti_wilink_st.h b/include/linux/ti_wilink_st.h index f2293028ab9d65..81154b6d8985de 100644 --- a/include/linux/ti_wilink_st.h +++ b/include/linux/ti_wilink_st.h @@ -86,6 +86,7 @@ struct st_proto_s { extern long st_register(struct st_proto_s *); extern long st_unregister(struct st_proto_s *); +extern struct ti_st_plat_data *dt_pdata; /* * header information used by st_core.c diff --git a/include/linux/timekeeping.h b/include/linux/timekeeping.h index 09168c52ab6443..361f8bf1429dfe 100644 --- a/include/linux/timekeeping.h +++ b/include/linux/timekeeping.h @@ -249,6 +249,7 @@ static inline u64 ktime_get_raw_ns(void) extern u64 ktime_get_mono_fast_ns(void); extern u64 ktime_get_raw_fast_ns(void); +extern u64 ktime_get_boot_fast_ns(void); /* * Timespec interfaces utilizing the ktime based ones diff --git a/include/linux/usb/class-dual-role.h b/include/linux/usb/class-dual-role.h new file mode 100644 index 00000000000000..c6df2238012e4b --- /dev/null +++ b/include/linux/usb/class-dual-role.h @@ -0,0 +1,129 @@ +#ifndef __LINUX_CLASS_DUAL_ROLE_H__ +#define __LINUX_CLASS_DUAL_ROLE_H__ + +#include +#include +#include + +struct device; + +enum dual_role_supported_modes { + DUAL_ROLE_SUPPORTED_MODES_DFP_AND_UFP = 0, + DUAL_ROLE_SUPPORTED_MODES_DFP, + DUAL_ROLE_SUPPORTED_MODES_UFP, +/*The following should be the last element*/ + DUAL_ROLE_PROP_SUPPORTED_MODES_TOTAL, +}; + +enum { + DUAL_ROLE_PROP_MODE_UFP = 0, + DUAL_ROLE_PROP_MODE_DFP, + DUAL_ROLE_PROP_MODE_NONE, +/*The following should be the last element*/ + DUAL_ROLE_PROP_MODE_TOTAL, +}; + +enum { + DUAL_ROLE_PROP_PR_SRC = 0, + DUAL_ROLE_PROP_PR_SNK, + DUAL_ROLE_PROP_PR_NONE, +/*The following should be the last element*/ + DUAL_ROLE_PROP_PR_TOTAL, + +}; + +enum { + DUAL_ROLE_PROP_DR_HOST = 0, + DUAL_ROLE_PROP_DR_DEVICE, + DUAL_ROLE_PROP_DR_NONE, +/*The following should be the last element*/ + DUAL_ROLE_PROP_DR_TOTAL, +}; + +enum { + DUAL_ROLE_PROP_VCONN_SUPPLY_NO = 0, + DUAL_ROLE_PROP_VCONN_SUPPLY_YES, +/*The following should be the last element*/ + DUAL_ROLE_PROP_VCONN_SUPPLY_TOTAL, +}; + +enum dual_role_property { + DUAL_ROLE_PROP_SUPPORTED_MODES = 0, + DUAL_ROLE_PROP_MODE, + DUAL_ROLE_PROP_PR, + DUAL_ROLE_PROP_DR, + DUAL_ROLE_PROP_VCONN_SUPPLY, +}; + +struct dual_role_phy_instance; + +/* Description of typec port */ +struct dual_role_phy_desc { + /* /sys/class/dual_role_usb// */ + const char *name; + enum dual_role_supported_modes supported_modes; + enum dual_role_property *properties; + size_t num_properties; + + /* Callback for "cat /sys/class/dual_role_usb//" */ + int (*get_property)(struct dual_role_phy_instance *dual_role, + enum dual_role_property prop, + unsigned int *val); + /* Callback for "echo > + * /sys/class/dual_role_usb//" */ + int (*set_property)(struct dual_role_phy_instance *dual_role, + enum dual_role_property prop, + const unsigned int *val); + /* Decides whether userspace can change a specific property */ + int (*property_is_writeable)(struct dual_role_phy_instance *dual_role, + enum dual_role_property prop); +}; + +struct dual_role_phy_instance { + const struct dual_role_phy_desc *desc; + + /* Driver private data */ + void *drv_data; + + struct device dev; + struct work_struct changed_work; +}; + +#if IS_ENABLED(CONFIG_DUAL_ROLE_USB_INTF) +extern void dual_role_instance_changed(struct dual_role_phy_instance + *dual_role); +extern struct dual_role_phy_instance *__must_check +devm_dual_role_instance_register(struct device *parent, + const struct dual_role_phy_desc *desc); +extern void devm_dual_role_instance_unregister(struct device *dev, + struct dual_role_phy_instance + *dual_role); +extern int dual_role_get_property(struct dual_role_phy_instance *dual_role, + enum dual_role_property prop, + unsigned int *val); +extern int dual_role_set_property(struct dual_role_phy_instance *dual_role, + enum dual_role_property prop, + const unsigned int *val); +extern int dual_role_property_is_writeable(struct dual_role_phy_instance + *dual_role, + enum dual_role_property prop); +extern void *dual_role_get_drvdata(struct dual_role_phy_instance *dual_role); +#else /* CONFIG_DUAL_ROLE_USB_INTF */ +static inline void dual_role_instance_changed(struct dual_role_phy_instance + *dual_role){} +static inline struct dual_role_phy_instance *__must_check +devm_dual_role_instance_register(struct device *parent, + const struct dual_role_phy_desc *desc) +{ + return ERR_PTR(-ENOSYS); +} +static inline void devm_dual_role_instance_unregister(struct device *dev, + struct dual_role_phy_instance + *dual_role){} +static inline void *dual_role_get_drvdata(struct dual_role_phy_instance + *dual_role) +{ + return ERR_PTR(-ENOSYS); +} +#endif /* CONFIG_DUAL_ROLE_USB_INTF */ +#endif /* __LINUX_CLASS_DUAL_ROLE_H__ */ diff --git a/include/linux/usb/composite.h b/include/linux/usb/composite.h index 4616a49a1c2e58..93f0253a7e012a 100644 --- a/include/linux/usb/composite.h +++ b/include/linux/usb/composite.h @@ -581,6 +581,7 @@ struct usb_function_instance { struct config_group group; struct list_head cfs_list; struct usb_function_driver *fd; + struct usb_function *f; int (*set_inst_name)(struct usb_function_instance *inst, const char *name); void (*free_func_inst)(struct usb_function_instance *inst); diff --git a/include/linux/usb/f_accessory.h b/include/linux/usb/f_accessory.h new file mode 100644 index 00000000000000..ebe3c4d5930984 --- /dev/null +++ b/include/linux/usb/f_accessory.h @@ -0,0 +1,23 @@ +/* + * Gadget Function Driver for Android USB accessories + * + * Copyright (C) 2011 Google, Inc. + * Author: Mike Lockwood + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#ifndef __LINUX_USB_F_ACCESSORY_H +#define __LINUX_USB_F_ACCESSORY_H + +#include + +#endif /* __LINUX_USB_F_ACCESSORY_H */ diff --git a/include/linux/usb/f_mtp.h b/include/linux/usb/f_mtp.h new file mode 100644 index 00000000000000..4e8417791bea47 --- /dev/null +++ b/include/linux/usb/f_mtp.h @@ -0,0 +1,23 @@ +/* + * Gadget Function Driver for MTP + * + * Copyright (C) 2010 Google, Inc. + * Author: Mike Lockwood + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#ifndef __LINUX_USB_F_MTP_H +#define __LINUX_USB_F_MTP_H + +#include + +#endif /* __LINUX_USB_F_MTP_H */ diff --git a/include/linux/usb/gadget.h b/include/linux/usb/gadget.h index 8e81f9eb95e401..e4516e9ded0f43 100644 --- a/include/linux/usb/gadget.h +++ b/include/linux/usb/gadget.h @@ -429,7 +429,9 @@ static inline struct usb_gadget *dev_to_usb_gadget(struct device *dev) */ static inline size_t usb_ep_align(struct usb_ep *ep, size_t len) { - return round_up(len, (size_t)le16_to_cpu(ep->desc->wMaxPacketSize)); + int max_packet_size = (size_t)usb_endpoint_maxp(ep->desc) & 0x7ff; + + return round_up(len, max_packet_size); } /** diff --git a/include/linux/virtio_net.h b/include/linux/virtio_net.h index 1c912f85e04175..f211c348e592fe 100644 --- a/include/linux/virtio_net.h +++ b/include/linux/virtio_net.h @@ -56,7 +56,8 @@ static inline int virtio_net_hdr_to_skb(struct sk_buff *skb, static inline int virtio_net_hdr_from_skb(const struct sk_buff *skb, struct virtio_net_hdr *hdr, - bool little_endian) + bool little_endian, + bool has_data_valid) { memset(hdr, 0, sizeof(*hdr)); @@ -91,7 +92,8 @@ static inline int virtio_net_hdr_from_skb(const struct sk_buff *skb, skb_checksum_start_offset(skb)); hdr->csum_offset = __cpu_to_virtio16(little_endian, skb->csum_offset); - } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) { + } else if (has_data_valid && + skb->ip_summed == CHECKSUM_UNNECESSARY) { hdr->flags = VIRTIO_NET_HDR_F_DATA_VALID; } /* else everything is zero */ diff --git a/include/linux/wakeup_reason.h b/include/linux/wakeup_reason.h new file mode 100644 index 00000000000000..d84d8c301546bb --- /dev/null +++ b/include/linux/wakeup_reason.h @@ -0,0 +1,32 @@ +/* + * include/linux/wakeup_reason.h + * + * Logs the reason which caused the kernel to resume + * from the suspend mode. + * + * Copyright (C) 2014 Google, Inc. + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _LINUX_WAKEUP_REASON_H +#define _LINUX_WAKEUP_REASON_H + +#define MAX_SUSPEND_ABORT_LEN 256 + +void log_wakeup_reason(int irq); +int check_wakeup_reason(int irq); + +#ifdef CONFIG_SUSPEND +void log_suspend_abort_reason(const char *fmt, ...); +#else +static inline void log_suspend_abort_reason(const char *fmt, ...) { } +#endif + +#endif /* _LINUX_WAKEUP_REASON_H */ diff --git a/include/linux/wlan_plat.h b/include/linux/wlan_plat.h new file mode 100644 index 00000000000000..8e8b06f1ba4ae3 --- /dev/null +++ b/include/linux/wlan_plat.h @@ -0,0 +1,30 @@ +/* include/linux/wlan_plat.h + * + * Copyright (C) 2010 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ +#ifndef _LINUX_WLAN_PLAT_H_ +#define _LINUX_WLAN_PLAT_H_ + +#define WLAN_PLAT_NODFS_FLAG 0x01 + +struct wifi_platform_data { + int (*set_power)(int val); + int (*set_reset)(int val); + int (*set_carddetect)(int val); + void *(*mem_prealloc)(int section, unsigned long size); + int (*get_mac_addr)(unsigned char *buf); + int (*get_wake_irq)(void); + void *(*get_country_code)(char *ccode, u32 flags); +}; + +#endif diff --git a/include/net/addrconf.h b/include/net/addrconf.h index 8f998afc138434..e1bd2bc0277a5e 100644 --- a/include/net/addrconf.h +++ b/include/net/addrconf.h @@ -242,6 +242,8 @@ static inline bool ipv6_is_mld(struct sk_buff *skb, int nexthdr, int offset) void addrconf_prefix_rcv(struct net_device *dev, u8 *opt, int len, bool sllao); +u32 addrconf_rt_table(const struct net_device *dev, u32 default_table); + /* * anycast prototypes (anycast.c) */ diff --git a/include/net/cfg80211.h b/include/net/cfg80211.h index 14b51d739c3b39..66167138120a2d 100644 --- a/include/net/cfg80211.h +++ b/include/net/cfg80211.h @@ -4583,6 +4583,17 @@ void cfg80211_rx_assoc_resp(struct net_device *dev, */ void cfg80211_assoc_timeout(struct net_device *dev, struct cfg80211_bss *bss); +/** + * cfg80211_abandon_assoc - notify cfg80211 of abandoned association attempt + * @dev: network device + * @bss: The BSS entry with which association was abandoned. + * + * Call this whenever - for reasons reported through other API, like deauth RX, + * an association attempt was abandoned. + * This function may sleep. The caller must hold the corresponding wdev's mutex. + */ +void cfg80211_abandon_assoc(struct net_device *dev, struct cfg80211_bss *bss); + /** * cfg80211_tx_mlme_mgmt - notification of transmitted deauth/disassoc frame * @dev: network device diff --git a/include/net/fib_rules.h b/include/net/fib_rules.h index 456e4a6006abff..8dbfdf728cd8ce 100644 --- a/include/net/fib_rules.h +++ b/include/net/fib_rules.h @@ -8,6 +8,11 @@ #include #include +struct fib_kuid_range { + kuid_t start; + kuid_t end; +}; + struct fib_rule { struct list_head list; int iifindex; @@ -30,6 +35,7 @@ struct fib_rule { int suppress_prefixlen; char iifname[IFNAMSIZ]; char oifname[IFNAMSIZ]; + struct fib_kuid_range uid_range; struct rcu_head rcu; }; @@ -92,7 +98,8 @@ struct fib_rules_ops { [FRA_SUPPRESS_PREFIXLEN] = { .type = NLA_U32 }, \ [FRA_SUPPRESS_IFGROUP] = { .type = NLA_U32 }, \ [FRA_GOTO] = { .type = NLA_U32 }, \ - [FRA_L3MDEV] = { .type = NLA_U8 } + [FRA_L3MDEV] = { .type = NLA_U8 }, \ + [FRA_UID_RANGE] = { .len = sizeof(struct fib_rule_uid_range) } static inline void fib_rule_get(struct fib_rule *rule) { diff --git a/include/net/flow.h b/include/net/flow.h index 035aa771696780..6bbbca8af8e3e2 100644 --- a/include/net/flow.h +++ b/include/net/flow.h @@ -11,6 +11,7 @@ #include #include #include +#include /* * ifindex generation is per-net namespace, and loopback is @@ -37,6 +38,7 @@ struct flowi_common { #define FLOWI_FLAG_SKIP_NH_OIF 0x04 __u32 flowic_secid; struct flowi_tunnel flowic_tun_key; + kuid_t flowic_uid; }; union flowi_uli { @@ -74,6 +76,7 @@ struct flowi4 { #define flowi4_flags __fl_common.flowic_flags #define flowi4_secid __fl_common.flowic_secid #define flowi4_tun_key __fl_common.flowic_tun_key +#define flowi4_uid __fl_common.flowic_uid /* (saddr,daddr) must be grouped, same order as in IP header */ __be32 saddr; @@ -93,7 +96,8 @@ static inline void flowi4_init_output(struct flowi4 *fl4, int oif, __u32 mark, __u8 tos, __u8 scope, __u8 proto, __u8 flags, __be32 daddr, __be32 saddr, - __be16 dport, __be16 sport) + __be16 dport, __be16 sport, + kuid_t uid) { fl4->flowi4_oif = oif; fl4->flowi4_iif = LOOPBACK_IFINDEX; @@ -104,6 +108,7 @@ static inline void flowi4_init_output(struct flowi4 *fl4, int oif, fl4->flowi4_flags = flags; fl4->flowi4_secid = 0; fl4->flowi4_tun_key.tun_id = 0; + fl4->flowi4_uid = uid; fl4->daddr = daddr; fl4->saddr = saddr; fl4->fl4_dport = dport; @@ -131,6 +136,7 @@ struct flowi6 { #define flowi6_flags __fl_common.flowic_flags #define flowi6_secid __fl_common.flowic_secid #define flowi6_tun_key __fl_common.flowic_tun_key +#define flowi6_uid __fl_common.flowic_uid struct in6_addr daddr; struct in6_addr saddr; /* Note: flowi6_tos is encoded in flowlabel, too. */ @@ -176,6 +182,7 @@ struct flowi { #define flowi_flags u.__fl_common.flowic_flags #define flowi_secid u.__fl_common.flowic_secid #define flowi_tun_key u.__fl_common.flowic_tun_key +#define flowi_uid u.__fl_common.flowic_uid } __attribute__((__aligned__(BITS_PER_LONG/8))); static inline struct flowi *flowi4_to_flowi(struct flowi4 *fl4) diff --git a/include/net/ip.h b/include/net/ip.h index d3a107850a41f0..b043c7d1914e4e 100644 --- a/include/net/ip.h +++ b/include/net/ip.h @@ -178,6 +178,7 @@ struct ip_reply_arg { /* -1 if not needed */ int bound_dev_if; u8 tos; + kuid_t uid; }; #define IP_REPLY_ARG_NOSRCCHECK 1 diff --git a/include/net/ip6_route.h b/include/net/ip6_route.h index f83e78d071a30c..9dc2c182a26321 100644 --- a/include/net/ip6_route.h +++ b/include/net/ip6_route.h @@ -140,9 +140,10 @@ int rt6_route_rcv(struct net_device *dev, u8 *opt, int len, const struct in6_addr *gwaddr); void ip6_update_pmtu(struct sk_buff *skb, struct net *net, __be32 mtu, int oif, - u32 mark); + u32 mark, kuid_t uid); void ip6_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, __be32 mtu); -void ip6_redirect(struct sk_buff *skb, struct net *net, int oif, u32 mark); +void ip6_redirect(struct sk_buff *skb, struct net *net, int oif, u32 mark, + kuid_t uid); void ip6_redirect_no_header(struct sk_buff *skb, struct net *net, int oif, u32 mark); void ip6_sk_redirect(struct sk_buff *skb, struct sock *sk); diff --git a/include/net/lwtunnel.h b/include/net/lwtunnel.h index ea3f80f58fd6d5..fc7c0dbdd1ffa3 100644 --- a/include/net/lwtunnel.h +++ b/include/net/lwtunnel.h @@ -43,13 +43,12 @@ struct lwtunnel_encap_ops { int (*get_encap_size)(struct lwtunnel_state *lwtstate); int (*cmp_encap)(struct lwtunnel_state *a, struct lwtunnel_state *b); int (*xmit)(struct sk_buff *skb); + + struct module *owner; }; #ifdef CONFIG_LWTUNNEL -static inline void lwtstate_free(struct lwtunnel_state *lws) -{ - kfree(lws); -} +void lwtstate_free(struct lwtunnel_state *lws); static inline struct lwtunnel_state * lwtstate_get(struct lwtunnel_state *lws) @@ -106,6 +105,8 @@ int lwtunnel_encap_add_ops(const struct lwtunnel_encap_ops *op, unsigned int num); int lwtunnel_encap_del_ops(const struct lwtunnel_encap_ops *op, unsigned int num); +int lwtunnel_valid_encap_type(u16 encap_type); +int lwtunnel_valid_encap_type_attr(struct nlattr *attr, int len); int lwtunnel_build_state(struct net_device *dev, u16 encap_type, struct nlattr *encap, unsigned int family, const void *cfg, @@ -169,6 +170,15 @@ static inline int lwtunnel_encap_del_ops(const struct lwtunnel_encap_ops *op, return -EOPNOTSUPP; } +static inline int lwtunnel_valid_encap_type(u16 encap_type) +{ + return -EOPNOTSUPP; +} +static inline int lwtunnel_valid_encap_type_attr(struct nlattr *attr, int len) +{ + return -EOPNOTSUPP; +} + static inline int lwtunnel_build_state(struct net_device *dev, u16 encap_type, struct nlattr *encap, unsigned int family, const void *cfg, diff --git a/include/net/route.h b/include/net/route.h index 0429d47cad25c2..c0874c87c17371 100644 --- a/include/net/route.h +++ b/include/net/route.h @@ -153,7 +153,7 @@ static inline struct rtable *ip_route_output_ports(struct net *net, struct flowi flowi4_init_output(fl4, oif, sk ? sk->sk_mark : 0, tos, RT_SCOPE_UNIVERSE, proto, sk ? inet_sk_flowi_flags(sk) : 0, - daddr, saddr, dport, sport); + daddr, saddr, dport, sport, sock_net_uid(net, sk)); if (sk) security_sk_classify_flow(sk, flowi4_to_flowi(fl4)); return ip_route_output_flow(net, fl4, sk); @@ -269,7 +269,8 @@ static inline void ip_route_connect_init(struct flowi4 *fl4, __be32 dst, __be32 flow_flags |= FLOWI_FLAG_ANYSRC; flowi4_init_output(fl4, oif, sk->sk_mark, tos, RT_SCOPE_UNIVERSE, - protocol, flow_flags, dst, src, dport, sport); + protocol, flow_flags, dst, src, dport, sport, + sk->sk_uid); } static inline struct rtable *ip_route_connect(struct flowi4 *fl4, diff --git a/include/net/sock.h b/include/net/sock.h index 92b269709b9a8a..97f8ed2202bff6 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -419,6 +419,7 @@ struct sock { u32 sk_max_ack_backlog; __u32 sk_priority; __u32 sk_mark; + kuid_t sk_uid; struct pid *sk_peer_pid; const struct cred *sk_peer_cred; long sk_rcvtimeo; @@ -1651,6 +1652,7 @@ static inline void sock_graft(struct sock *sk, struct socket *parent) sk->sk_wq = parent->wq; parent->sk = sk; sk_set_socket(sk, parent); + sk->sk_uid = SOCK_INODE(parent)->i_uid; security_sock_graft(sk, parent); write_unlock_bh(&sk->sk_callback_lock); } @@ -1658,6 +1660,11 @@ static inline void sock_graft(struct sock *sk, struct socket *parent) kuid_t sock_i_uid(struct sock *sk); unsigned long sock_i_ino(struct sock *sk); +static inline kuid_t sock_net_uid(const struct net *net, const struct sock *sk) +{ + return sk ? sk->sk_uid : make_kuid(net->user_ns, 0); +} + static inline u32 net_tx_rndhash(void) { u32 v = prandom_u32(); diff --git a/include/net/tcp.h b/include/net/tcp.h index 123979fe12bf78..2700f928678374 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -271,6 +271,7 @@ extern int sysctl_tcp_autocorking; extern int sysctl_tcp_invalid_ratelimit; extern int sysctl_tcp_pacing_ss_ratio; extern int sysctl_tcp_pacing_ca_ratio; +extern int sysctl_tcp_default_init_rwnd; extern atomic_long_t tcp_memory_allocated; extern struct percpu_counter tcp_sockets_allocated; diff --git a/include/rdma/ib_addr.h b/include/rdma/ib_addr.h index 931a47ba45718a..1beab5532035dc 100644 --- a/include/rdma/ib_addr.h +++ b/include/rdma/ib_addr.h @@ -205,10 +205,12 @@ static inline void iboe_addr_get_sgid(struct rdma_dev_addr *dev_addr, dev = dev_get_by_index(&init_net, dev_addr->bound_dev_if); if (dev) { - ip4 = (struct in_device *)dev->ip_ptr; - if (ip4 && ip4->ifa_list && ip4->ifa_list->ifa_address) + ip4 = in_dev_get(dev); + if (ip4 && ip4->ifa_list && ip4->ifa_list->ifa_address) { ipv6_addr_set_v4mapped(ip4->ifa_list->ifa_address, (struct in6_addr *)gid); + in_dev_put(ip4); + } dev_put(dev); } } diff --git a/include/sound/hdmi-codec.h b/include/sound/hdmi-codec.h index 530c57bdefa065..915c4357945c27 100644 --- a/include/sound/hdmi-codec.h +++ b/include/sound/hdmi-codec.h @@ -36,10 +36,10 @@ struct hdmi_codec_daifmt { HDMI_AC97, HDMI_SPDIF, } fmt; - int bit_clk_inv:1; - int frame_clk_inv:1; - int bit_clk_master:1; - int frame_clk_master:1; + unsigned int bit_clk_inv:1; + unsigned int frame_clk_inv:1; + unsigned int bit_clk_master:1; + unsigned int frame_clk_master:1; }; /* diff --git a/include/target/target_core_base.h b/include/target/target_core_base.h index c2119008990a36..48bc1ac1da43cf 100644 --- a/include/target/target_core_base.h +++ b/include/target/target_core_base.h @@ -538,6 +538,7 @@ struct se_node_acl { char initiatorname[TRANSPORT_IQN_LEN]; /* Used to signal demo mode created ACL, disabled by default */ bool dynamic_node_acl; + bool dynamic_stop; u32 queue_depth; u32 acl_index; enum target_prot_type saved_prot_type; diff --git a/include/trace/events/android_fs.h b/include/trace/events/android_fs.h new file mode 100644 index 00000000000000..49509533d3faab --- /dev/null +++ b/include/trace/events/android_fs.h @@ -0,0 +1,65 @@ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM android_fs + +#if !defined(_TRACE_ANDROID_FS_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_ANDROID_FS_H + +#include +#include + +DEFINE_EVENT(android_fs_data_start_template, android_fs_dataread_start, + TP_PROTO(struct inode *inode, loff_t offset, int bytes, + pid_t pid, char *pathname, char *command), + TP_ARGS(inode, offset, bytes, pid, pathname, command)); + +DEFINE_EVENT(android_fs_data_end_template, android_fs_dataread_end, + TP_PROTO(struct inode *inode, loff_t offset, int bytes), + TP_ARGS(inode, offset, bytes)); + +DEFINE_EVENT(android_fs_data_start_template, android_fs_datawrite_start, + TP_PROTO(struct inode *inode, loff_t offset, int bytes, + pid_t pid, char *pathname, char *command), + TP_ARGS(inode, offset, bytes, pid, pathname, command)); + +DEFINE_EVENT(android_fs_data_end_template, android_fs_datawrite_end, + TP_PROTO(struct inode *inode, loff_t offset, int bytes), + TP_ARGS(inode, offset, bytes)); + +#endif /* _TRACE_ANDROID_FS_H */ + +/* This part must be outside protection */ +#include + +#ifndef ANDROID_FSTRACE_GET_PATHNAME +#define ANDROID_FSTRACE_GET_PATHNAME + +/* Sizes an on-stack array, so careful if sizing this up ! */ +#define MAX_TRACE_PATHBUF_LEN 256 + +static inline char * +android_fstrace_get_pathname(char *buf, int buflen, struct inode *inode) +{ + char *path; + struct dentry *d; + + /* + * d_obtain_alias() will either iput() if it locates an existing + * dentry or transfer the reference to the new dentry created. + * So get an extra reference here. + */ + ihold(inode); + d = d_obtain_alias(inode); + if (likely(!IS_ERR(d))) { + path = dentry_path_raw(d, buf, buflen); + if (unlikely(IS_ERR(path))) { + strcpy(buf, "ERROR"); + path = buf; + } + dput(d); + } else { + strcpy(buf, "ERROR"); + path = buf; + } + return path; +} +#endif diff --git a/include/trace/events/android_fs_template.h b/include/trace/events/android_fs_template.h new file mode 100644 index 00000000000000..4e61ffe7a81485 --- /dev/null +++ b/include/trace/events/android_fs_template.h @@ -0,0 +1,57 @@ +#if !defined(_TRACE_ANDROID_FS_TEMPLATE_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_ANDROID_FS_TEMPLATE_H + +#include + +DECLARE_EVENT_CLASS(android_fs_data_start_template, + TP_PROTO(struct inode *inode, loff_t offset, int bytes, + pid_t pid, char *pathname, char *command), + TP_ARGS(inode, offset, bytes, pid, pathname, command), + TP_STRUCT__entry( + __string(pathbuf, pathname); + __field(loff_t, offset); + __field(int, bytes); + __field(loff_t, i_size); + __string(cmdline, command); + __field(pid_t, pid); + __field(ino_t, ino); + ), + TP_fast_assign( + { + __assign_str(pathbuf, pathname); + __entry->offset = offset; + __entry->bytes = bytes; + __entry->i_size = i_size_read(inode); + __assign_str(cmdline, command); + __entry->pid = pid; + __entry->ino = inode->i_ino; + } + ), + TP_printk("entry_name %s, offset %llu, bytes %d, cmdline %s," + " pid %d, i_size %llu, ino %lu", + __get_str(pathbuf), __entry->offset, __entry->bytes, + __get_str(cmdline), __entry->pid, __entry->i_size, + (unsigned long) __entry->ino) +); + +DECLARE_EVENT_CLASS(android_fs_data_end_template, + TP_PROTO(struct inode *inode, loff_t offset, int bytes), + TP_ARGS(inode, offset, bytes), + TP_STRUCT__entry( + __field(ino_t, ino); + __field(loff_t, offset); + __field(int, bytes); + ), + TP_fast_assign( + { + __entry->ino = inode->i_ino; + __entry->offset = offset; + __entry->bytes = bytes; + } + ), + TP_printk("ino %lu, offset %llu, bytes %d", + (unsigned long) __entry->ino, + __entry->offset, __entry->bytes) +); + +#endif /* _TRACE_ANDROID_FS_TEMPLATE_H */ diff --git a/include/trace/events/btrfs.h b/include/trace/events/btrfs.h index e030d6f6c19acb..6d7fe1169956bb 100644 --- a/include/trace/events/btrfs.h +++ b/include/trace/events/btrfs.h @@ -1162,22 +1162,26 @@ DECLARE_EVENT_CLASS(btrfs__work, __entry->func, __entry->ordered_func, __entry->ordered_free) ); -/* For situiations that the work is freed */ +/* + * For situiations when the work is freed, we pass fs_info and a tag that that + * matches address of the work structure so it can be paired with the + * scheduling event. + */ DECLARE_EVENT_CLASS(btrfs__work__done, - TP_PROTO(struct btrfs_work *work), + TP_PROTO(struct btrfs_fs_info *fs_info, void *wtag), - TP_ARGS(work), + TP_ARGS(fs_info, wtag), TP_STRUCT__entry_btrfs( - __field( void *, work ) + __field( void *, wtag ) ), - TP_fast_assign_btrfs(btrfs_work_owner(work), - __entry->work = work; + TP_fast_assign_btrfs(fs_info, + __entry->wtag = wtag; ), - TP_printk_btrfs("work->%p", __entry->work) + TP_printk_btrfs("work->%p", __entry->wtag) ); DEFINE_EVENT(btrfs__work, btrfs_work_queued, @@ -1196,9 +1200,9 @@ DEFINE_EVENT(btrfs__work, btrfs_work_sched, DEFINE_EVENT(btrfs__work__done, btrfs_all_work_done, - TP_PROTO(struct btrfs_work *work), + TP_PROTO(struct btrfs_fs_info *fs_info, void *wtag), - TP_ARGS(work) + TP_ARGS(fs_info, wtag) ); DEFINE_EVENT(btrfs__work, btrfs_ordered_sched, diff --git a/include/trace/events/cpufreq_interactive.h b/include/trace/events/cpufreq_interactive.h new file mode 100644 index 00000000000000..faecc0bfdefffe --- /dev/null +++ b/include/trace/events/cpufreq_interactive.h @@ -0,0 +1,112 @@ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM cpufreq_interactive + +#if !defined(_TRACE_CPUFREQ_INTERACTIVE_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_CPUFREQ_INTERACTIVE_H + +#include + +DECLARE_EVENT_CLASS(set, + TP_PROTO(u32 cpu_id, unsigned long targfreq, + unsigned long actualfreq), + TP_ARGS(cpu_id, targfreq, actualfreq), + + TP_STRUCT__entry( + __field(u32, cpu_id) + __field(unsigned long, targfreq) + __field(unsigned long, actualfreq) + ), + + TP_fast_assign( + __entry->cpu_id = (u32)cpu_id; + __entry->targfreq = targfreq; + __entry->actualfreq = actualfreq; + ), + + TP_printk("cpu=%u targ=%lu actual=%lu", + __entry->cpu_id, __entry->targfreq, + __entry->actualfreq) +); + +DEFINE_EVENT(set, cpufreq_interactive_setspeed, + TP_PROTO(u32 cpu_id, unsigned long targfreq, + unsigned long actualfreq), + TP_ARGS(cpu_id, targfreq, actualfreq) +); + +DECLARE_EVENT_CLASS(loadeval, + TP_PROTO(unsigned long cpu_id, unsigned long load, + unsigned long curtarg, unsigned long curactual, + unsigned long newtarg), + TP_ARGS(cpu_id, load, curtarg, curactual, newtarg), + + TP_STRUCT__entry( + __field(unsigned long, cpu_id) + __field(unsigned long, load) + __field(unsigned long, curtarg) + __field(unsigned long, curactual) + __field(unsigned long, newtarg) + ), + + TP_fast_assign( + __entry->cpu_id = cpu_id; + __entry->load = load; + __entry->curtarg = curtarg; + __entry->curactual = curactual; + __entry->newtarg = newtarg; + ), + + TP_printk("cpu=%lu load=%lu cur=%lu actual=%lu targ=%lu", + __entry->cpu_id, __entry->load, __entry->curtarg, + __entry->curactual, __entry->newtarg) +); + +DEFINE_EVENT(loadeval, cpufreq_interactive_target, + TP_PROTO(unsigned long cpu_id, unsigned long load, + unsigned long curtarg, unsigned long curactual, + unsigned long newtarg), + TP_ARGS(cpu_id, load, curtarg, curactual, newtarg) +); + +DEFINE_EVENT(loadeval, cpufreq_interactive_already, + TP_PROTO(unsigned long cpu_id, unsigned long load, + unsigned long curtarg, unsigned long curactual, + unsigned long newtarg), + TP_ARGS(cpu_id, load, curtarg, curactual, newtarg) +); + +DEFINE_EVENT(loadeval, cpufreq_interactive_notyet, + TP_PROTO(unsigned long cpu_id, unsigned long load, + unsigned long curtarg, unsigned long curactual, + unsigned long newtarg), + TP_ARGS(cpu_id, load, curtarg, curactual, newtarg) +); + +TRACE_EVENT(cpufreq_interactive_boost, + TP_PROTO(const char *s), + TP_ARGS(s), + TP_STRUCT__entry( + __string(s, s) + ), + TP_fast_assign( + __assign_str(s, s); + ), + TP_printk("%s", __get_str(s)) +); + +TRACE_EVENT(cpufreq_interactive_unboost, + TP_PROTO(const char *s), + TP_ARGS(s), + TP_STRUCT__entry( + __string(s, s) + ), + TP_fast_assign( + __assign_str(s, s); + ), + TP_printk("%s", __get_str(s)) +); + +#endif /* _TRACE_CPUFREQ_INTERACTIVE_H */ + +/* This part must be outside protection */ +#include diff --git a/include/trace/events/cpufreq_sched.h b/include/trace/events/cpufreq_sched.h new file mode 100644 index 00000000000000..a46cd088e96990 --- /dev/null +++ b/include/trace/events/cpufreq_sched.h @@ -0,0 +1,87 @@ +/* + * Copyright (C) 2015 Steve Muckle + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM cpufreq_sched + +#if !defined(_TRACE_CPUFREQ_SCHED_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_CPUFREQ_SCHED_H + +#include +#include + +TRACE_EVENT(cpufreq_sched_throttled, + TP_PROTO(unsigned int rem), + TP_ARGS(rem), + TP_STRUCT__entry( + __field( unsigned int, rem) + ), + TP_fast_assign( + __entry->rem = rem; + ), + TP_printk("throttled - %d usec remaining", __entry->rem) +); + +TRACE_EVENT(cpufreq_sched_request_opp, + TP_PROTO(int cpu, + unsigned long capacity, + unsigned int freq_new, + unsigned int requested_freq), + TP_ARGS(cpu, capacity, freq_new, requested_freq), + TP_STRUCT__entry( + __field( int, cpu) + __field( unsigned long, capacity) + __field( unsigned int, freq_new) + __field( unsigned int, requested_freq) + ), + TP_fast_assign( + __entry->cpu = cpu; + __entry->capacity = capacity; + __entry->freq_new = freq_new; + __entry->requested_freq = requested_freq; + ), + TP_printk("cpu %d cap change, cluster cap request %ld => OPP %d " + "(cur %d)", + __entry->cpu, __entry->capacity, __entry->freq_new, + __entry->requested_freq) +); + +TRACE_EVENT(cpufreq_sched_update_capacity, + TP_PROTO(int cpu, + bool request, + struct sched_capacity_reqs *scr, + unsigned long new_capacity), + TP_ARGS(cpu, request, scr, new_capacity), + TP_STRUCT__entry( + __field( int, cpu) + __field( bool, request) + __field( unsigned long, cfs) + __field( unsigned long, rt) + __field( unsigned long, dl) + __field( unsigned long, total) + __field( unsigned long, new_total) + ), + TP_fast_assign( + __entry->cpu = cpu; + __entry->request = request; + __entry->cfs = scr->cfs; + __entry->rt = scr->rt; + __entry->dl = scr->dl; + __entry->total = scr->total; + __entry->new_total = new_capacity; + ), + TP_printk("cpu=%d set_cap=%d cfs=%ld rt=%ld dl=%ld old_tot=%ld " + "new_tot=%ld", + __entry->cpu, __entry->request, __entry->cfs, __entry->rt, + __entry->dl, __entry->total, __entry->new_total) +); + +#endif /* _TRACE_CPUFREQ_SCHED_H */ + +/* This part must be outside protection */ +#include diff --git a/include/trace/events/gpu.h b/include/trace/events/gpu.h new file mode 100644 index 00000000000000..7e15cdfafe5a88 --- /dev/null +++ b/include/trace/events/gpu.h @@ -0,0 +1,143 @@ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM gpu + +#if !defined(_TRACE_GPU_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_GPU_H + +#include +#include + +#define show_secs_from_ns(ns) \ + ({ \ + u64 t = ns + (NSEC_PER_USEC / 2); \ + do_div(t, NSEC_PER_SEC); \ + t; \ + }) + +#define show_usecs_from_ns(ns) \ + ({ \ + u64 t = ns + (NSEC_PER_USEC / 2) ; \ + u32 rem; \ + do_div(t, NSEC_PER_USEC); \ + rem = do_div(t, USEC_PER_SEC); \ + }) + +/* + * The gpu_sched_switch event indicates that a switch from one GPU context to + * another occurred on one of the GPU hardware blocks. + * + * The gpu_name argument identifies the GPU hardware block. Each independently + * scheduled GPU hardware block should have a different name. This may be used + * in different ways for different GPUs. For example, if a GPU includes + * multiple processing cores it may use names "GPU 0", "GPU 1", etc. If a GPU + * includes a separately scheduled 2D and 3D hardware block, it might use the + * names "2D" and "3D". + * + * The timestamp argument is the timestamp at which the switch occurred on the + * GPU. These timestamps are in units of nanoseconds and must use + * approximately the same time as sched_clock, though they need not come from + * any CPU clock. The timestamps for a single hardware block must be + * monotonically nondecreasing. This means that if a variable compensation + * offset is used to translate from some other clock to the sched_clock, then + * care must be taken when increasing that offset, and doing so may result in + * multiple events with the same timestamp. + * + * The next_ctx_id argument identifies the next context that was running on + * the GPU hardware block. A value of 0 indicates that the hardware block + * will be idle. + * + * The next_prio argument indicates the priority of the next context at the + * time of the event. The exact numeric values may mean different things for + * different GPUs, but they should follow the rule that lower values indicate a + * higher priority. + * + * The next_job_id argument identifies the batch of work that the GPU will be + * working on. This should correspond to a job_id that was previously traced + * as a gpu_job_enqueue event when the batch of work was created. + */ +TRACE_EVENT(gpu_sched_switch, + + TP_PROTO(const char *gpu_name, u64 timestamp, + u32 next_ctx_id, s32 next_prio, u32 next_job_id), + + TP_ARGS(gpu_name, timestamp, next_ctx_id, next_prio, next_job_id), + + TP_STRUCT__entry( + __string( gpu_name, gpu_name ) + __field( u64, timestamp ) + __field( u32, next_ctx_id ) + __field( s32, next_prio ) + __field( u32, next_job_id ) + ), + + TP_fast_assign( + __assign_str(gpu_name, gpu_name); + __entry->timestamp = timestamp; + __entry->next_ctx_id = next_ctx_id; + __entry->next_prio = next_prio; + __entry->next_job_id = next_job_id; + ), + + TP_printk("gpu_name=%s ts=%llu.%06lu next_ctx_id=%lu next_prio=%ld " + "next_job_id=%lu", + __get_str(gpu_name), + (unsigned long long)show_secs_from_ns(__entry->timestamp), + (unsigned long)show_usecs_from_ns(__entry->timestamp), + (unsigned long)__entry->next_ctx_id, + (long)__entry->next_prio, + (unsigned long)__entry->next_job_id) +); + +/* + * The gpu_job_enqueue event indicates that a batch of work has been queued up + * to be processed by the GPU. This event is not intended to indicate that + * the batch of work has been submitted to the GPU hardware, but rather that + * it has been submitted to the GPU kernel driver. + * + * This event should be traced on the thread that initiated the work being + * queued. For example, if a batch of work is submitted to the kernel by a + * userland thread, the event should be traced on that thread. + * + * The ctx_id field identifies the GPU context in which the batch of work + * being queued is to be run. + * + * The job_id field identifies the batch of work being queued within the given + * GPU context. The first batch of work submitted for a given GPU context + * should have a job_id of 0, and each subsequent batch of work should + * increment the job_id by 1. + * + * The type field identifies the type of the job being enqueued. The job + * types may be different for different GPU hardware. For example, a GPU may + * differentiate between "2D", "3D", and "compute" jobs. + */ +TRACE_EVENT(gpu_job_enqueue, + + TP_PROTO(u32 ctx_id, u32 job_id, const char *type), + + TP_ARGS(ctx_id, job_id, type), + + TP_STRUCT__entry( + __field( u32, ctx_id ) + __field( u32, job_id ) + __string( type, type ) + ), + + TP_fast_assign( + __entry->ctx_id = ctx_id; + __entry->job_id = job_id; + __assign_str(type, type); + ), + + TP_printk("ctx_id=%lu job_id=%lu type=%s", + (unsigned long)__entry->ctx_id, + (unsigned long)__entry->job_id, + __get_str(type)) +); + +#undef show_secs_from_ns +#undef show_usecs_from_ns + +#endif /* _TRACE_GPU_H */ + +/* This part must be outside protection */ +#include diff --git a/include/trace/events/net.h b/include/trace/events/net.h index 49cc7c3de25221..89d009e1093876 100644 --- a/include/trace/events/net.h +++ b/include/trace/events/net.h @@ -57,7 +57,7 @@ TRACE_EVENT(net_dev_start_xmit, __entry->gso_type = skb_shinfo(skb)->gso_type; ), - TP_printk("dev=%s queue_mapping=%u skbaddr=%p vlan_tagged=%d vlan_proto=0x%04x vlan_tci=0x%04x protocol=0x%04x ip_summed=%d len=%u data_len=%u network_offset=%d transport_offset_valid=%d transport_offset=%d tx_flags=%d gso_size=%d gso_segs=%d gso_type=%#x", + TP_printk("dev=%s queue_mapping=%u skbaddr=%pK vlan_tagged=%d vlan_proto=0x%04x vlan_tci=0x%04x protocol=0x%04x ip_summed=%d len=%u data_len=%u network_offset=%d transport_offset_valid=%d transport_offset=%d tx_flags=%d gso_size=%d gso_segs=%d gso_type=%#x", __get_str(name), __entry->queue_mapping, __entry->skbaddr, __entry->vlan_tagged, __entry->vlan_proto, __entry->vlan_tci, __entry->protocol, __entry->ip_summed, __entry->len, @@ -90,7 +90,7 @@ TRACE_EVENT(net_dev_xmit, __assign_str(name, dev->name); ), - TP_printk("dev=%s skbaddr=%p len=%u rc=%d", + TP_printk("dev=%s skbaddr=%pK len=%u rc=%d", __get_str(name), __entry->skbaddr, __entry->len, __entry->rc) ); @@ -112,7 +112,7 @@ DECLARE_EVENT_CLASS(net_dev_template, __assign_str(name, skb->dev->name); ), - TP_printk("dev=%s skbaddr=%p len=%u", + TP_printk("dev=%s skbaddr=%pK len=%u", __get_str(name), __entry->skbaddr, __entry->len) ) @@ -191,7 +191,7 @@ DECLARE_EVENT_CLASS(net_dev_rx_verbose_template, __entry->gso_type = skb_shinfo(skb)->gso_type; ), - TP_printk("dev=%s napi_id=%#x queue_mapping=%u skbaddr=%p vlan_tagged=%d vlan_proto=0x%04x vlan_tci=0x%04x protocol=0x%04x ip_summed=%d hash=0x%08x l4_hash=%d len=%u data_len=%u truesize=%u mac_header_valid=%d mac_header=%d nr_frags=%d gso_size=%d gso_type=%#x", + TP_printk("dev=%s napi_id=%#x queue_mapping=%u skbaddr=%pK vlan_tagged=%d vlan_proto=0x%04x vlan_tci=0x%04x protocol=0x%04x ip_summed=%d hash=0x%08x l4_hash=%d len=%u data_len=%u truesize=%u mac_header_valid=%d mac_header=%d nr_frags=%d gso_size=%d gso_type=%#x", __get_str(name), __entry->napi_id, __entry->queue_mapping, __entry->skbaddr, __entry->vlan_tagged, __entry->vlan_proto, __entry->vlan_tci, __entry->protocol, __entry->ip_summed, diff --git a/include/trace/events/power.h b/include/trace/events/power.h index 54e3aad3280671..ec6f8156155810 100644 --- a/include/trace/events/power.h +++ b/include/trace/events/power.h @@ -147,6 +147,38 @@ DEFINE_EVENT(cpu, cpu_frequency, TP_ARGS(frequency, cpu_id) ); +TRACE_EVENT(cpu_frequency_limits, + + TP_PROTO(unsigned int max_freq, unsigned int min_freq, + unsigned int cpu_id), + + TP_ARGS(max_freq, min_freq, cpu_id), + + TP_STRUCT__entry( + __field( u32, min_freq ) + __field( u32, max_freq ) + __field( u32, cpu_id ) + ), + + TP_fast_assign( + __entry->min_freq = min_freq; + __entry->max_freq = max_freq; + __entry->cpu_id = cpu_id; + ), + + TP_printk("min=%lu max=%lu cpu_id=%lu", + (unsigned long)__entry->min_freq, + (unsigned long)__entry->max_freq, + (unsigned long)__entry->cpu_id) +); + +DEFINE_EVENT(cpu, cpu_capacity, + + TP_PROTO(unsigned int capacity, unsigned int cpu_id), + + TP_ARGS(capacity, cpu_id) +); + TRACE_EVENT(device_pm_callback_start, TP_PROTO(struct device *dev, const char *pm_ops, int event), @@ -300,6 +332,25 @@ DEFINE_EVENT(clock, clock_set_rate, TP_ARGS(name, state, cpu_id) ); +TRACE_EVENT(clock_set_parent, + + TP_PROTO(const char *name, const char *parent_name), + + TP_ARGS(name, parent_name), + + TP_STRUCT__entry( + __string( name, name ) + __string( parent_name, parent_name ) + ), + + TP_fast_assign( + __assign_str(name, name); + __assign_str(parent_name, parent_name); + ), + + TP_printk("%s parent=%s", __get_str(name), __get_str(parent_name)) +); + /* * The power domain events are used for power domains transitions */ diff --git a/include/trace/events/sched.h b/include/trace/events/sched.h index 9b90c57517a918..12927f326d610f 100644 --- a/include/trace/events/sched.h +++ b/include/trace/events/sched.h @@ -219,7 +219,7 @@ DECLARE_EVENT_CLASS(sched_process_template, DEFINE_EVENT(sched_process_template, sched_process_free, TP_PROTO(struct task_struct *p), TP_ARGS(p)); - + /* * Tracepoint for a task exiting: @@ -373,6 +373,30 @@ DEFINE_EVENT(sched_stat_template, sched_stat_blocked, TP_PROTO(struct task_struct *tsk, u64 delay), TP_ARGS(tsk, delay)); +/* + * Tracepoint for recording the cause of uninterruptible sleep. + */ +TRACE_EVENT(sched_blocked_reason, + + TP_PROTO(struct task_struct *tsk), + + TP_ARGS(tsk), + + TP_STRUCT__entry( + __field( pid_t, pid ) + __field( void*, caller ) + __field( bool, io_wait ) + ), + + TP_fast_assign( + __entry->pid = tsk->pid; + __entry->caller = (void*)get_wchan(tsk); + __entry->io_wait = tsk->in_iowait; + ), + + TP_printk("pid=%d iowait=%d caller=%pS", __entry->pid, __entry->io_wait, __entry->caller) +); + /* * Tracepoint for accounting runtime (time the task is executing * on a CPU). @@ -562,6 +586,505 @@ TRACE_EVENT(sched_wake_idle_without_ipi, TP_printk("cpu=%d", __entry->cpu) ); + +TRACE_EVENT(sched_contrib_scale_f, + + TP_PROTO(int cpu, unsigned long freq_scale_factor, + unsigned long cpu_scale_factor), + + TP_ARGS(cpu, freq_scale_factor, cpu_scale_factor), + + TP_STRUCT__entry( + __field(int, cpu) + __field(unsigned long, freq_scale_factor) + __field(unsigned long, cpu_scale_factor) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->freq_scale_factor = freq_scale_factor; + __entry->cpu_scale_factor = cpu_scale_factor; + ), + + TP_printk("cpu=%d freq_scale_factor=%lu cpu_scale_factor=%lu", + __entry->cpu, __entry->freq_scale_factor, + __entry->cpu_scale_factor) +); + +#ifdef CONFIG_SMP + +/* + * Tracepoint for accounting sched averages for tasks. + */ +TRACE_EVENT(sched_load_avg_task, + + TP_PROTO(struct task_struct *tsk, struct sched_avg *avg), + + TP_ARGS(tsk, avg), + + TP_STRUCT__entry( + __array( char, comm, TASK_COMM_LEN ) + __field( pid_t, pid ) + __field( int, cpu ) + __field( unsigned long, load_avg ) + __field( unsigned long, util_avg ) + __field( u64, load_sum ) + __field( u32, util_sum ) + __field( u32, period_contrib ) + ), + + TP_fast_assign( + memcpy(__entry->comm, tsk->comm, TASK_COMM_LEN); + __entry->pid = tsk->pid; + __entry->cpu = task_cpu(tsk); + __entry->load_avg = avg->load_avg; + __entry->util_avg = avg->util_avg; + __entry->load_sum = avg->load_sum; + __entry->util_sum = avg->util_sum; + __entry->period_contrib = avg->period_contrib; + ), + + TP_printk("comm=%s pid=%d cpu=%d load_avg=%lu util_avg=%lu load_sum=%llu" + " util_sum=%u period_contrib=%u", + __entry->comm, + __entry->pid, + __entry->cpu, + __entry->load_avg, + __entry->util_avg, + (u64)__entry->load_sum, + (u32)__entry->util_sum, + (u32)__entry->period_contrib) +); + +/* + * Tracepoint for accounting sched averages for cpus. + */ +TRACE_EVENT(sched_load_avg_cpu, + + TP_PROTO(int cpu, struct cfs_rq *cfs_rq), + + TP_ARGS(cpu, cfs_rq), + + TP_STRUCT__entry( + __field( int, cpu ) + __field( unsigned long, load_avg ) + __field( unsigned long, util_avg ) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->load_avg = cfs_rq->avg.load_avg; + __entry->util_avg = cfs_rq->avg.util_avg; + ), + + TP_printk("cpu=%d load_avg=%lu util_avg=%lu", + __entry->cpu, __entry->load_avg, __entry->util_avg) +); + +/* + * Tracepoint for sched_tune_config settings + */ +TRACE_EVENT(sched_tune_config, + + TP_PROTO(int boost), + + TP_ARGS(boost), + + TP_STRUCT__entry( + __field( int, boost ) + ), + + TP_fast_assign( + __entry->boost = boost; + ), + + TP_printk("boost=%d ", __entry->boost) +); + +/* + * Tracepoint for accounting CPU boosted utilization + */ +TRACE_EVENT(sched_boost_cpu, + + TP_PROTO(int cpu, unsigned long util, long margin), + + TP_ARGS(cpu, util, margin), + + TP_STRUCT__entry( + __field( int, cpu ) + __field( unsigned long, util ) + __field(long, margin ) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->util = util; + __entry->margin = margin; + ), + + TP_printk("cpu=%d util=%lu margin=%ld", + __entry->cpu, + __entry->util, + __entry->margin) +); + +/* + * Tracepoint for schedtune_tasks_update + */ +TRACE_EVENT(sched_tune_tasks_update, + + TP_PROTO(struct task_struct *tsk, int cpu, int tasks, int idx, + int boost, int max_boost), + + TP_ARGS(tsk, cpu, tasks, idx, boost, max_boost), + + TP_STRUCT__entry( + __array( char, comm, TASK_COMM_LEN ) + __field( pid_t, pid ) + __field( int, cpu ) + __field( int, tasks ) + __field( int, idx ) + __field( int, boost ) + __field( int, max_boost ) + ), + + TP_fast_assign( + memcpy(__entry->comm, tsk->comm, TASK_COMM_LEN); + __entry->pid = tsk->pid; + __entry->cpu = cpu; + __entry->tasks = tasks; + __entry->idx = idx; + __entry->boost = boost; + __entry->max_boost = max_boost; + ), + + TP_printk("pid=%d comm=%s " + "cpu=%d tasks=%d idx=%d boost=%d max_boost=%d", + __entry->pid, __entry->comm, + __entry->cpu, __entry->tasks, __entry->idx, + __entry->boost, __entry->max_boost) +); + +/* + * Tracepoint for schedtune_boostgroup_update + */ +TRACE_EVENT(sched_tune_boostgroup_update, + + TP_PROTO(int cpu, int variation, int max_boost), + + TP_ARGS(cpu, variation, max_boost), + + TP_STRUCT__entry( + __field( int, cpu ) + __field( int, variation ) + __field( int, max_boost ) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->variation = variation; + __entry->max_boost = max_boost; + ), + + TP_printk("cpu=%d variation=%d max_boost=%d", + __entry->cpu, __entry->variation, __entry->max_boost) +); + +/* + * Tracepoint for accounting task boosted utilization + */ +TRACE_EVENT(sched_boost_task, + + TP_PROTO(struct task_struct *tsk, unsigned long util, long margin), + + TP_ARGS(tsk, util, margin), + + TP_STRUCT__entry( + __array( char, comm, TASK_COMM_LEN ) + __field( pid_t, pid ) + __field( unsigned long, util ) + __field( long, margin ) + + ), + + TP_fast_assign( + memcpy(__entry->comm, tsk->comm, TASK_COMM_LEN); + __entry->pid = tsk->pid; + __entry->util = util; + __entry->margin = margin; + ), + + TP_printk("comm=%s pid=%d util=%lu margin=%ld", + __entry->comm, __entry->pid, + __entry->util, + __entry->margin) +); + +/* + * Tracepoint for accounting sched group energy + */ +TRACE_EVENT(sched_energy_diff, + + TP_PROTO(struct task_struct *tsk, int scpu, int dcpu, int udelta, + int nrgb, int nrga, int nrgd, int capb, int capa, int capd, + int nrgn, int nrgp), + + TP_ARGS(tsk, scpu, dcpu, udelta, + nrgb, nrga, nrgd, capb, capa, capd, + nrgn, nrgp), + + TP_STRUCT__entry( + __array( char, comm, TASK_COMM_LEN ) + __field( pid_t, pid ) + __field( int, scpu ) + __field( int, dcpu ) + __field( int, udelta ) + __field( int, nrgb ) + __field( int, nrga ) + __field( int, nrgd ) + __field( int, capb ) + __field( int, capa ) + __field( int, capd ) + __field( int, nrgn ) + __field( int, nrgp ) + ), + + TP_fast_assign( + memcpy(__entry->comm, tsk->comm, TASK_COMM_LEN); + __entry->pid = tsk->pid; + __entry->scpu = scpu; + __entry->dcpu = dcpu; + __entry->udelta = udelta; + __entry->nrgb = nrgb; + __entry->nrga = nrga; + __entry->nrgd = nrgd; + __entry->capb = capb; + __entry->capa = capa; + __entry->capd = capd; + __entry->nrgn = nrgn; + __entry->nrgp = nrgp; + ), + + TP_printk("pid=%d comm=%s " + "src_cpu=%d dst_cpu=%d usage_delta=%d " + "nrg_before=%d nrg_after=%d nrg_diff=%d " + "cap_before=%d cap_after=%d cap_delta=%d " + "nrg_delta=%d nrg_payoff=%d", + __entry->pid, __entry->comm, + __entry->scpu, __entry->dcpu, __entry->udelta, + __entry->nrgb, __entry->nrga, __entry->nrgd, + __entry->capb, __entry->capa, __entry->capd, + __entry->nrgn, __entry->nrgp) +); + +/* + * Tracepoint for schedtune_tasks_update + */ +TRACE_EVENT(sched_tune_filter, + + TP_PROTO(int nrg_delta, int cap_delta, + int nrg_gain, int cap_gain, + int payoff, int region), + + TP_ARGS(nrg_delta, cap_delta, nrg_gain, cap_gain, payoff, region), + + TP_STRUCT__entry( + __field( int, nrg_delta ) + __field( int, cap_delta ) + __field( int, nrg_gain ) + __field( int, cap_gain ) + __field( int, payoff ) + __field( int, region ) + ), + + TP_fast_assign( + __entry->nrg_delta = nrg_delta; + __entry->cap_delta = cap_delta; + __entry->nrg_gain = nrg_gain; + __entry->cap_gain = cap_gain; + __entry->payoff = payoff; + __entry->region = region; + ), + + TP_printk("nrg_delta=%d cap_delta=%d nrg_gain=%d cap_gain=%d payoff=%d region=%d", + __entry->nrg_delta, __entry->cap_delta, + __entry->nrg_gain, __entry->cap_gain, + __entry->payoff, __entry->region) +); + +/* + * Tracepoint for system overutilized flag + */ +TRACE_EVENT(sched_overutilized, + + TP_PROTO(bool overutilized), + + TP_ARGS(overutilized), + + TP_STRUCT__entry( + __field( bool, overutilized ) + ), + + TP_fast_assign( + __entry->overutilized = overutilized; + ), + + TP_printk("overutilized=%d", + __entry->overutilized ? 1 : 0) +); +#ifdef CONFIG_SCHED_WALT +struct rq; + +TRACE_EVENT(walt_update_task_ravg, + + TP_PROTO(struct task_struct *p, struct rq *rq, int evt, + u64 wallclock, u64 irqtime), + + TP_ARGS(p, rq, evt, wallclock, irqtime), + + TP_STRUCT__entry( + __array( char, comm, TASK_COMM_LEN ) + __field( pid_t, pid ) + __field( pid_t, cur_pid ) + __field(unsigned int, cur_freq ) + __field( u64, wallclock ) + __field( u64, mark_start ) + __field( u64, delta_m ) + __field( u64, win_start ) + __field( u64, delta ) + __field( u64, irqtime ) + __field( int, evt ) + __field(unsigned int, demand ) + __field(unsigned int, sum ) + __field( int, cpu ) + __field( u64, cs ) + __field( u64, ps ) + __field( u32, curr_window ) + __field( u32, prev_window ) + __field( u64, nt_cs ) + __field( u64, nt_ps ) + __field( u32, active_windows ) + ), + + TP_fast_assign( + __entry->wallclock = wallclock; + __entry->win_start = rq->window_start; + __entry->delta = (wallclock - rq->window_start); + __entry->evt = evt; + __entry->cpu = rq->cpu; + __entry->cur_pid = rq->curr->pid; + __entry->cur_freq = rq->cur_freq; + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->mark_start = p->ravg.mark_start; + __entry->delta_m = (wallclock - p->ravg.mark_start); + __entry->demand = p->ravg.demand; + __entry->sum = p->ravg.sum; + __entry->irqtime = irqtime; + __entry->cs = rq->curr_runnable_sum; + __entry->ps = rq->prev_runnable_sum; + __entry->curr_window = p->ravg.curr_window; + __entry->prev_window = p->ravg.prev_window; + __entry->nt_cs = rq->nt_curr_runnable_sum; + __entry->nt_ps = rq->nt_prev_runnable_sum; + __entry->active_windows = p->ravg.active_windows; + ), + + TP_printk("wc %llu ws %llu delta %llu event %d cpu %d cur_freq %u cur_pid %d task %d (%s) ms %llu delta %llu demand %u sum %u irqtime %llu" + " cs %llu ps %llu cur_window %u prev_window %u nt_cs %llu nt_ps %llu active_wins %u" + , __entry->wallclock, __entry->win_start, __entry->delta, + __entry->evt, __entry->cpu, + __entry->cur_freq, __entry->cur_pid, + __entry->pid, __entry->comm, __entry->mark_start, + __entry->delta_m, __entry->demand, + __entry->sum, __entry->irqtime, + __entry->cs, __entry->ps, + __entry->curr_window, __entry->prev_window, + __entry->nt_cs, __entry->nt_ps, + __entry->active_windows + ) +); + +TRACE_EVENT(walt_update_history, + + TP_PROTO(struct rq *rq, struct task_struct *p, u32 runtime, int samples, + int evt), + + TP_ARGS(rq, p, runtime, samples, evt), + + TP_STRUCT__entry( + __array( char, comm, TASK_COMM_LEN ) + __field( pid_t, pid ) + __field(unsigned int, runtime ) + __field( int, samples ) + __field( int, evt ) + __field( u64, demand ) + __field( u64, walt_avg ) + __field(unsigned int, pelt_avg ) + __array( u32, hist, RAVG_HIST_SIZE_MAX) + __field( int, cpu ) + ), + + TP_fast_assign( + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->runtime = runtime; + __entry->samples = samples; + __entry->evt = evt; + __entry->demand = p->ravg.demand; + __entry->walt_avg = (__entry->demand << 10); + __entry->walt_avg = div_u64(__entry->walt_avg, + walt_ravg_window); + __entry->pelt_avg = p->se.avg.util_avg; + memcpy(__entry->hist, p->ravg.sum_history, + RAVG_HIST_SIZE_MAX * sizeof(u32)); + __entry->cpu = rq->cpu; + ), + + TP_printk("%d (%s): runtime %u samples %d event %d demand %llu" + " walt %llu pelt %u (hist: %u %u %u %u %u) cpu %d", + __entry->pid, __entry->comm, + __entry->runtime, __entry->samples, __entry->evt, + __entry->demand, + __entry->walt_avg, + __entry->pelt_avg, + __entry->hist[0], __entry->hist[1], + __entry->hist[2], __entry->hist[3], + __entry->hist[4], __entry->cpu) +); + +TRACE_EVENT(walt_migration_update_sum, + + TP_PROTO(struct rq *rq, struct task_struct *p), + + TP_ARGS(rq, p), + + TP_STRUCT__entry( + __field(int, cpu ) + __field(int, pid ) + __field( u64, cs ) + __field( u64, ps ) + __field( s64, nt_cs ) + __field( s64, nt_ps ) + ), + + TP_fast_assign( + __entry->cpu = cpu_of(rq); + __entry->cs = rq->curr_runnable_sum; + __entry->ps = rq->prev_runnable_sum; + __entry->nt_cs = (s64)rq->nt_curr_runnable_sum; + __entry->nt_ps = (s64)rq->nt_prev_runnable_sum; + __entry->pid = p->pid; + ), + + TP_printk("cpu %d: cs %llu ps %llu nt_cs %lld nt_ps %lld pid %d", + __entry->cpu, __entry->cs, __entry->ps, + __entry->nt_cs, __entry->nt_ps, __entry->pid) +); +#endif /* CONFIG_SCHED_WALT */ + +#endif /* CONFIG_SMP */ + #endif /* _TRACE_SCHED_H */ /* This part must be outside protection */ diff --git a/include/trace/events/swiotlb.h b/include/trace/events/swiotlb.h index 7ea4c5e7c4487a..288c0c54a2b4ac 100644 --- a/include/trace/events/swiotlb.h +++ b/include/trace/events/swiotlb.h @@ -11,16 +11,16 @@ TRACE_EVENT(swiotlb_bounced, TP_PROTO(struct device *dev, dma_addr_t dev_addr, size_t size, - int swiotlb_force), + enum swiotlb_force swiotlb_force), TP_ARGS(dev, dev_addr, size, swiotlb_force), TP_STRUCT__entry( - __string( dev_name, dev_name(dev) ) - __field( u64, dma_mask ) - __field( dma_addr_t, dev_addr ) - __field( size_t, size ) - __field( int, swiotlb_force ) + __string( dev_name, dev_name(dev) ) + __field( u64, dma_mask ) + __field( dma_addr_t, dev_addr ) + __field( size_t, size ) + __field( enum swiotlb_force, swiotlb_force ) ), TP_fast_assign( @@ -37,7 +37,10 @@ TRACE_EVENT(swiotlb_bounced, __entry->dma_mask, (unsigned long long)__entry->dev_addr, __entry->size, - __entry->swiotlb_force ? "swiotlb_force" : "" ) + __print_symbolic(__entry->swiotlb_force, + { SWIOTLB_NORMAL, "NORMAL" }, + { SWIOTLB_FORCE, "FORCE" }, + { SWIOTLB_NO_FORCE, "NO_FORCE" })) ); #endif /* _TRACE_SWIOTLB_H */ diff --git a/include/uapi/linux/android/binder.h b/include/uapi/linux/android/binder.h index 41420e341e75de..51f891fb1b18ad 100644 --- a/include/uapi/linux/android/binder.h +++ b/include/uapi/linux/android/binder.h @@ -33,6 +33,8 @@ enum { BINDER_TYPE_HANDLE = B_PACK_CHARS('s', 'h', '*', B_TYPE_LARGE), BINDER_TYPE_WEAK_HANDLE = B_PACK_CHARS('w', 'h', '*', B_TYPE_LARGE), BINDER_TYPE_FD = B_PACK_CHARS('f', 'd', '*', B_TYPE_LARGE), + BINDER_TYPE_FDA = B_PACK_CHARS('f', 'd', 'a', B_TYPE_LARGE), + BINDER_TYPE_PTR = B_PACK_CHARS('p', 't', '*', B_TYPE_LARGE), }; enum { @@ -48,6 +50,14 @@ typedef __u64 binder_size_t; typedef __u64 binder_uintptr_t; #endif +/** + * struct binder_object_header - header shared by all binder metadata objects. + * @type: type of the object + */ +struct binder_object_header { + __u32 type; +}; + /* * This is the flattened representation of a Binder object for transfer * between processes. The 'offsets' supplied as part of a binder transaction @@ -56,9 +66,8 @@ typedef __u64 binder_uintptr_t; * between processes. */ struct flat_binder_object { - /* 8 bytes for large_flat_header. */ - __u32 type; - __u32 flags; + struct binder_object_header hdr; + __u32 flags; /* 8 bytes of data. */ union { @@ -70,6 +79,84 @@ struct flat_binder_object { binder_uintptr_t cookie; }; +/** + * struct binder_fd_object - describes a filedescriptor to be fixed up. + * @hdr: common header structure + * @pad_flags: padding to remain compatible with old userspace code + * @pad_binder: padding to remain compatible with old userspace code + * @fd: file descriptor + * @cookie: opaque data, used by user-space + */ +struct binder_fd_object { + struct binder_object_header hdr; + __u32 pad_flags; + union { + binder_uintptr_t pad_binder; + __u32 fd; + }; + + binder_uintptr_t cookie; +}; + +/* struct binder_buffer_object - object describing a userspace buffer + * @hdr: common header structure + * @flags: one or more BINDER_BUFFER_* flags + * @buffer: address of the buffer + * @length: length of the buffer + * @parent: index in offset array pointing to parent buffer + * @parent_offset: offset in @parent pointing to this buffer + * + * A binder_buffer object represents an object that the + * binder kernel driver can copy verbatim to the target + * address space. A buffer itself may be pointed to from + * within another buffer, meaning that the pointer inside + * that other buffer needs to be fixed up as well. This + * can be done by setting the BINDER_BUFFER_FLAG_HAS_PARENT + * flag in @flags, by setting @parent buffer to the index + * in the offset array pointing to the parent binder_buffer_object, + * and by setting @parent_offset to the offset in the parent buffer + * at which the pointer to this buffer is located. + */ +struct binder_buffer_object { + struct binder_object_header hdr; + __u32 flags; + binder_uintptr_t buffer; + binder_size_t length; + binder_size_t parent; + binder_size_t parent_offset; +}; + +enum { + BINDER_BUFFER_FLAG_HAS_PARENT = 0x01, +}; + +/* struct binder_fd_array_object - object describing an array of fds in a buffer + * @hdr: common header structure + * @num_fds: number of file descriptors in the buffer + * @parent: index in offset array to buffer holding the fd array + * @parent_offset: start offset of fd array in the buffer + * + * A binder_fd_array object represents an array of file + * descriptors embedded in a binder_buffer_object. It is + * different from a regular binder_buffer_object because it + * describes a list of file descriptors to fix up, not an opaque + * blob of memory, and hence the kernel needs to treat it differently. + * + * An example of how this would be used is with Android's + * native_handle_t object, which is a struct with a list of integers + * and a list of file descriptors. The native_handle_t struct itself + * will be represented by a struct binder_buffer_objct, whereas the + * embedded list of file descriptors is represented by a + * struct binder_fd_array_object with that binder_buffer_object as + * a parent. + */ +struct binder_fd_array_object { + struct binder_object_header hdr; + binder_size_t num_fds; + binder_size_t parent; + binder_size_t parent_offset; +}; + /* * On 64-bit platforms where user code may run in 32-bits the driver must * translate the buffer (and local binder) addresses appropriately. @@ -162,6 +249,11 @@ struct binder_transaction_data { } data; }; +struct binder_transaction_data_sg { + struct binder_transaction_data transaction_data; + binder_size_t buffers_size; +}; + struct binder_ptr_cookie { binder_uintptr_t ptr; binder_uintptr_t cookie; @@ -346,6 +438,12 @@ enum binder_driver_command_protocol { /* * void *: cookie */ + + BC_TRANSACTION_SG = _IOW('c', 17, struct binder_transaction_data_sg), + BC_REPLY_SG = _IOW('c', 18, struct binder_transaction_data_sg), + /* + * binder_transaction_data_sg: the sent command. + */ }; #endif /* _UAPI_LINUX_BINDER_H */ diff --git a/include/uapi/linux/fib_rules.h b/include/uapi/linux/fib_rules.h index 14404b3ebb89a5..bbf02a63a01133 100644 --- a/include/uapi/linux/fib_rules.h +++ b/include/uapi/linux/fib_rules.h @@ -29,6 +29,11 @@ struct fib_rule_hdr { __u32 flags; }; +struct fib_rule_uid_range { + __u32 start; + __u32 end; +}; + enum { FRA_UNSPEC, FRA_DST, /* destination address */ @@ -51,6 +56,7 @@ enum { FRA_OIFNAME, FRA_PAD, FRA_L3MDEV, /* iif or oif is l3mdev goto its table */ + FRA_UID_RANGE, /* UID range */ __FRA_MAX }; diff --git a/include/uapi/linux/fs.h b/include/uapi/linux/fs.h index acb2b6152ba012..92d9c68c696c6b 100644 --- a/include/uapi/linux/fs.h +++ b/include/uapi/linux/fs.h @@ -236,6 +236,8 @@ struct fsxattr { #define FICLONERANGE _IOW(0x94, 13, struct file_clone_range) #define FIDEDUPERANGE _IOWR(0x94, 54, struct file_dedupe_range) +#define FIDTRIM _IOWR('f', 128, struct fstrim_range) /* Deep discard trim */ + #define FS_IOC_GETFLAGS _IOR('f', 1, long) #define FS_IOC_SETFLAGS _IOW('f', 2, long) #define FS_IOC_GETVERSION _IOR('v', 1, long) diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h index 42fa977e3b14b3..093237817ed0b2 100644 --- a/include/uapi/linux/fuse.h +++ b/include/uapi/linux/fuse.h @@ -375,6 +375,7 @@ enum fuse_opcode { FUSE_READDIRPLUS = 44, FUSE_RENAME2 = 45, FUSE_LSEEK = 46, + FUSE_CANONICAL_PATH= 2016, /* CUSE specific operations */ CUSE_INIT = 4096, diff --git a/include/uapi/linux/if_pppolac.h b/include/uapi/linux/if_pppolac.h new file mode 100644 index 00000000000000..b7eb8153ef666e --- /dev/null +++ b/include/uapi/linux/if_pppolac.h @@ -0,0 +1,33 @@ +/* include/uapi/linux/if_pppolac.h + * + * Header for PPP on L2TP Access Concentrator / PPPoLAC Socket (RFC 2661) + * + * Copyright (C) 2009 Google, Inc. + * Author: Chia-chi Yeh + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _UAPI_LINUX_IF_PPPOLAC_H +#define _UAPI_LINUX_IF_PPPOLAC_H + +#include +#include + +struct sockaddr_pppolac { + sa_family_t sa_family; /* AF_PPPOX */ + unsigned int sa_protocol; /* PX_PROTO_OLAC */ + int udp_socket; + struct __attribute__((packed)) { + __u16 tunnel, session; + } local, remote; +} __attribute__((packed)); + +#endif /* _UAPI_LINUX_IF_PPPOLAC_H */ diff --git a/include/uapi/linux/if_pppopns.h b/include/uapi/linux/if_pppopns.h new file mode 100644 index 00000000000000..a392b52ea6ecfb --- /dev/null +++ b/include/uapi/linux/if_pppopns.h @@ -0,0 +1,32 @@ +/* include/uapi/linux/if_pppopns.h + * + * Header for PPP on PPTP Network Server / PPPoPNS Socket (RFC 2637) + * + * Copyright (C) 2009 Google, Inc. + * Author: Chia-chi Yeh + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _UAPI_LINUX_IF_PPPOPNS_H +#define _UAPI_LINUX_IF_PPPOPNS_H + +#include +#include + +struct sockaddr_pppopns { + sa_family_t sa_family; /* AF_PPPOX */ + unsigned int sa_protocol; /* PX_PROTO_OPNS */ + int tcp_socket; + __u16 local; + __u16 remote; +} __attribute__((packed)); + +#endif /* _UAPI_LINUX_IF_PPPOPNS_H */ diff --git a/include/uapi/linux/if_pppox.h b/include/uapi/linux/if_pppox.h index d37bbb17a007c1..6aad18a517d3ea 100644 --- a/include/uapi/linux/if_pppox.h +++ b/include/uapi/linux/if_pppox.h @@ -24,6 +24,8 @@ #include #include #include +#include +#include #include #include @@ -59,7 +61,9 @@ struct pptp_addr { #define PX_PROTO_OE 0 /* Currently just PPPoE */ #define PX_PROTO_OL2TP 1 /* Now L2TP also */ #define PX_PROTO_PPTP 2 -#define PX_MAX_PROTO 3 +#define PX_PROTO_OLAC 3 +#define PX_PROTO_OPNS 4 +#define PX_MAX_PROTO 5 struct sockaddr_pppox { __kernel_sa_family_t sa_family; /* address family, AF_PPPOX */ diff --git a/include/uapi/linux/ipv6.h b/include/uapi/linux/ipv6.h index 8c2772340c3f1c..1049c7828141d5 100644 --- a/include/uapi/linux/ipv6.h +++ b/include/uapi/linux/ipv6.h @@ -164,6 +164,7 @@ enum { DEVCONF_ACCEPT_DAD, DEVCONF_FORCE_TLLAO, DEVCONF_NDISC_NOTIFY, + DEVCONF_ACCEPT_RA_RT_TABLE, DEVCONF_MLDV1_UNSOLICITED_REPORT_INTERVAL, DEVCONF_MLDV2_UNSOLICITED_REPORT_INTERVAL, DEVCONF_SUPPRESS_FRAG_NDISC, diff --git a/include/uapi/linux/keychord.h b/include/uapi/linux/keychord.h new file mode 100644 index 00000000000000..ea7cf4d27bbdf1 --- /dev/null +++ b/include/uapi/linux/keychord.h @@ -0,0 +1,52 @@ +/* + * Key chord input driver + * + * Copyright (C) 2008 Google, Inc. + * Author: Mike Lockwood + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * +*/ + +#ifndef _UAPI_LINUX_KEYCHORD_H_ +#define _UAPI_LINUX_KEYCHORD_H_ + +#include + +#define KEYCHORD_VERSION 1 + +/* + * One or more input_keychord structs are written to /dev/keychord + * at once to specify the list of keychords to monitor. + * Reading /dev/keychord returns the id of a keychord when the + * keychord combination is pressed. A keychord is signalled when + * all of the keys in the keycode list are in the pressed state. + * The order in which the keys are pressed does not matter. + * The keychord will not be signalled if keys not in the keycode + * list are pressed. + * Keychords will not be signalled on key release events. + */ +struct input_keychord { + /* should be KEYCHORD_VERSION */ + __u16 version; + /* + * client specified ID, returned from read() + * when this keychord is pressed. + */ + __u16 id; + + /* number of keycodes in this keychord */ + __u16 count; + + /* variable length array of keycodes */ + __u16 keycodes[]; +}; + +#endif /* _UAPI_LINUX_KEYCHORD_H_ */ diff --git a/include/uapi/linux/magic.h b/include/uapi/linux/magic.h index 9bd559472c9280..bd017699420e22 100644 --- a/include/uapi/linux/magic.h +++ b/include/uapi/linux/magic.h @@ -53,6 +53,8 @@ #define REISER2FS_SUPER_MAGIC_STRING "ReIsEr2Fs" #define REISER2FS_JR_SUPER_MAGIC_STRING "ReIsEr3Fs" +#define SDCARDFS_SUPER_MAGIC 0x5dca2df5 + #define SMB_SUPER_MAGIC 0x517B #define CGROUP_SUPER_MAGIC 0x27e0eb #define CGROUP2_SUPER_MAGIC 0x63677270 diff --git a/include/uapi/linux/netfilter/xt_IDLETIMER.h b/include/uapi/linux/netfilter/xt_IDLETIMER.h index 208ae938733143..faaa28b3d0613e 100644 --- a/include/uapi/linux/netfilter/xt_IDLETIMER.h +++ b/include/uapi/linux/netfilter/xt_IDLETIMER.h @@ -4,6 +4,7 @@ * Header file for Xtables timer target module. * * Copyright (C) 2004, 2010 Nokia Corporation + * * Written by Timo Teras * * Converted to x_tables and forward-ported to 2.6.34 @@ -32,12 +33,19 @@ #include #define MAX_IDLETIMER_LABEL_SIZE 28 +#define NLMSG_MAX_SIZE 64 + +#define NL_EVENT_TYPE_INACTIVE 0 +#define NL_EVENT_TYPE_ACTIVE 1 struct idletimer_tg_info { __u32 timeout; char label[MAX_IDLETIMER_LABEL_SIZE]; + /* Use netlink messages for notification in addition to sysfs */ + __u8 send_nl_msg; + /* for kernel module internal use only */ struct idletimer_tg *timer __attribute__((aligned(8))); }; diff --git a/include/uapi/linux/netfilter/xt_socket.h b/include/uapi/linux/netfilter/xt_socket.h index 87644f83249499..7f00df6cd89752 100644 --- a/include/uapi/linux/netfilter/xt_socket.h +++ b/include/uapi/linux/netfilter/xt_socket.h @@ -26,4 +26,11 @@ struct xt_socket_mtinfo3 { | XT_SOCKET_NOWILDCARD \ | XT_SOCKET_RESTORESKMARK) +struct sock *xt_socket_lookup_slow_v4(struct net *net, + const struct sk_buff *skb, + const struct net_device *indev); +struct sock *xt_socket_lookup_slow_v6(struct net *net, + const struct sk_buff *skb, + const struct net_device *indev); + #endif /* _XT_SOCKET_H */ diff --git a/include/uapi/linux/nl80211.h b/include/uapi/linux/nl80211.h index 56368e9b462245..d3cbe48b286d8c 100644 --- a/include/uapi/linux/nl80211.h +++ b/include/uapi/linux/nl80211.h @@ -323,7 +323,7 @@ * @NL80211_CMD_GET_SCAN: get scan results * @NL80211_CMD_TRIGGER_SCAN: trigger a new scan with the given parameters * %NL80211_ATTR_TX_NO_CCK_RATE is used to decide whether to send the - * probe requests at CCK rate or not. %NL80211_ATTR_MAC can be used to + * probe requests at CCK rate or not. %NL80211_ATTR_BSSID can be used to * specify a BSSID to scan for; if not included, the wildcard BSSID will * be used. * @NL80211_CMD_NEW_SCAN_RESULTS: scan notification (as a reply to @@ -1937,6 +1937,9 @@ enum nl80211_commands { * @NL80211_ATTR_NAN_MATCH: used to report a match. This is a nested attribute. * See &enum nl80211_nan_match_attributes. * + * @NL80211_ATTR_BSSID: The BSSID of the AP. Note that %NL80211_ATTR_MAC is also + * used in various commands/events for specifying the BSSID. + * * @NUM_NL80211_ATTR: total number of nl80211_attrs available * @NL80211_ATTR_MAX: highest attribute number currently defined * @__NL80211_ATTR_AFTER_LAST: internal use @@ -2336,6 +2339,8 @@ enum nl80211_attrs { NL80211_ATTR_NAN_FUNC, NL80211_ATTR_NAN_MATCH, + NL80211_ATTR_BSSID, + /* add attributes here, update the policy in nl80211.c */ __NL80211_ATTR_AFTER_LAST, diff --git a/include/uapi/linux/prctl.h b/include/uapi/linux/prctl.h index a8d0759a9e400c..96b269bcfa17f6 100644 --- a/include/uapi/linux/prctl.h +++ b/include/uapi/linux/prctl.h @@ -197,4 +197,7 @@ struct prctl_mm_map { # define PR_CAP_AMBIENT_LOWER 3 # define PR_CAP_AMBIENT_CLEAR_ALL 4 +#define PR_SET_VMA 0x53564d41 +# define PR_SET_VMA_ANON_NAME 0 + #endif /* _LINUX_PRCTL_H */ diff --git a/include/uapi/linux/rtnetlink.h b/include/uapi/linux/rtnetlink.h index 5a78be51810123..e14377f2ec273a 100644 --- a/include/uapi/linux/rtnetlink.h +++ b/include/uapi/linux/rtnetlink.h @@ -318,6 +318,7 @@ enum rtattr_type_t { RTA_ENCAP, RTA_EXPIRES, RTA_PAD, + RTA_UID, __RTA_MAX }; diff --git a/include/uapi/linux/tee.h b/include/uapi/linux/tee.h new file mode 100644 index 00000000000000..35f5b95216d0c6 --- /dev/null +++ b/include/uapi/linux/tee.h @@ -0,0 +1,440 @@ +/* + * Copyright (c) 2015-2016, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __TEE_H +#define __TEE_H + +#include +#include + +/* + * This file describes the API provided by a TEE driver to user space. + * + * Each TEE driver defines a TEE specific protocol which is used for the + * data passed back and forth using TEE_IOC_CMD. + */ + +/* Helpers to make the ioctl defines */ +#define TEE_IOC_MAGIC 0xa4 +#define TEE_IOC_BASE 0 + +/* Flags relating to shared memory */ +#define TEE_IOCTL_SHM_MAPPED 0x1 /* memory mapped in normal world */ +#define TEE_IOCTL_SHM_DMA_BUF 0x2 /* dma-buf handle on shared memory */ + +#define TEE_MAX_ARG_SIZE 1024 + +#define TEE_GEN_CAP_GP (1 << 0)/* GlobalPlatform compliant TEE */ +#define TEE_GEN_CAP_PRIVILEGED (1 << 1)/* Privileged device (for supplicant) */ + +/* + * TEE Implementation ID + */ +#define TEE_IMPL_ID_OPTEE 1 + +/* + * OP-TEE specific capabilities + */ +#define TEE_OPTEE_CAP_TZ (1 << 0) + +/** + * struct tee_ioctl_version_data - TEE version + * @impl_id: [out] TEE implementation id + * @impl_caps: [out] Implementation specific capabilities + * @gen_caps: [out] Generic capabilities, defined by TEE_GEN_CAPS_* above + * + * Identifies the TEE implementation, @impl_id is one of TEE_IMPL_ID_* above. + * @impl_caps is implementation specific, for example TEE_OPTEE_CAP_* + * is valid when @impl_id == TEE_IMPL_ID_OPTEE. + */ +struct tee_ioctl_version_data { + __u32 impl_id; + __u32 impl_caps; + __u32 gen_caps; +}; + +/** + * TEE_IOC_VERSION - query version of TEE + * + * Takes a tee_ioctl_version_data struct and returns with the TEE version + * data filled in. + */ +#define TEE_IOC_VERSION _IOR(TEE_IOC_MAGIC, TEE_IOC_BASE + 0, \ + struct tee_ioctl_version_data) + +/** + * struct tee_ioctl_shm_alloc_data - Shared memory allocate argument + * @size: [in/out] Size of shared memory to allocate + * @flags: [in/out] Flags to/from allocation. + * @id: [out] Identifier of the shared memory + * + * The flags field should currently be zero as input. Updated by the call + * with actual flags as defined by TEE_IOCTL_SHM_* above. + * This structure is used as argument for TEE_IOC_SHM_ALLOC below. + */ +struct tee_ioctl_shm_alloc_data { + __u64 size; + __u32 flags; + __s32 id; +}; + +/** + * TEE_IOC_SHM_ALLOC - allocate shared memory + * + * Allocates shared memory between the user space process and secure OS. + * + * Returns a file descriptor on success or < 0 on failure + * + * The returned file descriptor is used to map the shared memory into user + * space. The shared memory is freed when the descriptor is closed and the + * memory is unmapped. + */ +#define TEE_IOC_SHM_ALLOC _IOWR(TEE_IOC_MAGIC, TEE_IOC_BASE + 1, \ + struct tee_ioctl_shm_alloc_data) + +/** + * struct tee_ioctl_shm_register_fd_data - Shared memory registering argument + * @fd: [in] file descriptor identifying the shared memory + * @size: [out] Size of shared memory to allocate + * @flags: [in] Flags to/from allocation. + * @id: [out] Identifier of the shared memory + * + * The flags field should currently be zero as input. Updated by the call + * with actual flags as defined by TEE_IOCTL_SHM_* above. + * This structure is used as argument for TEE_IOC_SHM_ALLOC below. + */ +struct tee_ioctl_shm_register_fd_data { + __s64 fd; + __u64 size; + __u32 flags; + __s32 id; +} __aligned(8); + +/** + * TEE_IOC_SHM_REGISTER_FD - register a shared memory from a file descriptor + * + * Returns a file descriptor on success or < 0 on failure + * + * The returned file descriptor refers to the shared memory object in kernel + * land. The shared memory is freed when the descriptor is closed. + */ +#define TEE_IOC_SHM_REGISTER_FD _IOWR(TEE_IOC_MAGIC, TEE_IOC_BASE + 8, \ + struct tee_ioctl_shm_register_fd_data) + +/** + * struct tee_ioctl_buf_data - Variable sized buffer + * @buf_ptr: [in] A __user pointer to a buffer + * @buf_len: [in] Length of the buffer above + * + * Used as argument for TEE_IOC_OPEN_SESSION, TEE_IOC_INVOKE, + * TEE_IOC_SUPPL_RECV, and TEE_IOC_SUPPL_SEND below. + */ +struct tee_ioctl_buf_data { + __u64 buf_ptr; + __u64 buf_len; +}; + +/* + * Attributes for struct tee_ioctl_param, selects field in the union + */ +#define TEE_IOCTL_PARAM_ATTR_TYPE_NONE 0 /* parameter not used */ + +/* + * These defines value parameters (struct tee_ioctl_param_value) + */ +#define TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT 1 +#define TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_OUTPUT 2 +#define TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INOUT 3 /* input and output */ + +/* + * These defines shared memory reference parameters (struct + * tee_ioctl_param_memref) + */ +#define TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT 5 +#define TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_OUTPUT 6 +#define TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INOUT 7 /* input and output */ + +/* + * Mask for the type part of the attribute, leaves room for more types + */ +#define TEE_IOCTL_PARAM_ATTR_TYPE_MASK 0xff + +/* Meta parameter carrying extra information about the message. */ +#define TEE_IOCTL_PARAM_ATTR_META 0x100 + +/* Mask of all known attr bits */ +#define TEE_IOCTL_PARAM_ATTR_MASK \ + (TEE_IOCTL_PARAM_ATTR_TYPE_MASK | TEE_IOCTL_PARAM_ATTR_META) + +/* + * Matches TEEC_LOGIN_* in GP TEE Client API + * Are only defined for GP compliant TEEs + */ +#define TEE_IOCTL_LOGIN_PUBLIC 0 +#define TEE_IOCTL_LOGIN_USER 1 +#define TEE_IOCTL_LOGIN_GROUP 2 +#define TEE_IOCTL_LOGIN_APPLICATION 4 +#define TEE_IOCTL_LOGIN_USER_APPLICATION 5 +#define TEE_IOCTL_LOGIN_GROUP_APPLICATION 6 + +/** + * struct tee_ioctl_param_memref - memory reference + * @shm_offs: Offset into the shared memory object + * @size: Size of the buffer + * @shm_id: Shared memory identifier + * + * Shared memory is allocated with TEE_IOC_SHM_ALLOC which returns an + * identifier representing the shared memory object. A memref can reference + * a part of a shared memory by specifying an offset (@shm_offs) and @size + * of the object. To supply the entire shared memory object set @shm_offs + * to 0 and @size to the previously returned size of the object. + */ +struct tee_ioctl_param_memref { + __u64 shm_offs; + __u64 size; + __s64 shm_id; +}; + +/** + * struct tee_ioctl_param_value - values + * @a: first value + * @b: second value + * @c: third value + * + * Value parameters are passed unchecked to the destination + */ +struct tee_ioctl_param_value { + __u64 a; + __u64 b; + __u64 c; +}; + +/** + * struct tee_ioctl_param - parameter + * @attr: attributes + * @memref: a memory reference + * @value: a value + * + * @attr & TEE_PARAM_ATTR_TYPE_MASK indicates if memref or value is used in + * the union. TEE_PARAM_ATTR_TYPE_VALUE_* indicates value and + * TEE_PARAM_ATTR_TYPE_MEMREF_* indicates memref. TEE_PARAM_ATTR_TYPE_NONE + * indicates that none of the members are used. + */ +struct tee_ioctl_param { + __u64 attr; + union { + struct tee_ioctl_param_memref memref; + struct tee_ioctl_param_value value; + } u; +}; + +#define TEE_IOCTL_UUID_LEN 16 + +/** + * struct tee_ioctl_open_session_arg - Open session argument + * @uuid: [in] UUID of the Trusted Application + * @clnt_uuid: [in] UUID of client + * @clnt_login: [in] Login class of client, TEE_IOCTL_LOGIN_* above + * @cancel_id: [in] Cancellation id, a unique value to identify this request + * @session: [out] Session id + * @ret: [out] return value + * @ret_origin [out] origin of the return value + * @num_params [in] number of parameters following this struct + */ +struct tee_ioctl_open_session_arg { + __u8 uuid[TEE_IOCTL_UUID_LEN]; + __u8 clnt_uuid[TEE_IOCTL_UUID_LEN]; + __u32 clnt_login; + __u32 cancel_id; + __u32 session; + __u32 ret; + __u32 ret_origin; + __u32 num_params; + /* + * this struct is 8 byte aligned since the 'struct tee_ioctl_param' + * which follows requires 8 byte alignment. + * + * Commented out element used to visualize the layout dynamic part + * of the struct. This field is not available at all if + * num_params == 0. + * + * struct tee_ioctl_param params[num_params]; + */ +} __aligned(8); + +/** + * TEE_IOC_OPEN_SESSION - opens a session to a Trusted Application + * + * Takes a struct tee_ioctl_buf_data which contains a struct + * tee_ioctl_open_session_arg followed by any array of struct + * tee_ioctl_param + */ +#define TEE_IOC_OPEN_SESSION _IOR(TEE_IOC_MAGIC, TEE_IOC_BASE + 2, \ + struct tee_ioctl_buf_data) + +/** + * struct tee_ioctl_invoke_func_arg - Invokes a function in a Trusted + * Application + * @func: [in] Trusted Application function, specific to the TA + * @session: [in] Session id + * @cancel_id: [in] Cancellation id, a unique value to identify this request + * @ret: [out] return value + * @ret_origin [out] origin of the return value + * @num_params [in] number of parameters following this struct + */ +struct tee_ioctl_invoke_arg { + __u32 func; + __u32 session; + __u32 cancel_id; + __u32 ret; + __u32 ret_origin; + __u32 num_params; + /* + * this struct is 8 byte aligned since the 'struct tee_ioctl_param' + * which follows requires 8 byte alignment. + * + * Commented out element used to visualize the layout dynamic part + * of the struct. This field is not available at all if + * num_params == 0. + * + * struct tee_ioctl_param params[num_params]; + */ +} __aligned(8); + +/** + * TEE_IOC_INVOKE - Invokes a function in a Trusted Application + * + * Takes a struct tee_ioctl_buf_data which contains a struct + * tee_invoke_func_arg followed by any array of struct tee_param + */ +#define TEE_IOC_INVOKE _IOR(TEE_IOC_MAGIC, TEE_IOC_BASE + 3, \ + struct tee_ioctl_buf_data) + +/** + * struct tee_ioctl_cancel_arg - Cancels an open session or invoke ioctl + * @cancel_id: [in] Cancellation id, a unique value to identify this request + * @session: [in] Session id, if the session is opened, else set to 0 + */ +struct tee_ioctl_cancel_arg { + __u32 cancel_id; + __u32 session; +}; + +/** + * TEE_IOC_CANCEL - Cancels an open session or invoke + */ +#define TEE_IOC_CANCEL _IOR(TEE_IOC_MAGIC, TEE_IOC_BASE + 4, \ + struct tee_ioctl_cancel_arg) + +/** + * struct tee_ioctl_close_session_arg - Closes an open session + * @session: [in] Session id + */ +struct tee_ioctl_close_session_arg { + __u32 session; +}; + +/** + * TEE_IOC_CLOSE_SESSION - Closes a session + */ +#define TEE_IOC_CLOSE_SESSION _IOR(TEE_IOC_MAGIC, TEE_IOC_BASE + 5, \ + struct tee_ioctl_close_session_arg) + +/** + * struct tee_iocl_supp_recv_arg - Receive a request for a supplicant function + * @func: [in] supplicant function + * @num_params [in/out] number of parameters following this struct + * + * @num_params is the number of params that tee-supplicant has room to + * receive when input, @num_params is the number of actual params + * tee-supplicant receives when output. + */ +struct tee_iocl_supp_recv_arg { + __u32 func; + __u32 num_params; + /* + * this struct is 8 byte aligned since the 'struct tee_ioctl_param' + * which follows requires 8 byte alignment. + * + * Commented out element used to visualize the layout dynamic part + * of the struct. This field is not available at all if + * num_params == 0. + * + * struct tee_ioctl_param params[num_params]; + */ +} __aligned(8); + +/** + * TEE_IOC_SUPPL_RECV - Receive a request for a supplicant function + * + * Takes a struct tee_ioctl_buf_data which contains a struct + * tee_iocl_supp_recv_arg followed by any array of struct tee_param + */ +#define TEE_IOC_SUPPL_RECV _IOR(TEE_IOC_MAGIC, TEE_IOC_BASE + 6, \ + struct tee_ioctl_buf_data) + +/** + * struct tee_iocl_supp_send_arg - Send a response to a received request + * @ret: [out] return value + * @num_params [in] number of parameters following this struct + */ +struct tee_iocl_supp_send_arg { + __u32 ret; + __u32 num_params; + /* + * this struct is 8 byte aligned since the 'struct tee_ioctl_param' + * which follows requires 8 byte alignment. + * + * Commented out element used to visualize the layout dynamic part + * of the struct. This field is not available at all if + * num_params == 0. + * + * struct tee_ioctl_param params[num_params]; + */ +} __aligned(8); +/** + * TEE_IOC_SUPPL_SEND - Receive a request for a supplicant function + * + * Takes a struct tee_ioctl_buf_data which contains a struct + * tee_iocl_supp_send_arg followed by any array of struct tee_param + */ +#define TEE_IOC_SUPPL_SEND _IOR(TEE_IOC_MAGIC, TEE_IOC_BASE + 7, \ + struct tee_ioctl_buf_data) + +/* + * Five syscalls are used when communicating with the TEE driver. + * open(): opens the device associated with the driver + * ioctl(): as described above operating on the file descriptor from open() + * close(): two cases + * - closes the device file descriptor + * - closes a file descriptor connected to allocated shared memory + * mmap(): maps shared memory into user space using information from struct + * tee_ioctl_shm_alloc_data + * munmap(): unmaps previously shared memory + */ + +#endif /*__TEE_H*/ diff --git a/include/uapi/linux/usb/f_accessory.h b/include/uapi/linux/usb/f_accessory.h new file mode 100644 index 00000000000000..0baeb7d0d74ce4 --- /dev/null +++ b/include/uapi/linux/usb/f_accessory.h @@ -0,0 +1,146 @@ +/* + * Gadget Function Driver for Android USB accessories + * + * Copyright (C) 2011 Google, Inc. + * Author: Mike Lockwood + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#ifndef _UAPI_LINUX_USB_F_ACCESSORY_H +#define _UAPI_LINUX_USB_F_ACCESSORY_H + +/* Use Google Vendor ID when in accessory mode */ +#define USB_ACCESSORY_VENDOR_ID 0x18D1 + + +/* Product ID to use when in accessory mode */ +#define USB_ACCESSORY_PRODUCT_ID 0x2D00 + +/* Product ID to use when in accessory mode and adb is enabled */ +#define USB_ACCESSORY_ADB_PRODUCT_ID 0x2D01 + +/* Indexes for strings sent by the host via ACCESSORY_SEND_STRING */ +#define ACCESSORY_STRING_MANUFACTURER 0 +#define ACCESSORY_STRING_MODEL 1 +#define ACCESSORY_STRING_DESCRIPTION 2 +#define ACCESSORY_STRING_VERSION 3 +#define ACCESSORY_STRING_URI 4 +#define ACCESSORY_STRING_SERIAL 5 + +/* Control request for retrieving device's protocol version + * + * requestType: USB_DIR_IN | USB_TYPE_VENDOR + * request: ACCESSORY_GET_PROTOCOL + * value: 0 + * index: 0 + * data version number (16 bits little endian) + * 1 for original accessory support + * 2 adds HID and device to host audio support + */ +#define ACCESSORY_GET_PROTOCOL 51 + +/* Control request for host to send a string to the device + * + * requestType: USB_DIR_OUT | USB_TYPE_VENDOR + * request: ACCESSORY_SEND_STRING + * value: 0 + * index: string ID + * data zero terminated UTF8 string + * + * The device can later retrieve these strings via the + * ACCESSORY_GET_STRING_* ioctls + */ +#define ACCESSORY_SEND_STRING 52 + +/* Control request for starting device in accessory mode. + * The host sends this after setting all its strings to the device. + * + * requestType: USB_DIR_OUT | USB_TYPE_VENDOR + * request: ACCESSORY_START + * value: 0 + * index: 0 + * data none + */ +#define ACCESSORY_START 53 + +/* Control request for registering a HID device. + * Upon registering, a unique ID is sent by the accessory in the + * value parameter. This ID will be used for future commands for + * the device + * + * requestType: USB_DIR_OUT | USB_TYPE_VENDOR + * request: ACCESSORY_REGISTER_HID_DEVICE + * value: Accessory assigned ID for the HID device + * index: total length of the HID report descriptor + * data none + */ +#define ACCESSORY_REGISTER_HID 54 + +/* Control request for unregistering a HID device. + * + * requestType: USB_DIR_OUT | USB_TYPE_VENDOR + * request: ACCESSORY_REGISTER_HID + * value: Accessory assigned ID for the HID device + * index: 0 + * data none + */ +#define ACCESSORY_UNREGISTER_HID 55 + +/* Control request for sending the HID report descriptor. + * If the HID descriptor is longer than the endpoint zero max packet size, + * the descriptor will be sent in multiple ACCESSORY_SET_HID_REPORT_DESC + * commands. The data for the descriptor must be sent sequentially + * if multiple packets are needed. + * + * requestType: USB_DIR_OUT | USB_TYPE_VENDOR + * request: ACCESSORY_SET_HID_REPORT_DESC + * value: Accessory assigned ID for the HID device + * index: offset of data in descriptor + * (needed when HID descriptor is too big for one packet) + * data the HID report descriptor + */ +#define ACCESSORY_SET_HID_REPORT_DESC 56 + +/* Control request for sending HID events. + * + * requestType: USB_DIR_OUT | USB_TYPE_VENDOR + * request: ACCESSORY_SEND_HID_EVENT + * value: Accessory assigned ID for the HID device + * index: 0 + * data the HID report for the event + */ +#define ACCESSORY_SEND_HID_EVENT 57 + +/* Control request for setting the audio mode. + * + * requestType: USB_DIR_OUT | USB_TYPE_VENDOR + * request: ACCESSORY_SET_AUDIO_MODE + * value: 0 - no audio + * 1 - device to host, 44100 16-bit stereo PCM + * index: 0 + * data none + */ +#define ACCESSORY_SET_AUDIO_MODE 58 + +/* ioctls for retrieving strings set by the host */ +#define ACCESSORY_GET_STRING_MANUFACTURER _IOW('M', 1, char[256]) +#define ACCESSORY_GET_STRING_MODEL _IOW('M', 2, char[256]) +#define ACCESSORY_GET_STRING_DESCRIPTION _IOW('M', 3, char[256]) +#define ACCESSORY_GET_STRING_VERSION _IOW('M', 4, char[256]) +#define ACCESSORY_GET_STRING_URI _IOW('M', 5, char[256]) +#define ACCESSORY_GET_STRING_SERIAL _IOW('M', 6, char[256]) +/* returns 1 if there is a start request pending */ +#define ACCESSORY_IS_START_REQUESTED _IO('M', 7) +/* returns audio mode (set via the ACCESSORY_SET_AUDIO_MODE control request) */ +#define ACCESSORY_GET_AUDIO_MODE _IO('M', 8) + +#endif /* _UAPI_LINUX_USB_F_ACCESSORY_H */ diff --git a/include/uapi/linux/usb/f_mtp.h b/include/uapi/linux/usb/f_mtp.h new file mode 100644 index 00000000000000..503291855abda2 --- /dev/null +++ b/include/uapi/linux/usb/f_mtp.h @@ -0,0 +1,61 @@ +/* + * Gadget Function Driver for MTP + * + * Copyright (C) 2010 Google, Inc. + * Author: Mike Lockwood + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#ifndef _UAPI_LINUX_USB_F_MTP_H +#define _UAPI_LINUX_USB_F_MTP_H + +#include +#include + +struct mtp_file_range { + /* file descriptor for file to transfer */ + int fd; + /* offset in file for start of transfer */ + loff_t offset; + /* number of bytes to transfer */ + int64_t length; + /* MTP command ID for data header, + * used only for MTP_SEND_FILE_WITH_HEADER + */ + uint16_t command; + /* MTP transaction ID for data header, + * used only for MTP_SEND_FILE_WITH_HEADER + */ + uint32_t transaction_id; +}; + +struct mtp_event { + /* size of the event */ + size_t length; + /* event data to send */ + void *data; +}; + +/* Sends the specified file range to the host */ +#define MTP_SEND_FILE _IOW('M', 0, struct mtp_file_range) +/* Receives data from the host and writes it to a file. + * The file is created if it does not exist. + */ +#define MTP_RECEIVE_FILE _IOW('M', 1, struct mtp_file_range) +/* Sends an event to the host via the interrupt endpoint */ +#define MTP_SEND_EVENT _IOW('M', 3, struct mtp_event) +/* Sends the specified file range to the host, + * with a 12 byte MTP data packet header at the beginning. + */ +#define MTP_SEND_FILE_WITH_HEADER _IOW('M', 4, struct mtp_file_range) + +#endif /* _UAPI_LINUX_USB_F_MTP_H */ diff --git a/include/uapi/rdma/cxgb3-abi.h b/include/uapi/rdma/cxgb3-abi.h index 48a19bda071b8d..d24eee12128fc5 100644 --- a/include/uapi/rdma/cxgb3-abi.h +++ b/include/uapi/rdma/cxgb3-abi.h @@ -30,7 +30,7 @@ * SOFTWARE. */ #ifndef CXGB3_ABI_USER_H -#define CXBG3_ABI_USER_H +#define CXGB3_ABI_USER_H #include diff --git a/init/Kconfig b/init/Kconfig index 34407f15e6d34d..2e15b932aeeae2 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -400,6 +400,15 @@ config IRQ_TIME_ACCOUNTING If in doubt, say N here. +config SCHED_WALT + bool "Support window based load tracking" + depends on SMP + help + This feature will allow the scheduler to maintain a tunable window + based set of metrics for tasks and runqueues. These metrics can be + used to guide task placement as well as task frequency requirements + for cpufreq governors. + config BSD_PROCESS_ACCT bool "BSD Process Accounting" depends on MULTIUSER @@ -971,6 +980,82 @@ menuconfig CGROUPS if CGROUPS +config CGROUP_DEBUG + bool "Example debug cgroup subsystem" + default n + help + This option enables a simple cgroup subsystem that + exports useful debugging information about the cgroups + framework. + + Say N if unsure. + +config CGROUP_FREEZER + bool "Freezer cgroup subsystem" + help + Provides a way to freeze and unfreeze all tasks in a + cgroup. + +config CGROUP_PIDS + bool "PIDs cgroup subsystem" + help + Provides enforcement of process number limits in the scope of a + cgroup. Any attempt to fork more processes than is allowed in the + cgroup will fail. PIDs are fundamentally a global resource because it + is fairly trivial to reach PID exhaustion before you reach even a + conservative kmemcg limit. As a result, it is possible to grind a + system to halt without being limited by other cgroup policies. The + PIDs cgroup subsystem is designed to stop this from happening. + + It should be noted that organisational operations (such as attaching + to a cgroup hierarchy will *not* be blocked by the PIDs subsystem), + since the PIDs limit only affects a process's ability to fork, not to + attach to a cgroup. + +config CGROUP_DEVICE + bool "Device controller for cgroups" + help + Provides a cgroup implementing whitelists for devices which + a process in the cgroup can mknod or open. + +config CPUSETS + bool "Cpuset support" + help + This option will let you create and manage CPUSETs which + allow dynamically partitioning a system into sets of CPUs and + Memory Nodes and assigning tasks to run only within those sets. + This is primarily useful on large SMP or NUMA systems. + + Say N if unsure. + +config PROC_PID_CPUSET + bool "Include legacy /proc//cpuset file" + depends on CPUSETS + default y + +config CGROUP_CPUACCT + bool "Simple CPU accounting cgroup subsystem" + help + Provides a simple Resource Controller for monitoring the + total CPU consumed by the tasks in a cgroup. + +config CGROUP_SCHEDTUNE + bool "CFS tasks boosting cgroup subsystem (EXPERIMENTAL)" + depends on SCHED_TUNE + help + This option provides the "schedtune" controller which improves the + flexibility of the task boosting mechanism by introducing the support + to define "per task" boost values. + + This new controller: + 1. allows only a two layers hierarchy, where the root defines the + system-wide boost value and its direct childrens define each one a + different "class of tasks" to be boosted with a different value + 2. supports up to 16 different task classes, each one which could be + configured with a different boost value + + Say N if unsure. + config PAGE_COUNTER bool @@ -1248,6 +1333,43 @@ config SCHED_AUTOGROUP desktop applications. Task group autogeneration is currently based upon task session. +config SCHED_TUNE + bool "Boosting for CFS tasks (EXPERIMENTAL)" + depends on SMP + help + This option enables the system-wide support for task boosting. + When this support is enabled a new sysctl interface is exposed to + userspace via: + /proc/sys/kernel/sched_cfs_boost + which allows to set a system-wide boost value in range [0..100]. + + The currently boosting strategy is implemented in such a way that: + - a 0% boost value requires to operate in "standard" mode by + scheduling all tasks at the minimum capacities required by their + workload demand + - a 100% boost value requires to push at maximum the task + performances, "regardless" of the incurred energy consumption + + A boost value in between these two boundaries is used to bias the + power/performance trade-off, the higher the boost value the more the + scheduler is biased toward performance boosting instead of energy + efficiency. + + Since this support exposes a single system-wide knob, the specified + boost value is applied to all (CFS) tasks in the system. + + If unsure, say N. + +config DEFAULT_USE_ENERGY_AWARE + bool "Default to enabling the Energy Aware Scheduler feature" + default n + help + This option defaults the ENERGY_AWARE scheduling feature to true, + as without SCHED_DEBUG set this feature can't be enabled or disabled + via sysctl. + + Say N if unsure. + config SYSFS_DEPRECATED bool "Enable deprecated sysfs features to support old userspace tools" depends on SYSFS diff --git a/init/Makefile b/init/Makefile index c4fb45525d0880..d210b235c5d702 100644 --- a/init/Makefile +++ b/init/Makefile @@ -5,11 +5,8 @@ ccflags-y := -fno-function-sections -fno-data-sections obj-y := main.o version.o mounts.o -ifneq ($(CONFIG_BLK_DEV_INITRD),y) obj-y += noinitramfs.o -else obj-$(CONFIG_BLK_DEV_INITRD) += initramfs.o -endif obj-$(CONFIG_GENERIC_CALIBRATE_DELAY) += calibrate.o ifneq ($(CONFIG_ARCH_INIT_TASK),y) @@ -20,6 +17,7 @@ mounts-y := do_mounts.o mounts-$(CONFIG_BLK_DEV_RAM) += do_mounts_rd.o mounts-$(CONFIG_BLK_DEV_INITRD) += do_mounts_initrd.o mounts-$(CONFIG_BLK_DEV_MD) += do_mounts_md.o +mounts-$(CONFIG_BLK_DEV_DM) += do_mounts_dm.o # dependencies on generated files need to be listed explicitly $(obj)/version.o: include/generated/compile.h diff --git a/init/do_mounts.c b/init/do_mounts.c index dea5de95c2dd23..1902a1c808319f 100644 --- a/init/do_mounts.c +++ b/init/do_mounts.c @@ -566,6 +566,7 @@ void __init prepare_namespace(void) wait_for_device_probe(); md_run_setup(); + dm_run_setup(); if (saved_root_name[0]) { root_device_name = saved_root_name; diff --git a/init/do_mounts.h b/init/do_mounts.h index 067af1d9e8b620..ecb275782c03ef 100644 --- a/init/do_mounts.h +++ b/init/do_mounts.h @@ -74,3 +74,13 @@ void md_run_setup(void); static inline void md_run_setup(void) {} #endif + +#ifdef CONFIG_BLK_DEV_DM + +void dm_run_setup(void); + +#else + +static inline void dm_run_setup(void) {} + +#endif diff --git a/init/do_mounts_dm.c b/init/do_mounts_dm.c new file mode 100644 index 00000000000000..a557c5ee00a7b1 --- /dev/null +++ b/init/do_mounts_dm.c @@ -0,0 +1,426 @@ +/* do_mounts_dm.c + * Copyright (C) 2010 The Chromium OS Authors + * All Rights Reserved. + * Based on do_mounts_md.c + * + * This file is released under the GPL. + */ +#include +#include +#include + +#include "do_mounts.h" +#include "../drivers/md/dm.h" + +#define DM_MAX_NAME 32 +#define DM_MAX_UUID 129 +#define DM_NO_UUID "none" + +#define DM_MSG_PREFIX "init" + +/* Separators used for parsing the dm= argument. */ +#define DM_FIELD_SEP ' ' +#define DM_LINE_SEP ',' + +/* + * When the device-mapper and any targets are compiled into the kernel + * (not a module), one target may be created and used as the root device at + * boot time with the parameters given with the boot line dm=... + * The code for that is here. + */ + +struct dm_setup_target { + sector_t begin; + sector_t length; + char *type; + char *params; + /* simple singly linked list */ + struct dm_setup_target *next; +}; + +static struct { + int minor; + int ro; + char name[DM_MAX_NAME]; + char uuid[DM_MAX_UUID]; + char *targets; + struct dm_setup_target *target; + int target_count; +} dm_setup_args __initdata; + +static __initdata int dm_early_setup; + +static size_t __init get_dm_option(char *str, char **next, char sep) +{ + size_t len = 0; + char *endp = NULL; + + if (!str) + return 0; + + endp = strchr(str, sep); + if (!endp) { /* act like strchrnul */ + len = strlen(str); + endp = str + len; + } else { + len = endp - str; + } + + if (endp == str) + return 0; + + if (!next) + return len; + + if (*endp == 0) { + /* Don't advance past the nul. */ + *next = endp; + } else { + *next = endp + 1; + } + return len; +} + +static int __init dm_setup_args_init(void) +{ + dm_setup_args.minor = 0; + dm_setup_args.ro = 0; + dm_setup_args.target = NULL; + dm_setup_args.target_count = 0; + return 0; +} + +static int __init dm_setup_cleanup(void) +{ + struct dm_setup_target *target = dm_setup_args.target; + struct dm_setup_target *old_target = NULL; + while (target) { + kfree(target->type); + kfree(target->params); + old_target = target; + target = target->next; + kfree(old_target); + dm_setup_args.target_count--; + } + BUG_ON(dm_setup_args.target_count); + return 0; +} + +static char * __init dm_setup_parse_device_args(char *str) +{ + char *next = NULL; + size_t len = 0; + + /* Grab the logical name of the device to be exported to udev */ + len = get_dm_option(str, &next, DM_FIELD_SEP); + if (!len) { + DMERR("failed to parse device name"); + goto parse_fail; + } + len = min(len + 1, sizeof(dm_setup_args.name)); + strlcpy(dm_setup_args.name, str, len); /* includes nul */ + str = skip_spaces(next); + + /* Grab the UUID value or "none" */ + len = get_dm_option(str, &next, DM_FIELD_SEP); + if (!len) { + DMERR("failed to parse device uuid"); + goto parse_fail; + } + len = min(len + 1, sizeof(dm_setup_args.uuid)); + strlcpy(dm_setup_args.uuid, str, len); + str = skip_spaces(next); + + /* Determine if the table/device will be read only or read-write */ + if (!strncmp("ro,", str, 3)) { + dm_setup_args.ro = 1; + } else if (!strncmp("rw,", str, 3)) { + dm_setup_args.ro = 0; + } else { + DMERR("failed to parse table mode"); + goto parse_fail; + } + str = skip_spaces(str + 3); + + return str; + +parse_fail: + return NULL; +} + +static void __init dm_substitute_devices(char *str, size_t str_len) +{ + char *candidate = str; + char *candidate_end = str; + char old_char; + size_t len = 0; + dev_t dev; + + if (str_len < 3) + return; + + while (str && *str) { + candidate = strchr(str, '/'); + if (!candidate) + break; + + /* Avoid embedded slashes */ + if (candidate != str && *(candidate - 1) != DM_FIELD_SEP) { + str = strchr(candidate, DM_FIELD_SEP); + continue; + } + + len = get_dm_option(candidate, &candidate_end, DM_FIELD_SEP); + str = skip_spaces(candidate_end); + if (len < 3 || len > 37) /* name_to_dev_t max; maj:mix min */ + continue; + + /* Temporarily terminate with a nul */ + if (*candidate_end) + candidate_end--; + old_char = *candidate_end; + *candidate_end = '\0'; + + DMDEBUG("converting candidate device '%s' to dev_t", candidate); + /* Use the boot-time specific device naming */ + dev = name_to_dev_t(candidate); + *candidate_end = old_char; + + DMDEBUG(" -> %u", dev); + /* No suitable replacement found */ + if (!dev) + continue; + + /* Rewrite the /dev/path as a major:minor */ + len = snprintf(candidate, len, "%u:%u", MAJOR(dev), MINOR(dev)); + if (!len) { + DMERR("error substituting device major/minor."); + break; + } + candidate += len; + /* Pad out with spaces (fixing our nul) */ + while (candidate < candidate_end) + *(candidate++) = DM_FIELD_SEP; + } +} + +static int __init dm_setup_parse_targets(char *str) +{ + char *next = NULL; + size_t len = 0; + struct dm_setup_target **target = NULL; + + /* Targets are defined as per the table format but with a + * comma as a newline separator. */ + target = &dm_setup_args.target; + while (str && *str) { + *target = kzalloc(sizeof(struct dm_setup_target), GFP_KERNEL); + if (!*target) { + DMERR("failed to allocate memory for target %d", + dm_setup_args.target_count); + goto parse_fail; + } + dm_setup_args.target_count++; + + (*target)->begin = simple_strtoull(str, &next, 10); + if (!next || *next != DM_FIELD_SEP) { + DMERR("failed to parse starting sector for target %d", + dm_setup_args.target_count - 1); + goto parse_fail; + } + str = skip_spaces(next + 1); + + (*target)->length = simple_strtoull(str, &next, 10); + if (!next || *next != DM_FIELD_SEP) { + DMERR("failed to parse length for target %d", + dm_setup_args.target_count - 1); + goto parse_fail; + } + str = skip_spaces(next + 1); + + len = get_dm_option(str, &next, DM_FIELD_SEP); + if (!len || + !((*target)->type = kstrndup(str, len, GFP_KERNEL))) { + DMERR("failed to parse type for target %d", + dm_setup_args.target_count - 1); + goto parse_fail; + } + str = skip_spaces(next); + + len = get_dm_option(str, &next, DM_LINE_SEP); + if (!len || + !((*target)->params = kstrndup(str, len, GFP_KERNEL))) { + DMERR("failed to parse params for target %d", + dm_setup_args.target_count - 1); + goto parse_fail; + } + str = skip_spaces(next); + + /* Before moving on, walk through the copied target and + * attempt to replace all /dev/xxx with the major:minor number. + * It may not be possible to resolve them traditionally at + * boot-time. */ + dm_substitute_devices((*target)->params, len); + + target = &((*target)->next); + } + DMDEBUG("parsed %d targets", dm_setup_args.target_count); + + return 0; + +parse_fail: + return 1; +} + +/* + * Parse the command-line parameters given our kernel, but do not + * actually try to invoke the DM device now; that is handled by + * dm_setup_drive after the low-level disk drivers have initialised. + * dm format is as follows: + * dm="name uuid fmode,[table line 1],[table line 2],..." + * May be used with root=/dev/dm-0 as it always uses the first dm minor. + */ + +static int __init dm_setup(char *str) +{ + dm_setup_args_init(); + + str = dm_setup_parse_device_args(str); + if (!str) { + DMDEBUG("str is NULL"); + goto parse_fail; + } + + /* Target parsing is delayed until we have dynamic memory */ + dm_setup_args.targets = str; + + printk(KERN_INFO "dm: will configure '%s' on dm-%d\n", + dm_setup_args.name, dm_setup_args.minor); + + dm_early_setup = 1; + return 1; + +parse_fail: + printk(KERN_WARNING "dm: Invalid arguments supplied to dm=.\n"); + return 0; +} + + +static void __init dm_setup_drive(void) +{ + struct mapped_device *md = NULL; + struct dm_table *table = NULL; + struct dm_setup_target *target; + char *uuid = dm_setup_args.uuid; + fmode_t fmode = FMODE_READ; + + /* Finish parsing the targets. */ + if (dm_setup_parse_targets(dm_setup_args.targets)) + goto parse_fail; + + if (dm_create(dm_setup_args.minor, &md)) { + DMDEBUG("failed to create the device"); + goto dm_create_fail; + } + DMDEBUG("created device '%s'", dm_device_name(md)); + + /* In addition to flagging the table below, the disk must be + * set explicitly ro/rw. */ + set_disk_ro(dm_disk(md), dm_setup_args.ro); + + if (!dm_setup_args.ro) + fmode |= FMODE_WRITE; + if (dm_table_create(&table, fmode, dm_setup_args.target_count, md)) { + DMDEBUG("failed to create the table"); + goto dm_table_create_fail; + } + + dm_lock_md_type(md); + target = dm_setup_args.target; + while (target) { + DMINFO("adding target '%llu %llu %s %s'", + (unsigned long long) target->begin, + (unsigned long long) target->length, target->type, + target->params); + if (dm_table_add_target(table, target->type, target->begin, + target->length, target->params)) { + DMDEBUG("failed to add the target to the table"); + goto add_target_fail; + } + target = target->next; + } + + if (dm_table_complete(table)) { + DMDEBUG("failed to complete the table"); + goto table_complete_fail; + } + + if (dm_get_md_type(md) == DM_TYPE_NONE) { + dm_set_md_type(md, dm_table_get_type(table)); + if (dm_setup_md_queue(md, table)) { + DMWARN("unable to set up device queue for new table."); + goto setup_md_queue_fail; + } + } else if (dm_get_md_type(md) != dm_table_get_type(table)) { + DMWARN("can't change device type after initial table load."); + goto setup_md_queue_fail; + } + + /* Suspend the device so that we can bind it to the table. */ + if (dm_suspend(md, 0)) { + DMDEBUG("failed to suspend the device pre-bind"); + goto suspend_fail; + } + + /* Bind the table to the device. This is the only way to associate + * md->map with the table and set the disk capacity directly. */ + if (dm_swap_table(md, table)) { /* should return NULL. */ + DMDEBUG("failed to bind the device to the table"); + goto table_bind_fail; + } + + /* Finally, resume and the device should be ready. */ + if (dm_resume(md)) { + DMDEBUG("failed to resume the device"); + goto resume_fail; + } + + /* Export the dm device via the ioctl interface */ + if (!strcmp(DM_NO_UUID, dm_setup_args.uuid)) + uuid = NULL; + if (dm_ioctl_export(md, dm_setup_args.name, uuid)) { + DMDEBUG("failed to export device with given name and uuid"); + goto export_fail; + } + printk(KERN_INFO "dm: dm-%d is ready\n", dm_setup_args.minor); + + dm_unlock_md_type(md); + dm_setup_cleanup(); + return; + +export_fail: +resume_fail: +table_bind_fail: +suspend_fail: +setup_md_queue_fail: +table_complete_fail: +add_target_fail: + dm_unlock_md_type(md); +dm_table_create_fail: + dm_put(md); +dm_create_fail: + dm_setup_cleanup(); +parse_fail: + printk(KERN_WARNING "dm: starting dm-%d (%s) failed\n", + dm_setup_args.minor, dm_setup_args.name); +} + +__setup("dm=", dm_setup); + +void __init dm_run_setup(void) +{ + if (!dm_early_setup) + return; + printk(KERN_INFO "dm: attempting early device configuration.\n"); + dm_setup_drive(); +} diff --git a/init/initramfs.c b/init/initramfs.c index b32ad7d97ac94f..f8ce812ba43e93 100644 --- a/init/initramfs.c +++ b/init/initramfs.c @@ -18,6 +18,7 @@ #include #include #include +#include static ssize_t __init xwrite(int fd, const char *p, size_t count) { @@ -605,9 +606,25 @@ static void __init clean_rootfs(void) } #endif +static int __initdata do_skip_initramfs; + +static int __init skip_initramfs_param(char *str) +{ + if (*str) + return 0; + do_skip_initramfs = 1; + return 1; +} +__setup("skip_initramfs", skip_initramfs_param); + static int __init populate_rootfs(void) { - char *err = unpack_to_rootfs(__initramfs_start, __initramfs_size); + char *err; + + if (do_skip_initramfs) + return default_rootfs(); + + err = unpack_to_rootfs(__initramfs_start, __initramfs_size); if (err) panic("%s", err); /* Failed to decompress INTERNAL initramfs */ if (initrd_start) { diff --git a/init/noinitramfs.c b/init/noinitramfs.c index 267739d851791b..bcc8bcb053eeb0 100644 --- a/init/noinitramfs.c +++ b/init/noinitramfs.c @@ -21,11 +21,16 @@ #include #include #include +#include +#include /* * Create a simple rootfs that is similar to the default initramfs */ -static int __init default_rootfs(void) +#if !IS_BUILTIN(CONFIG_BLK_DEV_INITRD) +static +#endif +int __init default_rootfs(void) { int err; @@ -49,4 +54,6 @@ static int __init default_rootfs(void) printk(KERN_WARNING "Failed to create a rootfs\n"); return err; } +#if !IS_BUILTIN(CONFIG_BLK_DEV_INITRD) rootfs_initcall(default_rootfs); +#endif diff --git a/ipc/mqueue.c b/ipc/mqueue.c index 8cbd6e6894d514..a37a10bc07ea0f 100644 --- a/ipc/mqueue.c +++ b/ipc/mqueue.c @@ -745,7 +745,7 @@ static struct file *do_create(struct ipc_namespace *ipc_ns, struct inode *dir, } mode &= ~current_umask(); - ret = vfs_create(dir, path->dentry, mode, true); + ret = vfs_create2(path->mnt, dir, path->dentry, mode, true); path->dentry->d_fsdata = NULL; if (ret) return ERR_PTR(ret); @@ -761,7 +761,7 @@ static struct file *do_open(struct path *path, int oflag) if ((oflag & O_ACCMODE) == (O_RDWR | O_WRONLY)) return ERR_PTR(-EINVAL); acc = oflag2acc[oflag & O_ACCMODE]; - if (inode_permission(d_inode(path->dentry), acc)) + if (inode_permission2(path->mnt, d_inode(path->dentry), acc)) return ERR_PTR(-EACCES); return dentry_open(path, oflag, current_cred()); } @@ -794,7 +794,7 @@ SYSCALL_DEFINE4(mq_open, const char __user *, u_name, int, oflag, umode_t, mode, ro = mnt_want_write(mnt); /* we'll drop it in any case */ error = 0; inode_lock(d_inode(root)); - path.dentry = lookup_one_len(name->name, root, strlen(name->name)); + path.dentry = lookup_one_len2(name->name, mnt, root, strlen(name->name)); if (IS_ERR(path.dentry)) { error = PTR_ERR(path.dentry); goto out_putfd; @@ -865,7 +865,7 @@ SYSCALL_DEFINE1(mq_unlink, const char __user *, u_name) if (err) goto out_name; inode_lock_nested(d_inode(mnt->mnt_root), I_MUTEX_PARENT); - dentry = lookup_one_len(name->name, mnt->mnt_root, + dentry = lookup_one_len2(name->name, mnt, mnt->mnt_root, strlen(name->name)); if (IS_ERR(dentry)) { err = PTR_ERR(dentry); @@ -877,7 +877,7 @@ SYSCALL_DEFINE1(mq_unlink, const char __user *, u_name) err = -ENOENT; } else { ihold(inode); - err = vfs_unlink(d_inode(dentry->d_parent), dentry, NULL); + err = vfs_unlink2(mnt, d_inode(dentry->d_parent), dentry, NULL); } dput(dentry); diff --git a/kernel/capability.c b/kernel/capability.c index 00411c82dac57c..4984e1f552ebfb 100644 --- a/kernel/capability.c +++ b/kernel/capability.c @@ -456,6 +456,19 @@ bool file_ns_capable(const struct file *file, struct user_namespace *ns, } EXPORT_SYMBOL(file_ns_capable); +/** + * privileged_wrt_inode_uidgid - Do capabilities in the namespace work over the inode? + * @ns: The user namespace in question + * @inode: The inode in question + * + * Return true if the inode uid and gid are within the namespace. + */ +bool privileged_wrt_inode_uidgid(struct user_namespace *ns, const struct inode *inode) +{ + return kuid_has_mapping(ns, inode->i_uid) && + kgid_has_mapping(ns, inode->i_gid); +} + /** * capable_wrt_inode_uidgid - Check nsown_capable and uid and gid mapped * @inode: The inode in question @@ -469,7 +482,26 @@ bool capable_wrt_inode_uidgid(const struct inode *inode, int cap) { struct user_namespace *ns = current_user_ns(); - return ns_capable(ns, cap) && kuid_has_mapping(ns, inode->i_uid) && - kgid_has_mapping(ns, inode->i_gid); + return ns_capable(ns, cap) && privileged_wrt_inode_uidgid(ns, inode); } EXPORT_SYMBOL(capable_wrt_inode_uidgid); + +/** + * ptracer_capable - Determine if the ptracer holds CAP_SYS_PTRACE in the namespace + * @tsk: The task that may be ptraced + * @ns: The user namespace to search for CAP_SYS_PTRACE in + * + * Return true if the task that is ptracing the current task had CAP_SYS_PTRACE + * in the specified user namespace. + */ +bool ptracer_capable(struct task_struct *tsk, struct user_namespace *ns) +{ + int ret = 0; /* An absent tracer adds no restrictions */ + const struct cred *cred; + rcu_read_lock(); + cred = rcu_dereference(tsk->ptracer_cred); + if (cred) + ret = security_capable_noaudit(cred, ns, CAP_SYS_PTRACE); + rcu_read_unlock(); + return (ret == 0); +} diff --git a/kernel/cgroup.c b/kernel/cgroup.c index 85bc9beb046d9a..3c60e6a759050c 100644 --- a/kernel/cgroup.c +++ b/kernel/cgroup.c @@ -2856,7 +2856,8 @@ static int cgroup_procs_write_permission(struct task_struct *task, */ if (!uid_eq(cred->euid, GLOBAL_ROOT_UID) && !uid_eq(cred->euid, tcred->uid) && - !uid_eq(cred->euid, tcred->suid)) + !uid_eq(cred->euid, tcred->suid) && + !ns_capable(tcred->user_ns, CAP_SYS_RESOURCE)) ret = -EACCES; if (!ret && cgroup_on_dfl(dst_cgrp)) { @@ -5219,6 +5220,11 @@ static struct cgroup_subsys_state *css_create(struct cgroup *cgrp, return ERR_PTR(err); } +/* + * The returned cgroup is fully initialized including its control mask, but + * it isn't associated with its kernfs_node and doesn't have the control + * mask applied. + */ static struct cgroup *cgroup_create(struct cgroup *parent) { struct cgroup_root *root = parent->root; @@ -5283,11 +5289,6 @@ static struct cgroup *cgroup_create(struct cgroup *parent) cgroup_propagate_control(cgrp); - /* @cgrp doesn't have dir yet so the following will only create csses */ - ret = cgroup_apply_control_enable(cgrp); - if (ret) - goto out_destroy; - return cgrp; out_cancel_ref: @@ -5295,9 +5296,6 @@ static struct cgroup *cgroup_create(struct cgroup *parent) out_free_cgrp: kfree(cgrp); return ERR_PTR(ret); -out_destroy: - cgroup_destroy_locked(cgrp); - return ERR_PTR(ret); } static int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name, diff --git a/kernel/configs/README.android b/kernel/configs/README.android new file mode 100644 index 00000000000000..2e2d7c001275dd --- /dev/null +++ b/kernel/configs/README.android @@ -0,0 +1,15 @@ +The android-*.config files in this directory are meant to be used as a base +for an Android kernel config. All devices should have the options in +android-base.config enabled. While not mandatory, the options in +android-recommended.config enable advanced Android features. + +Assuming you already have a minimalist defconfig for your device, a possible +way to enable these options would be: + + ARCH= scripts/kconfig/merge_config.sh /_defconfig kernel/configs/android-base.config kernel/configs/android-recommended.config + +This will generate a .config that can then be used to save a new defconfig or +compile a new kernel with Android features enabled. + +Because there is no tool to consistently generate these config fragments, +lets keep them alphabetically sorted instead of random. diff --git a/kernel/configs/android-base.config b/kernel/configs/android-base.config index 1a8f34f6360112..4732628dcf28bf 100644 --- a/kernel/configs/android-base.config +++ b/kernel/configs/android-base.config @@ -21,6 +21,7 @@ CONFIG_CP15_BARRIER_EMULATION=y CONFIG_DEFAULT_SECURITY_SELINUX=y CONFIG_EMBEDDED=y CONFIG_FB=y +CONFIG_HARDENED_USERCOPY=y CONFIG_HIGH_RES_TIMERS=y CONFIG_INET6_AH=y CONFIG_INET6_ESP=y @@ -76,6 +77,9 @@ CONFIG_NETFILTER_XT_MATCH_MAC=y CONFIG_NETFILTER_XT_MATCH_MARK=y CONFIG_NETFILTER_XT_MATCH_PKTTYPE=y CONFIG_NETFILTER_XT_MATCH_POLICY=y +CONFIG_NETFILTER_XT_MATCH_QTAGUID=y +CONFIG_NETFILTER_XT_MATCH_QUOTA2=y +CONFIG_NETFILTER_XT_MATCH_QUOTA2_LOG=y CONFIG_NETFILTER_XT_MATCH_QUOTA=y CONFIG_NETFILTER_XT_MATCH_SOCKET=y CONFIG_NETFILTER_XT_MATCH_STATE=y @@ -124,26 +128,41 @@ CONFIG_PACKET=y CONFIG_PM_AUTOSLEEP=y CONFIG_PM_WAKELOCKS=y CONFIG_PPP=y +CONFIG_PPPOLAC=y +CONFIG_PPPOPNS=y CONFIG_PPP_BSDCOMP=y CONFIG_PPP_DEFLATE=y CONFIG_PPP_MPPE=y CONFIG_PREEMPT=y +CONFIG_PROFILING=y +CONFIG_QFMT_V2=y CONFIG_QUOTA=y +CONFIG_QUOTA_NETLINK_INTERFACE=y +CONFIG_QUOTA_TREE=y +CONFIG_QUOTACTL=y +CONFIG_RANDOMIZE_BASE=y CONFIG_RTC_CLASS=y CONFIG_RT_GROUP_SCHED=y CONFIG_SECCOMP=y CONFIG_SECURITY=y CONFIG_SECURITY_NETWORK=y +CONFIG_SECURITY_PERF_EVENTS_RESTRICT=y CONFIG_SECURITY_SELINUX=y CONFIG_SETEND_EMULATION=y CONFIG_STAGING=y CONFIG_SWP_EMULATION=y CONFIG_SYNC=y CONFIG_TUN=y +CONFIG_UID_CPUTIME=y CONFIG_UNIX=y CONFIG_USB_GADGET=y CONFIG_USB_CONFIGFS=y CONFIG_USB_CONFIGFS_F_FS=y +CONFIG_USB_CONFIGFS_F_MTP=y +CONFIG_USB_CONFIGFS_F_PTP=y +CONFIG_USB_CONFIGFS_F_ACC=y +CONFIG_USB_CONFIGFS_F_AUDIO_SRC=y +CONFIG_USB_CONFIGFS_UEVENT=y CONFIG_USB_CONFIGFS_F_MIDI=y CONFIG_USB_OTG_WAKELOCK=y CONFIG_XFRM_USER=y diff --git a/kernel/configs/android-recommended.config b/kernel/configs/android-recommended.config index 297756be369c68..919e82743da52b 100644 --- a/kernel/configs/android-recommended.config +++ b/kernel/configs/android-recommended.config @@ -1,15 +1,18 @@ # KEEP ALPHABETICALLY SORTED +# CONFIG_AIO is not set # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set # CONFIG_INPUT_MOUSE is not set # CONFIG_LEGACY_PTYS is not set # CONFIG_NF_CONNTRACK_SIP is not set # CONFIG_PM_WAKELOCKS_GC is not set # CONFIG_VT is not set +CONFIG_ARM64_SW_TTBR0_PAN=y CONFIG_BACKLIGHT_LCD_SUPPORT=y CONFIG_BLK_DEV_DM=y CONFIG_BLK_DEV_LOOP=y CONFIG_BLK_DEV_RAM=y CONFIG_BLK_DEV_RAM_SIZE=8192 +CONFIG_CC_STACKPROTECTOR_STRONG=y CONFIG_COMPACTION=y CONFIG_DEBUG_RODATA=y CONFIG_DM_CRYPT=y @@ -75,6 +78,8 @@ CONFIG_HID_ZYDACRON=y CONFIG_INPUT_EVDEV=y CONFIG_INPUT_GPIO=y CONFIG_INPUT_JOYSTICK=y +CONFIG_INPUT_KEYCHORD=y +CONFIG_INPUT_KEYRESET=y CONFIG_INPUT_MISC=y CONFIG_INPUT_TABLET=y CONFIG_INPUT_UINPUT=y @@ -89,6 +94,7 @@ CONFIG_LOGIRUMBLEPAD2_FF=y CONFIG_LOGITECH_FF=y CONFIG_MD=y CONFIG_MEDIA_SUPPORT=y +CONFIG_MEMORY_STATE_TIME=y CONFIG_MSDOS_FS=y CONFIG_PANIC_TIMEOUT=5 CONFIG_PANTHERLORD_FF=y diff --git a/kernel/cpu.c b/kernel/cpu.c index 29de1a9352c005..029ff74c23d1a7 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -659,7 +659,6 @@ void __init cpuhp_threads_init(void) kthread_unpark(this_cpu_read(cpuhp_state.thread)); } -#ifdef CONFIG_HOTPLUG_CPU EXPORT_SYMBOL(register_cpu_notifier); EXPORT_SYMBOL(__register_cpu_notifier); void unregister_cpu_notifier(struct notifier_block *nb) @@ -676,6 +675,7 @@ void __unregister_cpu_notifier(struct notifier_block *nb) } EXPORT_SYMBOL(__unregister_cpu_notifier); +#ifdef CONFIG_HOTPLUG_CPU /** * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU * @cpu: a CPU id @@ -1154,6 +1154,7 @@ void __weak arch_enable_nonboot_cpus_end(void) void enable_nonboot_cpus(void) { int cpu, error; + struct device *cpu_device; /* Allow everyone to use the CPU hotplug again */ cpu_maps_update_begin(); @@ -1171,6 +1172,12 @@ void enable_nonboot_cpus(void) trace_suspend_resume(TPS("CPU_ON"), cpu, false); if (!error) { pr_info("CPU%d is up\n", cpu); + cpu_device = get_cpu_device(cpu); + if (!cpu_device) + pr_err("%s: failed to get cpu%d device\n", + __func__, cpu); + else + kobject_uevent(&cpu_device->kobj, KOBJ_ONLINE); continue; } pr_warn("Error taking CPU%d up: %d\n", cpu, error); @@ -1932,3 +1939,23 @@ void __init boot_cpu_state_init(void) { per_cpu_ptr(&cpuhp_state, smp_processor_id())->state = CPUHP_ONLINE; } + +static ATOMIC_NOTIFIER_HEAD(idle_notifier); + +void idle_notifier_register(struct notifier_block *n) +{ + atomic_notifier_chain_register(&idle_notifier, n); +} +EXPORT_SYMBOL_GPL(idle_notifier_register); + +void idle_notifier_unregister(struct notifier_block *n) +{ + atomic_notifier_chain_unregister(&idle_notifier, n); +} +EXPORT_SYMBOL_GPL(idle_notifier_unregister); + +void idle_notifier_call_chain(unsigned long val) +{ + atomic_notifier_call_chain(&idle_notifier, val, NULL); +} +EXPORT_SYMBOL_GPL(idle_notifier_call_chain); diff --git a/kernel/cpuset.c b/kernel/cpuset.c index 29f815d2ef7e3c..f8b460f9d861eb 100644 --- a/kernel/cpuset.c +++ b/kernel/cpuset.c @@ -99,6 +99,7 @@ struct cpuset { /* user-configured CPUs and Memory Nodes allow to tasks */ cpumask_var_t cpus_allowed; + cpumask_var_t cpus_requested; nodemask_t mems_allowed; /* effective CPUs and Memory Nodes allow to tasks */ @@ -398,7 +399,7 @@ static void cpuset_update_task_spread_flag(struct cpuset *cs, static int is_cpuset_subset(const struct cpuset *p, const struct cpuset *q) { - return cpumask_subset(p->cpus_allowed, q->cpus_allowed) && + return cpumask_subset(p->cpus_requested, q->cpus_requested) && nodes_subset(p->mems_allowed, q->mems_allowed) && is_cpu_exclusive(p) <= is_cpu_exclusive(q) && is_mem_exclusive(p) <= is_mem_exclusive(q); @@ -498,7 +499,7 @@ static int validate_change(struct cpuset *cur, struct cpuset *trial) cpuset_for_each_child(c, css, par) { if ((is_cpu_exclusive(trial) || is_cpu_exclusive(c)) && c != cur && - cpumask_intersects(trial->cpus_allowed, c->cpus_allowed)) + cpumask_intersects(trial->cpus_requested, c->cpus_requested)) goto out; if ((is_mem_exclusive(trial) || is_mem_exclusive(c)) && c != cur && @@ -957,17 +958,18 @@ static int update_cpumask(struct cpuset *cs, struct cpuset *trialcs, if (!*buf) { cpumask_clear(trialcs->cpus_allowed); } else { - retval = cpulist_parse(buf, trialcs->cpus_allowed); + retval = cpulist_parse(buf, trialcs->cpus_requested); if (retval < 0) return retval; - if (!cpumask_subset(trialcs->cpus_allowed, - top_cpuset.cpus_allowed)) + if (!cpumask_subset(trialcs->cpus_requested, cpu_present_mask)) return -EINVAL; + + cpumask_and(trialcs->cpus_allowed, trialcs->cpus_requested, cpu_active_mask); } /* Nothing to do if the cpus didn't change */ - if (cpumask_equal(cs->cpus_allowed, trialcs->cpus_allowed)) + if (cpumask_equal(cs->cpus_requested, trialcs->cpus_requested)) return 0; retval = validate_change(cs, trialcs); @@ -976,6 +978,7 @@ static int update_cpumask(struct cpuset *cs, struct cpuset *trialcs, spin_lock_irq(&callback_lock); cpumask_copy(cs->cpus_allowed, trialcs->cpus_allowed); + cpumask_copy(cs->cpus_requested, trialcs->cpus_requested); spin_unlock_irq(&callback_lock); /* use trialcs->cpus_allowed as a temp variable */ @@ -1760,7 +1763,7 @@ static int cpuset_common_seq_show(struct seq_file *sf, void *v) switch (type) { case FILE_CPULIST: - seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->cpus_allowed)); + seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->cpus_requested)); break; case FILE_MEMLIST: seq_printf(sf, "%*pbl\n", nodemask_pr_args(&cs->mems_allowed)); @@ -1949,11 +1952,14 @@ cpuset_css_alloc(struct cgroup_subsys_state *parent_css) return ERR_PTR(-ENOMEM); if (!alloc_cpumask_var(&cs->cpus_allowed, GFP_KERNEL)) goto free_cs; + if (!alloc_cpumask_var(&cs->cpus_requested, GFP_KERNEL)) + goto free_allowed; if (!alloc_cpumask_var(&cs->effective_cpus, GFP_KERNEL)) - goto free_cpus; + goto free_requested; set_bit(CS_SCHED_LOAD_BALANCE, &cs->flags); cpumask_clear(cs->cpus_allowed); + cpumask_clear(cs->cpus_requested); nodes_clear(cs->mems_allowed); cpumask_clear(cs->effective_cpus); nodes_clear(cs->effective_mems); @@ -1962,7 +1968,9 @@ cpuset_css_alloc(struct cgroup_subsys_state *parent_css) return &cs->css; -free_cpus: +free_requested: + free_cpumask_var(cs->cpus_requested); +free_allowed: free_cpumask_var(cs->cpus_allowed); free_cs: kfree(cs); @@ -2025,6 +2033,7 @@ static int cpuset_css_online(struct cgroup_subsys_state *css) cs->mems_allowed = parent->mems_allowed; cs->effective_mems = parent->mems_allowed; cpumask_copy(cs->cpus_allowed, parent->cpus_allowed); + cpumask_copy(cs->cpus_requested, parent->cpus_requested); cpumask_copy(cs->effective_cpus, parent->cpus_allowed); spin_unlock_irq(&callback_lock); out_unlock: @@ -2059,6 +2068,7 @@ static void cpuset_css_free(struct cgroup_subsys_state *css) free_cpumask_var(cs->effective_cpus); free_cpumask_var(cs->cpus_allowed); + free_cpumask_var(cs->cpus_requested); kfree(cs); } @@ -2123,8 +2133,11 @@ int __init cpuset_init(void) BUG(); if (!alloc_cpumask_var(&top_cpuset.effective_cpus, GFP_KERNEL)) BUG(); + if (!alloc_cpumask_var(&top_cpuset.cpus_requested, GFP_KERNEL)) + BUG(); cpumask_setall(top_cpuset.cpus_allowed); + cpumask_setall(top_cpuset.cpus_requested); nodes_setall(top_cpuset.mems_allowed); cpumask_setall(top_cpuset.effective_cpus); nodes_setall(top_cpuset.effective_mems); @@ -2258,7 +2271,7 @@ static void cpuset_hotplug_update_tasks(struct cpuset *cs) goto retry; } - cpumask_and(&new_cpus, cs->cpus_allowed, parent_cs(cs)->effective_cpus); + cpumask_and(&new_cpus, cs->cpus_requested, parent_cs(cs)->effective_cpus); nodes_and(new_mems, cs->mems_allowed, parent_cs(cs)->effective_mems); cpus_updated = !cpumask_equal(&new_cpus, cs->effective_cpus); diff --git a/kernel/debug/debug_core.c b/kernel/debug/debug_core.c index 0874e2edd2756b..79517e5549f119 100644 --- a/kernel/debug/debug_core.c +++ b/kernel/debug/debug_core.c @@ -598,11 +598,11 @@ static int kgdb_cpu_enter(struct kgdb_state *ks, struct pt_regs *regs, /* * Wait for the other CPUs to be notified and be waiting for us: */ - time_left = loops_per_jiffy * HZ; + time_left = MSEC_PER_SEC; while (kgdb_do_roundup && --time_left && (atomic_read(&masters_in_kgdb) + atomic_read(&slaves_in_kgdb)) != online_cpus) - cpu_relax(); + udelay(1000); if (!time_left) pr_crit("Timed out waiting for secondary CPUs.\n"); diff --git a/kernel/debug/kdb/kdb_io.c b/kernel/debug/kdb/kdb_io.c index fc1ef736253c79..0b891286a150c8 100644 --- a/kernel/debug/kdb/kdb_io.c +++ b/kernel/debug/kdb/kdb_io.c @@ -216,7 +216,7 @@ static char *kdb_read(char *buffer, size_t bufsize) int i; int diag, dtab_count; int key; - + static int last_crlf; diag = kdbgetintenv("DTABCOUNT", &dtab_count); if (diag) @@ -237,6 +237,9 @@ static char *kdb_read(char *buffer, size_t bufsize) return buffer; if (key != 9) tab = 0; + if (key != 10 && key != 13) + last_crlf = 0; + switch (key) { case 8: /* backspace */ if (cp > buffer) { @@ -254,7 +257,12 @@ static char *kdb_read(char *buffer, size_t bufsize) *cp = tmp; } break; - case 13: /* enter */ + case 10: /* new line */ + case 13: /* carriage return */ + /* handle \n after \r */ + if (last_crlf && last_crlf != key) + break; + last_crlf = key; *lastchar++ = '\n'; *lastchar++ = '\0'; if (!KDB_STATE(KGDB_TRANS)) { diff --git a/kernel/events/core.c b/kernel/events/core.c index 02c8421f8c0190..4353f3a658913e 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -389,8 +389,13 @@ static struct srcu_struct pmus_srcu; * 0 - disallow raw tracepoint access for unpriv * 1 - disallow cpu events for unpriv * 2 - disallow kernel profiling for unpriv + * 3 - disallow all unpriv perf event use */ +#ifdef CONFIG_SECURITY_PERF_EVENTS_RESTRICT +int sysctl_perf_event_paranoid __read_mostly = 3; +#else int sysctl_perf_event_paranoid __read_mostly = 2; +#endif /* Minimum for 512 kiB + 1 user control page */ int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); /* 'free' kiB per user */ @@ -1469,7 +1474,6 @@ ctx_group_list(struct perf_event *event, struct perf_event_context *ctx) static void list_add_event(struct perf_event *event, struct perf_event_context *ctx) { - lockdep_assert_held(&ctx->lock); WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT); @@ -1624,6 +1628,8 @@ static void perf_group_attach(struct perf_event *event) { struct perf_event *group_leader = event->group_leader, *pos; + lockdep_assert_held(&event->ctx->lock); + /* * We can have double attach due to group movement in perf_event_open. */ @@ -1697,6 +1703,8 @@ static void perf_group_detach(struct perf_event *event) struct perf_event *sibling, *tmp; struct list_head *list = NULL; + lockdep_assert_held(&event->ctx->lock); + /* * We can have double detach due to exit/hot-unplug + close. */ @@ -1895,9 +1903,29 @@ __perf_remove_from_context(struct perf_event *event, */ static void perf_remove_from_context(struct perf_event *event, unsigned long flags) { - lockdep_assert_held(&event->ctx->mutex); + struct perf_event_context *ctx = event->ctx; + + lockdep_assert_held(&ctx->mutex); event_function_call(event, __perf_remove_from_context, (void *)flags); + + /* + * The above event_function_call() can NO-OP when it hits + * TASK_TOMBSTONE. In that case we must already have been detached + * from the context (by perf_event_exit_event()) but the grouping + * might still be in-tact. + */ + WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT); + if ((flags & DETACH_GROUP) && + (event->attach_state & PERF_ATTACH_GROUP)) { + /* + * Since in that case we cannot possibly be scheduled, simply + * detach now. + */ + raw_spin_lock_irq(&ctx->lock); + perf_group_detach(event); + raw_spin_unlock_irq(&ctx->lock); + } } /* @@ -3438,14 +3466,15 @@ struct perf_read_data { int ret; }; -static int find_cpu_to_read(struct perf_event *event, int local_cpu) +static int __perf_event_read_cpu(struct perf_event *event, int event_cpu) { - int event_cpu = event->oncpu; u16 local_pkg, event_pkg; if (event->group_caps & PERF_EV_CAP_READ_ACTIVE_PKG) { - event_pkg = topology_physical_package_id(event_cpu); - local_pkg = topology_physical_package_id(local_cpu); + int local_cpu = smp_processor_id(); + + event_pkg = topology_physical_package_id(event_cpu); + local_pkg = topology_physical_package_id(local_cpu); if (event_pkg == local_pkg) return local_cpu; @@ -3575,7 +3604,7 @@ u64 perf_event_read_local(struct perf_event *event) static int perf_event_read(struct perf_event *event, bool group) { - int ret = 0, cpu_to_read, local_cpu; + int event_cpu, ret = 0; /* * If event is enabled and currently active on a CPU, update the @@ -3588,21 +3617,25 @@ static int perf_event_read(struct perf_event *event, bool group) .ret = 0, }; - local_cpu = get_cpu(); - cpu_to_read = find_cpu_to_read(event, local_cpu); - put_cpu(); + event_cpu = READ_ONCE(event->oncpu); + if ((unsigned)event_cpu >= nr_cpu_ids) + return 0; + + preempt_disable(); + event_cpu = __perf_event_read_cpu(event, event_cpu); /* * Purposely ignore the smp_call_function_single() return * value. * - * If event->oncpu isn't a valid CPU it means the event got + * If event_cpu isn't a valid CPU it means the event got * scheduled out and that will have updated the event count. * * Therefore, either way, we'll have an up-to-date event count * after this. */ - (void)smp_call_function_single(cpu_to_read, __perf_event_read, &data, 1); + (void)smp_call_function_single(event_cpu, __perf_event_read, &data, 1); + preempt_enable(); ret = data.ret; } else if (event->state == PERF_EVENT_STATE_INACTIVE) { struct perf_event_context *ctx = event->ctx; @@ -6583,6 +6616,27 @@ static void perf_event_mmap_event(struct perf_mmap_event *mmap_event) char *buf = NULL; char *name; + if (vma->vm_flags & VM_READ) + prot |= PROT_READ; + if (vma->vm_flags & VM_WRITE) + prot |= PROT_WRITE; + if (vma->vm_flags & VM_EXEC) + prot |= PROT_EXEC; + + if (vma->vm_flags & VM_MAYSHARE) + flags = MAP_SHARED; + else + flags = MAP_PRIVATE; + + if (vma->vm_flags & VM_DENYWRITE) + flags |= MAP_DENYWRITE; + if (vma->vm_flags & VM_MAYEXEC) + flags |= MAP_EXECUTABLE; + if (vma->vm_flags & VM_LOCKED) + flags |= MAP_LOCKED; + if (vma->vm_flags & VM_HUGETLB) + flags |= MAP_HUGETLB; + if (file) { struct inode *inode; dev_t dev; @@ -6609,27 +6663,6 @@ static void perf_event_mmap_event(struct perf_mmap_event *mmap_event) maj = MAJOR(dev); min = MINOR(dev); - if (vma->vm_flags & VM_READ) - prot |= PROT_READ; - if (vma->vm_flags & VM_WRITE) - prot |= PROT_WRITE; - if (vma->vm_flags & VM_EXEC) - prot |= PROT_EXEC; - - if (vma->vm_flags & VM_MAYSHARE) - flags = MAP_SHARED; - else - flags = MAP_PRIVATE; - - if (vma->vm_flags & VM_DENYWRITE) - flags |= MAP_DENYWRITE; - if (vma->vm_flags & VM_MAYEXEC) - flags |= MAP_EXECUTABLE; - if (vma->vm_flags & VM_LOCKED) - flags |= MAP_LOCKED; - if (vma->vm_flags & VM_HUGETLB) - flags |= MAP_HUGETLB; - goto got_name; } else { if (vma->vm_ops && vma->vm_ops->name) { @@ -9503,6 +9536,37 @@ static int perf_event_set_clock(struct perf_event *event, clockid_t clk_id) return 0; } +/* + * Variation on perf_event_ctx_lock_nested(), except we take two context + * mutexes. + */ +static struct perf_event_context * +__perf_event_ctx_lock_double(struct perf_event *group_leader, + struct perf_event_context *ctx) +{ + struct perf_event_context *gctx; + +again: + rcu_read_lock(); + gctx = READ_ONCE(group_leader->ctx); + if (!atomic_inc_not_zero(&gctx->refcount)) { + rcu_read_unlock(); + goto again; + } + rcu_read_unlock(); + + mutex_lock_double(&gctx->mutex, &ctx->mutex); + + if (group_leader->ctx != gctx) { + mutex_unlock(&ctx->mutex); + mutex_unlock(&gctx->mutex); + put_ctx(gctx); + goto again; + } + + return gctx; +} + /** * sys_perf_event_open - open a performance event, associate it to a task/cpu * @@ -9533,6 +9597,9 @@ SYSCALL_DEFINE5(perf_event_open, if (flags & ~PERF_FLAG_ALL) return -EINVAL; + if (perf_paranoid_any() && !capable(CAP_SYS_ADMIN)) + return -EACCES; + err = perf_copy_attr(attr_uptr, &attr); if (err) return err; @@ -9746,12 +9813,31 @@ SYSCALL_DEFINE5(perf_event_open, } if (move_group) { - gctx = group_leader->ctx; - mutex_lock_double(&gctx->mutex, &ctx->mutex); + gctx = __perf_event_ctx_lock_double(group_leader, ctx); + if (gctx->task == TASK_TOMBSTONE) { err = -ESRCH; goto err_locked; } + + /* + * Check if we raced against another sys_perf_event_open() call + * moving the software group underneath us. + */ + if (!(group_leader->group_caps & PERF_EV_CAP_SOFTWARE)) { + /* + * If someone moved the group out from under us, check + * if this new event wound up on the same ctx, if so + * its the regular !move_group case, otherwise fail. + */ + if (gctx != ctx) { + err = -EINVAL; + goto err_locked; + } else { + perf_event_ctx_unlock(group_leader, gctx); + move_group = 0; + } + } } else { mutex_lock(&ctx->mutex); } @@ -9853,7 +9939,7 @@ SYSCALL_DEFINE5(perf_event_open, perf_unpin_context(ctx); if (move_group) - mutex_unlock(&gctx->mutex); + perf_event_ctx_unlock(group_leader, gctx); mutex_unlock(&ctx->mutex); if (task) { @@ -9879,7 +9965,7 @@ SYSCALL_DEFINE5(perf_event_open, err_locked: if (move_group) - mutex_unlock(&gctx->mutex); + perf_event_ctx_unlock(group_leader, gctx); mutex_unlock(&ctx->mutex); /* err_file: */ fput(event_file); diff --git a/kernel/exit.c b/kernel/exit.c index 3076f3089919b6..46a7c2bafda25d 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -55,6 +55,8 @@ #include #include +#include "sched/tune.h" + #include #include #include @@ -775,6 +777,9 @@ void __noreturn do_exit(long code) } exit_signals(tsk); /* sets PF_EXITING */ + + schedtune_exit_task(tsk); + /* * Ensure that all new tsk->pi_lock acquisitions must observe * PF_EXITING. Serializes against futex.c:attach_to_pi_owner(). diff --git a/kernel/fork.c b/kernel/fork.c index 997ac1d584f76b..05f6855940b24b 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -745,7 +745,8 @@ static void mm_init_owner(struct mm_struct *mm, struct task_struct *p) #endif } -static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p) +static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, + struct user_namespace *user_ns) { mm->mmap = NULL; mm->mm_rb = RB_ROOT; @@ -785,6 +786,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p) if (init_new_context(p, mm)) goto fail_nocontext; + mm->user_ns = get_user_ns(user_ns); return mm; fail_nocontext: @@ -830,7 +832,7 @@ struct mm_struct *mm_alloc(void) return NULL; memset(mm, 0, sizeof(*mm)); - return mm_init(mm, current); + return mm_init(mm, current, current_user_ns()); } /* @@ -845,6 +847,7 @@ void __mmdrop(struct mm_struct *mm) destroy_context(mm); mmu_notifier_mm_destroy(mm); check_mm(mm); + put_user_ns(mm->user_ns); free_mm(mm); } EXPORT_SYMBOL_GPL(__mmdrop); @@ -1007,7 +1010,8 @@ struct mm_struct *mm_access(struct task_struct *task, unsigned int mode) mm = get_task_mm(task); if (mm && mm != current->mm && - !ptrace_may_access(task, mode)) { + !ptrace_may_access(task, mode) && + !capable(CAP_SYS_RESOURCE)) { mmput(mm); mm = ERR_PTR(-EACCES); } @@ -1126,7 +1130,7 @@ static struct mm_struct *dup_mm(struct task_struct *tsk) memcpy(mm, oldmm, sizeof(*mm)); - if (!mm_init(mm, tsk)) + if (!mm_init(mm, tsk, mm->user_ns)) goto fail_nomem; err = dup_mmap(mm, oldmm); diff --git a/kernel/irq/affinity.c b/kernel/irq/affinity.c index 17f51d63da5631..668f51b861f773 100644 --- a/kernel/irq/affinity.c +++ b/kernel/irq/affinity.c @@ -37,10 +37,10 @@ static void irq_spread_init_one(struct cpumask *irqmsk, struct cpumask *nmsk, static int get_nodes_in_cpumask(const struct cpumask *mask, nodemask_t *nodemsk) { - int n, nodes; + int n, nodes = 0; /* Calculate the number of nodes in the supplied affinity mask */ - for (n = 0, nodes = 0; n < num_online_nodes(); n++) { + for_each_online_node(n) { if (cpumask_intersects(mask, cpumask_of_node(n))) { node_set(n, *nodemsk); nodes++; @@ -81,7 +81,7 @@ struct cpumask *irq_create_affinity_masks(const struct cpumask *affinity, nodes = get_nodes_in_cpumask(affinity, &nodemsk); /* - * If the number of nodes in the mask is less than or equal the + * If the number of nodes in the mask is greater than or equal the * number of vectors we just spread the vectors across the nodes. */ if (nvec <= nodes) { diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c index 8c0a0ae43521c7..b59e6768c5e94a 100644 --- a/kernel/irq/irqdomain.c +++ b/kernel/irq/irqdomain.c @@ -1346,6 +1346,30 @@ void irq_domain_free_irqs_parent(struct irq_domain *domain, } EXPORT_SYMBOL_GPL(irq_domain_free_irqs_parent); +static void __irq_domain_activate_irq(struct irq_data *irq_data) +{ + if (irq_data && irq_data->domain) { + struct irq_domain *domain = irq_data->domain; + + if (irq_data->parent_data) + __irq_domain_activate_irq(irq_data->parent_data); + if (domain->ops->activate) + domain->ops->activate(domain, irq_data); + } +} + +static void __irq_domain_deactivate_irq(struct irq_data *irq_data) +{ + if (irq_data && irq_data->domain) { + struct irq_domain *domain = irq_data->domain; + + if (domain->ops->deactivate) + domain->ops->deactivate(domain, irq_data); + if (irq_data->parent_data) + __irq_domain_deactivate_irq(irq_data->parent_data); + } +} + /** * irq_domain_activate_irq - Call domain_ops->activate recursively to activate * interrupt @@ -1356,13 +1380,9 @@ EXPORT_SYMBOL_GPL(irq_domain_free_irqs_parent); */ void irq_domain_activate_irq(struct irq_data *irq_data) { - if (irq_data && irq_data->domain) { - struct irq_domain *domain = irq_data->domain; - - if (irq_data->parent_data) - irq_domain_activate_irq(irq_data->parent_data); - if (domain->ops->activate) - domain->ops->activate(domain, irq_data); + if (!irqd_is_activated(irq_data)) { + __irq_domain_activate_irq(irq_data); + irqd_set_activated(irq_data); } } @@ -1376,13 +1396,9 @@ void irq_domain_activate_irq(struct irq_data *irq_data) */ void irq_domain_deactivate_irq(struct irq_data *irq_data) { - if (irq_data && irq_data->domain) { - struct irq_domain *domain = irq_data->domain; - - if (domain->ops->deactivate) - domain->ops->deactivate(domain, irq_data); - if (irq_data->parent_data) - irq_domain_deactivate_irq(irq_data->parent_data); + if (irqd_is_activated(irq_data)) { + __irq_domain_deactivate_irq(irq_data); + irqd_clr_activated(irq_data); } } diff --git a/kernel/jump_label.c b/kernel/jump_label.c index 93ad6c1fb9b621..a9b8cf50059151 100644 --- a/kernel/jump_label.c +++ b/kernel/jump_label.c @@ -182,6 +182,13 @@ void static_key_slow_dec_deferred(struct static_key_deferred *key) } EXPORT_SYMBOL_GPL(static_key_slow_dec_deferred); +void static_key_deferred_flush(struct static_key_deferred *key) +{ + STATIC_KEY_CHECK_USE(); + flush_delayed_work(&key->work); +} +EXPORT_SYMBOL_GPL(static_key_deferred_flush); + void jump_label_rate_limit(struct static_key_deferred *key, unsigned long rl) { diff --git a/kernel/memremap.c b/kernel/memremap.c index b501e390bb3440..9ecedc28b928de 100644 --- a/kernel/memremap.c +++ b/kernel/memremap.c @@ -246,7 +246,9 @@ static void devm_memremap_pages_release(struct device *dev, void *data) /* pages are dead and unused, undo the arch mapping */ align_start = res->start & ~(SECTION_SIZE - 1); align_size = ALIGN(resource_size(res), SECTION_SIZE); + mem_hotplug_begin(); arch_remove_memory(align_start, align_size); + mem_hotplug_done(); untrack_pfn(NULL, PHYS_PFN(align_start), align_size); pgmap_radix_release(res); dev_WARN_ONCE(dev, pgmap->altmap && pgmap->altmap->alloc, @@ -358,7 +360,9 @@ void *devm_memremap_pages(struct device *dev, struct resource *res, if (error) goto err_pfn_remap; + mem_hotplug_begin(); error = arch_add_memory(nid, align_start, align_size, true); + mem_hotplug_done(); if (error) goto err_add_memory; diff --git a/kernel/pid_namespace.c b/kernel/pid_namespace.c index df9e8e9e0be771..eef2ce9686366a 100644 --- a/kernel/pid_namespace.c +++ b/kernel/pid_namespace.c @@ -151,8 +151,12 @@ static struct pid_namespace *create_pid_namespace(struct user_namespace *user_ns static void delayed_free_pidns(struct rcu_head *p) { - kmem_cache_free(pid_ns_cachep, - container_of(p, struct pid_namespace, rcu)); + struct pid_namespace *ns = container_of(p, struct pid_namespace, rcu); + + dec_pid_namespaces(ns->ucounts); + put_user_ns(ns->user_ns); + + kmem_cache_free(pid_ns_cachep, ns); } static void destroy_pid_namespace(struct pid_namespace *ns) @@ -162,8 +166,6 @@ static void destroy_pid_namespace(struct pid_namespace *ns) ns_free_inum(&ns->ns); for (i = 0; i < PIDMAP_ENTRIES; i++) kfree(ns->pidmap[i].page); - dec_pid_namespaces(ns->ucounts); - put_user_ns(ns->user_ns); call_rcu(&ns->rcu, delayed_free_pidns); } diff --git a/kernel/power/Kconfig b/kernel/power/Kconfig index e8517b63eb372e..f4330a2842a396 100644 --- a/kernel/power/Kconfig +++ b/kernel/power/Kconfig @@ -1,6 +1,7 @@ config SUSPEND bool "Suspend to RAM and standby" depends on ARCH_SUSPEND_POSSIBLE + select RTC_LIB default y ---help--- Allow the system to enter sleep states in which main memory is diff --git a/kernel/power/Makefile b/kernel/power/Makefile index eb4f717705baeb..80578f272be46a 100644 --- a/kernel/power/Makefile +++ b/kernel/power/Makefile @@ -14,3 +14,5 @@ obj-$(CONFIG_PM_AUTOSLEEP) += autosleep.o obj-$(CONFIG_PM_WAKELOCKS) += wakelock.o obj-$(CONFIG_MAGIC_SYSRQ) += poweroff.o + +obj-$(CONFIG_SUSPEND) += wakeup_reason.o diff --git a/kernel/power/process.c b/kernel/power/process.c index 2fba066e125fa9..68d27ae215a2a1 100644 --- a/kernel/power/process.c +++ b/kernel/power/process.c @@ -18,6 +18,7 @@ #include #include #include +#include /* * Timeout for stopping processes @@ -34,6 +35,9 @@ static int try_to_freeze_tasks(bool user_only) unsigned int elapsed_msecs; bool wakeup = false; int sleep_usecs = USEC_PER_MSEC; +#ifdef CONFIG_PM_SLEEP + char suspend_abort[MAX_SUSPEND_ABORT_LEN]; +#endif start = ktime_get_boottime(); @@ -63,6 +67,11 @@ static int try_to_freeze_tasks(bool user_only) break; if (pm_wakeup_pending()) { +#ifdef CONFIG_PM_SLEEP + pm_get_active_wakeup_sources(suspend_abort, + MAX_SUSPEND_ABORT_LEN); + log_suspend_abort_reason(suspend_abort); +#endif wakeup = true; break; } @@ -81,26 +90,27 @@ static int try_to_freeze_tasks(bool user_only) elapsed = ktime_sub(end, start); elapsed_msecs = ktime_to_ms(elapsed); - if (todo) { + if (wakeup) { pr_cont("\n"); - pr_err("Freezing of tasks %s after %d.%03d seconds " - "(%d tasks refusing to freeze, wq_busy=%d):\n", - wakeup ? "aborted" : "failed", + pr_err("Freezing of tasks aborted after %d.%03d seconds", + elapsed_msecs / 1000, elapsed_msecs % 1000); + } else if (todo) { + pr_cont("\n"); + pr_err("Freezing of tasks failed after %d.%03d seconds" + " (%d tasks refusing to freeze, wq_busy=%d):\n", elapsed_msecs / 1000, elapsed_msecs % 1000, todo - wq_busy, wq_busy); if (wq_busy) show_workqueue_state(); - if (!wakeup) { - read_lock(&tasklist_lock); - for_each_process_thread(g, p) { - if (p != current && !freezer_should_skip(p) - && freezing(p) && !frozen(p)) - sched_show_task(p); - } - read_unlock(&tasklist_lock); + read_lock(&tasklist_lock); + for_each_process_thread(g, p) { + if (p != current && !freezer_should_skip(p) + && freezing(p) && !frozen(p)) + sched_show_task(p); } + read_unlock(&tasklist_lock); } else { pr_cont("(elapsed %d.%03d seconds) ", elapsed_msecs / 1000, elapsed_msecs % 1000); diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c index 6ccb08f57fcb43..2d0c99b3f34c85 100644 --- a/kernel/power/suspend.c +++ b/kernel/power/suspend.c @@ -26,9 +26,11 @@ #include #include #include +#include #include #include #include +#include #include "power.h" @@ -322,7 +324,8 @@ void __weak arch_suspend_enable_irqs(void) */ static int suspend_enter(suspend_state_t state, bool *wakeup) { - int error; + char suspend_abort[MAX_SUSPEND_ABORT_LEN]; + int error, last_dev; error = platform_suspend_prepare(state); if (error) @@ -330,7 +333,11 @@ static int suspend_enter(suspend_state_t state, bool *wakeup) error = dpm_suspend_late(PMSG_SUSPEND); if (error) { + last_dev = suspend_stats.last_failed_dev + REC_FAILED_NUM - 1; + last_dev %= REC_FAILED_NUM; pr_err("PM: late suspend of devices failed\n"); + log_suspend_abort_reason("%s device failed to power down", + suspend_stats.failed_devs[last_dev]); goto Platform_finish; } error = platform_suspend_prepare_late(state); @@ -339,7 +346,11 @@ static int suspend_enter(suspend_state_t state, bool *wakeup) error = dpm_suspend_noirq(PMSG_SUSPEND); if (error) { + last_dev = suspend_stats.last_failed_dev + REC_FAILED_NUM - 1; + last_dev %= REC_FAILED_NUM; pr_err("PM: noirq suspend of devices failed\n"); + log_suspend_abort_reason("noirq suspend of %s device failed", + suspend_stats.failed_devs[last_dev]); goto Platform_early_resume; } error = platform_suspend_prepare_noirq(state); @@ -363,8 +374,10 @@ static int suspend_enter(suspend_state_t state, bool *wakeup) } error = disable_nonboot_cpus(); - if (error || suspend_test(TEST_CPUS)) + if (error || suspend_test(TEST_CPUS)) { + log_suspend_abort_reason("Disabling non-boot cpus failed"); goto Enable_cpus; + } arch_suspend_disable_irqs(); BUG_ON(!irqs_disabled()); @@ -380,6 +393,9 @@ static int suspend_enter(suspend_state_t state, bool *wakeup) state, false); events_check_enabled = false; } else if (*wakeup) { + pm_get_active_wakeup_sources(suspend_abort, + MAX_SUSPEND_ABORT_LEN); + log_suspend_abort_reason(suspend_abort); error = -EBUSY; } syscore_resume(); @@ -427,6 +443,7 @@ int suspend_devices_and_enter(suspend_state_t state) error = dpm_suspend_start(PMSG_SUSPEND); if (error) { pr_err("PM: Some devices failed to suspend, or early wake event detected\n"); + log_suspend_abort_reason("Some devices failed to suspend, or early wake event detected"); goto Recover_platform; } suspend_test_finish("suspend devices"); @@ -527,6 +544,18 @@ static int enter_state(suspend_state_t state) return error; } +static void pm_suspend_marker(char *annotation) +{ + struct timespec ts; + struct rtc_time tm; + + getnstimeofday(&ts); + rtc_time_to_tm(ts.tv_sec, &tm); + pr_info("PM: suspend %s %d-%02d-%02d %02d:%02d:%02d.%09lu UTC\n", + annotation, tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, + tm.tm_hour, tm.tm_min, tm.tm_sec, ts.tv_nsec); +} + /** * pm_suspend - Externally visible function for suspending the system. * @state: System sleep state to enter. @@ -541,6 +570,7 @@ int pm_suspend(suspend_state_t state) if (state <= PM_SUSPEND_ON || state >= PM_SUSPEND_MAX) return -EINVAL; + pm_suspend_marker("entry"); error = enter_state(state); if (error) { suspend_stats.fail++; @@ -548,6 +578,7 @@ int pm_suspend(suspend_state_t state) } else { suspend_stats.success++; } + pm_suspend_marker("exit"); return error; } EXPORT_SYMBOL(pm_suspend); diff --git a/kernel/power/wakeup_reason.c b/kernel/power/wakeup_reason.c new file mode 100644 index 00000000000000..252611fad2fee2 --- /dev/null +++ b/kernel/power/wakeup_reason.c @@ -0,0 +1,225 @@ +/* + * kernel/power/wakeup_reason.c + * + * Logs the reasons which caused the kernel to resume from + * the suspend mode. + * + * Copyright (C) 2014 Google, Inc. + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +#define MAX_WAKEUP_REASON_IRQS 32 +static int irq_list[MAX_WAKEUP_REASON_IRQS]; +static int irqcount; +static bool suspend_abort; +static char abort_reason[MAX_SUSPEND_ABORT_LEN]; +static struct kobject *wakeup_reason; +static DEFINE_SPINLOCK(resume_reason_lock); + +static ktime_t last_monotime; /* monotonic time before last suspend */ +static ktime_t curr_monotime; /* monotonic time after last suspend */ +static ktime_t last_stime; /* monotonic boottime offset before last suspend */ +static ktime_t curr_stime; /* monotonic boottime offset after last suspend */ + +static ssize_t last_resume_reason_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + int irq_no, buf_offset = 0; + struct irq_desc *desc; + spin_lock(&resume_reason_lock); + if (suspend_abort) { + buf_offset = sprintf(buf, "Abort: %s", abort_reason); + } else { + for (irq_no = 0; irq_no < irqcount; irq_no++) { + desc = irq_to_desc(irq_list[irq_no]); + if (desc && desc->action && desc->action->name) + buf_offset += sprintf(buf + buf_offset, "%d %s\n", + irq_list[irq_no], desc->action->name); + else + buf_offset += sprintf(buf + buf_offset, "%d\n", + irq_list[irq_no]); + } + } + spin_unlock(&resume_reason_lock); + return buf_offset; +} + +static ssize_t last_suspend_time_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct timespec sleep_time; + struct timespec total_time; + struct timespec suspend_resume_time; + + /* + * total_time is calculated from monotonic bootoffsets because + * unlike CLOCK_MONOTONIC it include the time spent in suspend state. + */ + total_time = ktime_to_timespec(ktime_sub(curr_stime, last_stime)); + + /* + * suspend_resume_time is calculated as monotonic (CLOCK_MONOTONIC) + * time interval before entering suspend and post suspend. + */ + suspend_resume_time = ktime_to_timespec(ktime_sub(curr_monotime, last_monotime)); + + /* sleep_time = total_time - suspend_resume_time */ + sleep_time = timespec_sub(total_time, suspend_resume_time); + + /* Export suspend_resume_time and sleep_time in pair here. */ + return sprintf(buf, "%lu.%09lu %lu.%09lu\n", + suspend_resume_time.tv_sec, suspend_resume_time.tv_nsec, + sleep_time.tv_sec, sleep_time.tv_nsec); +} + +static struct kobj_attribute resume_reason = __ATTR_RO(last_resume_reason); +static struct kobj_attribute suspend_time = __ATTR_RO(last_suspend_time); + +static struct attribute *attrs[] = { + &resume_reason.attr, + &suspend_time.attr, + NULL, +}; +static struct attribute_group attr_group = { + .attrs = attrs, +}; + +/* + * logs all the wake up reasons to the kernel + * stores the irqs to expose them to the userspace via sysfs + */ +void log_wakeup_reason(int irq) +{ + struct irq_desc *desc; + desc = irq_to_desc(irq); + if (desc && desc->action && desc->action->name) + printk(KERN_INFO "Resume caused by IRQ %d, %s\n", irq, + desc->action->name); + else + printk(KERN_INFO "Resume caused by IRQ %d\n", irq); + + spin_lock(&resume_reason_lock); + if (irqcount == MAX_WAKEUP_REASON_IRQS) { + spin_unlock(&resume_reason_lock); + printk(KERN_WARNING "Resume caused by more than %d IRQs\n", + MAX_WAKEUP_REASON_IRQS); + return; + } + + irq_list[irqcount++] = irq; + spin_unlock(&resume_reason_lock); +} + +int check_wakeup_reason(int irq) +{ + int irq_no; + int ret = false; + + spin_lock(&resume_reason_lock); + for (irq_no = 0; irq_no < irqcount; irq_no++) + if (irq_list[irq_no] == irq) { + ret = true; + break; + } + spin_unlock(&resume_reason_lock); + return ret; +} + +void log_suspend_abort_reason(const char *fmt, ...) +{ + va_list args; + + spin_lock(&resume_reason_lock); + + //Suspend abort reason has already been logged. + if (suspend_abort) { + spin_unlock(&resume_reason_lock); + return; + } + + suspend_abort = true; + va_start(args, fmt); + vsnprintf(abort_reason, MAX_SUSPEND_ABORT_LEN, fmt, args); + va_end(args); + spin_unlock(&resume_reason_lock); +} + +/* Detects a suspend and clears all the previous wake up reasons*/ +static int wakeup_reason_pm_event(struct notifier_block *notifier, + unsigned long pm_event, void *unused) +{ + switch (pm_event) { + case PM_SUSPEND_PREPARE: + spin_lock(&resume_reason_lock); + irqcount = 0; + suspend_abort = false; + spin_unlock(&resume_reason_lock); + /* monotonic time since boot */ + last_monotime = ktime_get(); + /* monotonic time since boot including the time spent in suspend */ + last_stime = ktime_get_boottime(); + break; + case PM_POST_SUSPEND: + /* monotonic time since boot */ + curr_monotime = ktime_get(); + /* monotonic time since boot including the time spent in suspend */ + curr_stime = ktime_get_boottime(); + break; + default: + break; + } + return NOTIFY_DONE; +} + +static struct notifier_block wakeup_reason_pm_notifier_block = { + .notifier_call = wakeup_reason_pm_event, +}; + +/* Initializes the sysfs parameter + * registers the pm_event notifier + */ +int __init wakeup_reason_init(void) +{ + int retval; + + retval = register_pm_notifier(&wakeup_reason_pm_notifier_block); + if (retval) + printk(KERN_WARNING "[%s] failed to register PM notifier %d\n", + __func__, retval); + + wakeup_reason = kobject_create_and_add("wakeup_reasons", kernel_kobj); + if (!wakeup_reason) { + printk(KERN_WARNING "[%s] failed to create a sysfs kobject\n", + __func__); + return 1; + } + retval = sysfs_create_group(wakeup_reason, &attr_group); + if (retval) { + kobject_put(wakeup_reason); + printk(KERN_WARNING "[%s] failed to create a sysfs group %d\n", + __func__, retval); + } + return 0; +} + +late_initcall(wakeup_reason_init); diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index f7a55e9ff2f762..b38f3fb2a00669 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -56,6 +56,10 @@ #include "braille.h" #include "internal.h" +#ifdef CONFIG_EARLY_PRINTK_DIRECT +extern void printascii(char *); +#endif + int console_printk[4] = { CONSOLE_LOGLEVEL_DEFAULT, /* console_loglevel */ MESSAGE_LOGLEVEL_DEFAULT, /* default_message_loglevel */ @@ -1870,6 +1874,10 @@ asmlinkage int vprintk_emit(int facility, int level, } } +#ifdef CONFIG_EARLY_PRINTK_DIRECT + printascii(text); +#endif + if (level == LOGLEVEL_DEFAULT) level = default_message_loglevel; diff --git a/kernel/ptrace.c b/kernel/ptrace.c index e6474f7272ec2c..49ba7c1ade9d07 100644 --- a/kernel/ptrace.c +++ b/kernel/ptrace.c @@ -27,6 +27,35 @@ #include #include +/* + * Access another process' address space via ptrace. + * Source/target buffer must be kernel space, + * Do not walk the page table directly, use get_user_pages + */ +int ptrace_access_vm(struct task_struct *tsk, unsigned long addr, + void *buf, int len, unsigned int gup_flags) +{ + struct mm_struct *mm; + int ret; + + mm = get_task_mm(tsk); + if (!mm) + return 0; + + if (!tsk->ptrace || + (current != tsk->parent) || + ((get_dumpable(mm) != SUID_DUMP_USER) && + !ptracer_capable(tsk, mm->user_ns))) { + mmput(mm); + return 0; + } + + ret = __access_remote_vm(tsk, mm, addr, buf, len, gup_flags); + mmput(mm); + + return ret; +} + /* * ptrace a task: make the debugger its new parent and @@ -39,6 +68,9 @@ void __ptrace_link(struct task_struct *child, struct task_struct *new_parent) BUG_ON(!list_empty(&child->ptrace_entry)); list_add(&child->ptrace_entry, &new_parent->ptraced); child->parent = new_parent; + rcu_read_lock(); + child->ptracer_cred = get_cred(__task_cred(new_parent)); + rcu_read_unlock(); } /** @@ -71,12 +103,16 @@ void __ptrace_link(struct task_struct *child, struct task_struct *new_parent) */ void __ptrace_unlink(struct task_struct *child) { + const struct cred *old_cred; BUG_ON(!child->ptrace); clear_tsk_thread_flag(child, TIF_SYSCALL_TRACE); child->parent = child->real_parent; list_del_init(&child->ptrace_entry); + old_cred = child->ptracer_cred; + child->ptracer_cred = NULL; + put_cred(old_cred); spin_lock(&child->sighand->siglock); child->ptrace = 0; @@ -220,7 +256,7 @@ static int ptrace_has_cap(struct user_namespace *ns, unsigned int mode) static int __ptrace_may_access(struct task_struct *task, unsigned int mode) { const struct cred *cred = current_cred(), *tcred; - int dumpable = 0; + struct mm_struct *mm; kuid_t caller_uid; kgid_t caller_gid; @@ -271,16 +307,11 @@ static int __ptrace_may_access(struct task_struct *task, unsigned int mode) return -EPERM; ok: rcu_read_unlock(); - smp_rmb(); - if (task->mm) - dumpable = get_dumpable(task->mm); - rcu_read_lock(); - if (dumpable != SUID_DUMP_USER && - !ptrace_has_cap(__task_cred(task)->user_ns, mode)) { - rcu_read_unlock(); - return -EPERM; - } - rcu_read_unlock(); + mm = task->mm; + if (mm && + ((get_dumpable(mm) != SUID_DUMP_USER) && + !ptrace_has_cap(mm->user_ns, mode))) + return -EPERM; return security_ptrace_access_check(task, mode); } @@ -344,10 +375,6 @@ static int ptrace_attach(struct task_struct *task, long request, if (seize) flags |= PT_SEIZED; - rcu_read_lock(); - if (ns_capable(__task_cred(task)->user_ns, CAP_SYS_PTRACE)) - flags |= PT_PTRACE_CAP; - rcu_read_unlock(); task->ptrace = flags; __ptrace_link(task, current); @@ -537,7 +564,8 @@ int ptrace_readdata(struct task_struct *tsk, unsigned long src, char __user *dst int this_len, retval; this_len = (len > sizeof(buf)) ? sizeof(buf) : len; - retval = access_process_vm(tsk, src, buf, this_len, FOLL_FORCE); + retval = ptrace_access_vm(tsk, src, buf, this_len, FOLL_FORCE); + if (!retval) { if (copied) break; @@ -564,7 +592,7 @@ int ptrace_writedata(struct task_struct *tsk, char __user *src, unsigned long ds this_len = (len > sizeof(buf)) ? sizeof(buf) : len; if (copy_from_user(buf, src, this_len)) return -EFAULT; - retval = access_process_vm(tsk, dst, buf, this_len, + retval = ptrace_access_vm(tsk, dst, buf, this_len, FOLL_FORCE | FOLL_WRITE); if (!retval) { if (copied) @@ -1128,7 +1156,7 @@ int generic_ptrace_peekdata(struct task_struct *tsk, unsigned long addr, unsigned long tmp; int copied; - copied = access_process_vm(tsk, addr, &tmp, sizeof(tmp), FOLL_FORCE); + copied = ptrace_access_vm(tsk, addr, &tmp, sizeof(tmp), FOLL_FORCE); if (copied != sizeof(tmp)) return -EIO; return put_user(tmp, (unsigned long __user *)data); @@ -1139,7 +1167,7 @@ int generic_ptrace_pokedata(struct task_struct *tsk, unsigned long addr, { int copied; - copied = access_process_vm(tsk, addr, &data, sizeof(data), + copied = ptrace_access_vm(tsk, addr, &data, sizeof(data), FOLL_FORCE | FOLL_WRITE); return (copied == sizeof(data)) ? 0 : -EIO; } @@ -1157,7 +1185,7 @@ int compat_ptrace_request(struct task_struct *child, compat_long_t request, switch (request) { case PTRACE_PEEKTEXT: case PTRACE_PEEKDATA: - ret = access_process_vm(child, addr, &word, sizeof(word), + ret = ptrace_access_vm(child, addr, &word, sizeof(word), FOLL_FORCE); if (ret != sizeof(word)) ret = -EIO; @@ -1167,7 +1195,7 @@ int compat_ptrace_request(struct task_struct *child, compat_long_t request, case PTRACE_POKETEXT: case PTRACE_POKEDATA: - ret = access_process_vm(child, addr, &data, sizeof(data), + ret = ptrace_access_vm(child, addr, &data, sizeof(data), FOLL_FORCE | FOLL_WRITE); ret = (ret != sizeof(data) ? -EIO : 0); break; diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h index 80adef7d4c3d01..0d6ff3e471be6c 100644 --- a/kernel/rcu/rcu.h +++ b/kernel/rcu/rcu.h @@ -136,6 +136,7 @@ int rcu_jiffies_till_stall_check(void); #define TPS(x) tracepoint_string(x) void rcu_early_boot_tests(void); +void rcu_test_sync_prims(void); /* * This function really isn't for public consumption, but RCU is special in diff --git a/kernel/rcu/tiny.c b/kernel/rcu/tiny.c index 1898559e6b60dd..b23a4d076f3d2c 100644 --- a/kernel/rcu/tiny.c +++ b/kernel/rcu/tiny.c @@ -185,9 +185,6 @@ static __latent_entropy void rcu_process_callbacks(struct softirq_action *unused * benefits of doing might_sleep() to reduce latency.) * * Cool, huh? (Due to Josh Triplett.) - * - * But we want to make this a static inline later. The cond_resched() - * currently makes this problematic. */ void synchronize_sched(void) { @@ -195,7 +192,6 @@ void synchronize_sched(void) lock_is_held(&rcu_lock_map) || lock_is_held(&rcu_sched_lock_map), "Illegal synchronize_sched() in RCU read-side critical section"); - cond_resched(); } EXPORT_SYMBOL_GPL(synchronize_sched); diff --git a/kernel/rcu/tiny_plugin.h b/kernel/rcu/tiny_plugin.h index 196f0302e2f432..c64b827ecbca19 100644 --- a/kernel/rcu/tiny_plugin.h +++ b/kernel/rcu/tiny_plugin.h @@ -60,12 +60,17 @@ EXPORT_SYMBOL_GPL(rcu_scheduler_active); /* * During boot, we forgive RCU lockdep issues. After this function is - * invoked, we start taking RCU lockdep issues seriously. + * invoked, we start taking RCU lockdep issues seriously. Note that unlike + * Tree RCU, Tiny RCU transitions directly from RCU_SCHEDULER_INACTIVE + * to RCU_SCHEDULER_RUNNING, skipping the RCU_SCHEDULER_INIT stage. + * The reason for this is that Tiny RCU does not need kthreads, so does + * not have to care about the fact that the scheduler is half-initialized + * at a certain phase of the boot process. */ void __init rcu_scheduler_starting(void) { WARN_ON(nr_context_switches() > 0); - rcu_scheduler_active = 1; + rcu_scheduler_active = RCU_SCHEDULER_RUNNING; } #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 69a5611a7e7c03..10f62c6f48e77a 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -127,13 +127,16 @@ int rcu_num_nodes __read_mostly = NUM_RCU_NODES; /* Total # rcu_nodes in use. */ int sysctl_panic_on_rcu_stall __read_mostly; /* - * The rcu_scheduler_active variable transitions from zero to one just - * before the first task is spawned. So when this variable is zero, RCU - * can assume that there is but one task, allowing RCU to (for example) + * The rcu_scheduler_active variable is initialized to the value + * RCU_SCHEDULER_INACTIVE and transitions RCU_SCHEDULER_INIT just before the + * first task is spawned. So when this variable is RCU_SCHEDULER_INACTIVE, + * RCU can assume that there is but one task, allowing RCU to (for example) * optimize synchronize_rcu() to a simple barrier(). When this variable - * is one, RCU must actually do all the hard work required to detect real - * grace periods. This variable is also used to suppress boot-time false - * positives from lockdep-RCU error checking. + * is RCU_SCHEDULER_INIT, RCU must actually do all the hard work required + * to detect real grace periods. This variable is also used to suppress + * boot-time false positives from lockdep-RCU error checking. Finally, it + * transitions from RCU_SCHEDULER_INIT to RCU_SCHEDULER_RUNNING after RCU + * is fully initialized, including all of its kthreads having been spawned. */ int rcu_scheduler_active __read_mostly; EXPORT_SYMBOL_GPL(rcu_scheduler_active); @@ -3985,18 +3988,22 @@ static int __init rcu_spawn_gp_kthread(void) early_initcall(rcu_spawn_gp_kthread); /* - * This function is invoked towards the end of the scheduler's initialization - * process. Before this is called, the idle task might contain - * RCU read-side critical sections (during which time, this idle - * task is booting the system). After this function is called, the - * idle tasks are prohibited from containing RCU read-side critical - * sections. This function also enables RCU lockdep checking. + * This function is invoked towards the end of the scheduler's + * initialization process. Before this is called, the idle task might + * contain synchronous grace-period primitives (during which time, this idle + * task is booting the system, and such primitives are no-ops). After this + * function is called, any synchronous grace-period primitives are run as + * expedited, with the requesting task driving the grace period forward. + * A later core_initcall() rcu_exp_runtime_mode() will switch to full + * runtime RCU functionality. */ void rcu_scheduler_starting(void) { WARN_ON(num_online_cpus() != 1); WARN_ON(nr_context_switches() > 0); - rcu_scheduler_active = 1; + rcu_test_sync_prims(); + rcu_scheduler_active = RCU_SCHEDULER_INIT; + rcu_test_sync_prims(); } /* diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h index 24343eb87b582e..78eba4120d4658 100644 --- a/kernel/rcu/tree_exp.h +++ b/kernel/rcu/tree_exp.h @@ -521,6 +521,20 @@ struct rcu_exp_work { struct work_struct rew_work; }; +/* + * Common code to drive an expedited grace period forward, used by + * workqueues and mid-boot-time tasks. + */ +static void rcu_exp_sel_wait_wake(struct rcu_state *rsp, + smp_call_func_t func, unsigned long s) +{ + /* Initialize the rcu_node tree in preparation for the wait. */ + sync_rcu_exp_select_cpus(rsp, func); + + /* Wait and clean up, including waking everyone. */ + rcu_exp_wait_wake(rsp, s); +} + /* * Work-queue handler to drive an expedited grace period forward. */ @@ -528,12 +542,8 @@ static void wait_rcu_exp_gp(struct work_struct *wp) { struct rcu_exp_work *rewp; - /* Initialize the rcu_node tree in preparation for the wait. */ rewp = container_of(wp, struct rcu_exp_work, rew_work); - sync_rcu_exp_select_cpus(rewp->rew_rsp, rewp->rew_func); - - /* Wait and clean up, including waking everyone. */ - rcu_exp_wait_wake(rewp->rew_rsp, rewp->rew_s); + rcu_exp_sel_wait_wake(rewp->rew_rsp, rewp->rew_func, rewp->rew_s); } /* @@ -559,12 +569,18 @@ static void _synchronize_rcu_expedited(struct rcu_state *rsp, if (exp_funnel_lock(rsp, s)) return; /* Someone else did our work for us. */ - /* Marshall arguments and schedule the expedited grace period. */ - rew.rew_func = func; - rew.rew_rsp = rsp; - rew.rew_s = s; - INIT_WORK_ONSTACK(&rew.rew_work, wait_rcu_exp_gp); - schedule_work(&rew.rew_work); + /* Ensure that load happens before action based on it. */ + if (unlikely(rcu_scheduler_active == RCU_SCHEDULER_INIT)) { + /* Direct call during scheduler init and early_initcalls(). */ + rcu_exp_sel_wait_wake(rsp, func, s); + } else { + /* Marshall arguments & schedule the expedited grace period. */ + rew.rew_func = func; + rew.rew_rsp = rsp; + rew.rew_s = s; + INIT_WORK_ONSTACK(&rew.rew_work, wait_rcu_exp_gp); + schedule_work(&rew.rew_work); + } /* Wait for expedited grace period to complete. */ rdp = per_cpu_ptr(rsp->rda, raw_smp_processor_id()); @@ -666,6 +682,8 @@ void synchronize_rcu_expedited(void) { struct rcu_state *rsp = rcu_state_p; + if (rcu_scheduler_active == RCU_SCHEDULER_INACTIVE) + return; _synchronize_rcu_expedited(rsp, sync_rcu_exp_handler); } EXPORT_SYMBOL_GPL(synchronize_rcu_expedited); @@ -683,3 +701,15 @@ void synchronize_rcu_expedited(void) EXPORT_SYMBOL_GPL(synchronize_rcu_expedited); #endif /* #else #ifdef CONFIG_PREEMPT_RCU */ + +/* + * Switch to run-time mode once Tree RCU has fully initialized. + */ +static int __init rcu_exp_runtime_mode(void) +{ + rcu_test_sync_prims(); + rcu_scheduler_active = RCU_SCHEDULER_RUNNING; + rcu_test_sync_prims(); + return 0; +} +core_initcall(rcu_exp_runtime_mode); diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 85c5a883c6e310..56583e764ebf39 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -670,7 +670,7 @@ void synchronize_rcu(void) lock_is_held(&rcu_lock_map) || lock_is_held(&rcu_sched_lock_map), "Illegal synchronize_rcu() in RCU read-side critical section"); - if (!rcu_scheduler_active) + if (rcu_scheduler_active == RCU_SCHEDULER_INACTIVE) return; if (rcu_gp_is_expedited()) synchronize_rcu_expedited(); diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c index f19271dce0a978..4f6db7e6a1179e 100644 --- a/kernel/rcu/update.c +++ b/kernel/rcu/update.c @@ -121,11 +121,14 @@ EXPORT_SYMBOL(rcu_read_lock_sched_held); * Should expedited grace-period primitives always fall back to their * non-expedited counterparts? Intended for use within RCU. Note * that if the user specifies both rcu_expedited and rcu_normal, then - * rcu_normal wins. + * rcu_normal wins. (Except during the time period during boot from + * when the first task is spawned until the rcu_exp_runtime_mode() + * core_initcall() is invoked, at which point everything is expedited.) */ bool rcu_gp_is_normal(void) { - return READ_ONCE(rcu_normal); + return READ_ONCE(rcu_normal) && + rcu_scheduler_active != RCU_SCHEDULER_INIT; } EXPORT_SYMBOL_GPL(rcu_gp_is_normal); @@ -135,13 +138,14 @@ static atomic_t rcu_expedited_nesting = /* * Should normal grace-period primitives be expedited? Intended for * use within RCU. Note that this function takes the rcu_expedited - * sysfs/boot variable into account as well as the rcu_expedite_gp() - * nesting. So looping on rcu_unexpedite_gp() until rcu_gp_is_expedited() - * returns false is a -really- bad idea. + * sysfs/boot variable and rcu_scheduler_active into account as well + * as the rcu_expedite_gp() nesting. So looping on rcu_unexpedite_gp() + * until rcu_gp_is_expedited() returns false is a -really- bad idea. */ bool rcu_gp_is_expedited(void) { - return rcu_expedited || atomic_read(&rcu_expedited_nesting); + return rcu_expedited || atomic_read(&rcu_expedited_nesting) || + rcu_scheduler_active == RCU_SCHEDULER_INIT; } EXPORT_SYMBOL_GPL(rcu_gp_is_expedited); @@ -257,7 +261,7 @@ EXPORT_SYMBOL_GPL(rcu_callback_map); int notrace debug_lockdep_rcu_enabled(void) { - return rcu_scheduler_active && debug_locks && + return rcu_scheduler_active != RCU_SCHEDULER_INACTIVE && debug_locks && current->lockdep_recursion == 0; } EXPORT_SYMBOL_GPL(debug_lockdep_rcu_enabled); @@ -591,7 +595,7 @@ EXPORT_SYMBOL_GPL(call_rcu_tasks); void synchronize_rcu_tasks(void) { /* Complain if the scheduler has not started. */ - RCU_LOCKDEP_WARN(!rcu_scheduler_active, + RCU_LOCKDEP_WARN(rcu_scheduler_active == RCU_SCHEDULER_INACTIVE, "synchronize_rcu_tasks called too soon"); /* Wait for the grace period. */ @@ -813,6 +817,23 @@ static void rcu_spawn_tasks_kthread(void) #endif /* #ifdef CONFIG_TASKS_RCU */ +/* + * Test each non-SRCU synchronous grace-period wait API. This is + * useful just after a change in mode for these primitives, and + * during early boot. + */ +void rcu_test_sync_prims(void) +{ + if (!IS_ENABLED(CONFIG_PROVE_RCU)) + return; + synchronize_rcu(); + synchronize_rcu_bh(); + synchronize_sched(); + synchronize_rcu_expedited(); + synchronize_rcu_bh_expedited(); + synchronize_sched_expedited(); +} + #ifdef CONFIG_PROVE_RCU /* @@ -865,6 +886,7 @@ void rcu_early_boot_tests(void) early_boot_test_call_rcu_bh(); if (rcu_self_test_sched) early_boot_test_call_rcu_sched(); + rcu_test_sync_prims(); } static int rcu_verify_early_boot_tests(void) diff --git a/kernel/relay.c b/kernel/relay.c index da79a109dbebc6..8f18d314a96a49 100644 --- a/kernel/relay.c +++ b/kernel/relay.c @@ -809,11 +809,11 @@ void relay_subbufs_consumed(struct rchan *chan, { struct rchan_buf *buf; - if (!chan) + if (!chan || cpu >= NR_CPUS) return; buf = *per_cpu_ptr(chan->buf, cpu); - if (cpu >= NR_CPUS || !buf || subbufs_consumed > chan->n_subbufs) + if (!buf || subbufs_consumed > chan->n_subbufs) return; if (subbufs_consumed > buf->subbufs_produced - buf->subbufs_consumed) diff --git a/kernel/sched/Makefile b/kernel/sched/Makefile index 5e59b832ae2b4b..62793dbd2e1244 100644 --- a/kernel/sched/Makefile +++ b/kernel/sched/Makefile @@ -18,10 +18,12 @@ endif obj-y += core.o loadavg.o clock.o cputime.o obj-y += idle_task.o fair.o rt.o deadline.o stop_task.o obj-y += wait.o swait.o completion.o idle.o -obj-$(CONFIG_SMP) += cpupri.o cpudeadline.o +obj-$(CONFIG_SMP) += cpupri.o cpudeadline.o energy.o +obj-$(CONFIG_SCHED_WALT) += walt.o obj-$(CONFIG_SCHED_AUTOGROUP) += auto_group.o obj-$(CONFIG_SCHEDSTATS) += stats.o obj-$(CONFIG_SCHED_DEBUG) += debug.o +obj-$(CONFIG_SCHED_TUNE) += tune.o obj-$(CONFIG_CGROUP_CPUACCT) += cpuacct.o obj-$(CONFIG_CPU_FREQ) += cpufreq.o -obj-$(CONFIG_CPU_FREQ_GOV_SCHEDUTIL) += cpufreq_schedutil.o +obj-$(CONFIG_CPU_FREQ_GOV_SCHED) += cpufreq_sched.o diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 154fd689fe02e9..b250aacec534c4 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -90,6 +90,7 @@ #define CREATE_TRACE_POINTS #include +#include "walt.h" DEFINE_MUTEX(sched_domains_mutex); DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); @@ -156,6 +157,18 @@ int sysctl_sched_rt_runtime = 950000; /* cpus with isolated domains */ cpumask_var_t cpu_isolated_map; +struct rq * +lock_rq_of(struct task_struct *p, struct rq_flags *flags) +{ + return task_rq_lock(p, flags); +} + +void +unlock_rq_of(struct rq *rq, struct task_struct *p, struct rq_flags *flags) +{ + task_rq_unlock(rq, p, flags); +} + /* * this_rq_lock - lock this runqueue and disable interrupts. */ @@ -1253,6 +1266,8 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu) p->sched_class->migrate_task_rq(p); p->se.nr_migrations++; perf_event_task_migrate(p); + + walt_fixup_busy_time(p, new_cpu); } __set_task_cpu(p, new_cpu); @@ -2009,6 +2024,10 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) { unsigned long flags; int cpu, success = 0; +#ifdef CONFIG_SMP + struct rq *rq; + u64 wallclock; +#endif /* * If we are going to wake up a thread waiting for CONDITION we @@ -2082,14 +2101,24 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) */ smp_cond_load_acquire(&p->on_cpu, !VAL); + rq = cpu_rq(task_cpu(p)); + + raw_spin_lock(&rq->lock); + wallclock = walt_ktime_clock(); + walt_update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); + walt_update_task_ravg(p, rq, TASK_WAKE, wallclock, 0); + raw_spin_unlock(&rq->lock); + p->sched_contributes_to_load = !!task_contributes_to_load(p); p->state = TASK_WAKING; cpu = select_task_rq(p, p->wake_cpu, SD_BALANCE_WAKE, wake_flags); + if (task_cpu(p) != cpu) { wake_flags |= WF_MIGRATED; set_task_cpu(p, cpu); } + #endif /* CONFIG_SMP */ ttwu_queue(p, cpu, wake_flags); @@ -2139,8 +2168,13 @@ static void try_to_wake_up_local(struct task_struct *p, struct pin_cookie cookie trace_sched_waking(p); - if (!task_on_rq_queued(p)) + if (!task_on_rq_queued(p)) { + u64 wallclock = walt_ktime_clock(); + + walt_update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); + walt_update_task_ravg(p, rq, TASK_WAKE, wallclock, 0); ttwu_activate(rq, p, ENQUEUE_WAKEUP); + } ttwu_do_wakeup(rq, p, 0, cookie); ttwu_stat(p, smp_processor_id(), 0); @@ -2205,6 +2239,7 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p) p->se.nr_migrations = 0; p->se.vruntime = 0; INIT_LIST_HEAD(&p->se.group_node); + walt_init_new_task_load(p); #ifdef CONFIG_FAIR_GROUP_SCHED p->se.cfs_rq = NULL; @@ -2564,6 +2599,9 @@ void wake_up_new_task(struct task_struct *p) struct rq *rq; raw_spin_lock_irqsave(&p->pi_lock, rf.flags); + + walt_init_new_task_load(p); + p->state = TASK_RUNNING; #ifdef CONFIG_SMP /* @@ -2579,7 +2617,9 @@ void wake_up_new_task(struct task_struct *p) rq = __task_rq_lock(p, &rf); post_init_entity_util_avg(&p->se); - activate_task(rq, p, 0); + walt_mark_task_starting(p); + + activate_task(rq, p, ENQUEUE_WAKEUP_NEW); p->on_rq = TASK_ON_RQ_QUEUED; trace_sched_wakeup_new(p); check_preempt_curr(rq, p, WF_FORK); @@ -2964,6 +3004,36 @@ unsigned long nr_iowait_cpu(int cpu) return atomic_read(&this->nr_iowait); } +#ifdef CONFIG_CPU_QUIET +u64 nr_running_integral(unsigned int cpu) +{ + unsigned int seqcnt; + u64 integral; + struct rq *q; + + if (cpu >= nr_cpu_ids) + return 0; + + q = cpu_rq(cpu); + + /* + * Update average to avoid reading stalled value if there were + * no run-queue changes for a long time. On the other hand if + * the changes are happening right now, just read current value + * directly. + */ + + seqcnt = read_seqcount_begin(&q->ave_seqcnt); + integral = do_nr_running_integral(q); + if (read_seqcount_retry(&q->ave_seqcnt, seqcnt)) { + read_seqcount_begin(&q->ave_seqcnt); + integral = q->nr_running_integral; + } + + return integral; +} +#endif + void get_iowait_load(unsigned long *nr_waiters, unsigned long *load) { struct rq *rq = this_rq(); @@ -3068,6 +3138,93 @@ unsigned long long task_sched_runtime(struct task_struct *p) return ns; } +#ifdef CONFIG_CPU_FREQ_GOV_SCHED + +static inline +unsigned long add_capacity_margin(unsigned long cpu_capacity) +{ + cpu_capacity = cpu_capacity * capacity_margin; + cpu_capacity /= SCHED_CAPACITY_SCALE; + return cpu_capacity; +} + +static inline +unsigned long sum_capacity_reqs(unsigned long cfs_cap, + struct sched_capacity_reqs *scr) +{ + unsigned long total = add_capacity_margin(cfs_cap + scr->rt); + return total += scr->dl; +} + +static void sched_freq_tick_pelt(int cpu) +{ + unsigned long cpu_utilization = capacity_max; + unsigned long capacity_curr = capacity_curr_of(cpu); + struct sched_capacity_reqs *scr; + + scr = &per_cpu(cpu_sched_capacity_reqs, cpu); + if (sum_capacity_reqs(cpu_utilization, scr) < capacity_curr) + return; + + /* + * To make free room for a task that is building up its "real" + * utilization and to harm its performance the least, request + * a jump to a higher OPP as soon as the margin of free capacity + * is impacted (specified by capacity_margin). + */ + set_cfs_cpu_capacity(cpu, true, cpu_utilization); +} + +#ifdef CONFIG_SCHED_WALT +static void sched_freq_tick_walt(int cpu) +{ + unsigned long cpu_utilization = cpu_util(cpu); + unsigned long capacity_curr = capacity_curr_of(cpu); + + if (walt_disabled || !sysctl_sched_use_walt_cpu_util) + return sched_freq_tick_pelt(cpu); + + /* + * Add a margin to the WALT utilization. + * NOTE: WALT tracks a single CPU signal for all the scheduling + * classes, thus this margin is going to be added to the DL class as + * well, which is something we do not do in sched_freq_tick_pelt case. + */ + cpu_utilization = add_capacity_margin(cpu_utilization); + if (cpu_utilization <= capacity_curr) + return; + + /* + * It is likely that the load is growing so we + * keep the added margin in our request as an + * extra boost. + */ + set_cfs_cpu_capacity(cpu, true, cpu_utilization); + +} +#define _sched_freq_tick(cpu) sched_freq_tick_walt(cpu) +#else +#define _sched_freq_tick(cpu) sched_freq_tick_pelt(cpu) +#endif /* CONFIG_SCHED_WALT */ + +static void sched_freq_tick(int cpu) +{ + unsigned long capacity_orig, capacity_curr; + + if (!sched_freq()) + return; + + capacity_orig = capacity_orig_of(cpu); + capacity_curr = capacity_curr_of(cpu); + if (capacity_curr == capacity_orig) + return; + + _sched_freq_tick(cpu); +} +#else +static inline void sched_freq_tick(int cpu) { } +#endif /* CONFIG_CPU_FREQ_GOV_SCHED */ + /* * This function gets called by the timer code, with HZ frequency. * We call it with interrupts disabled. @@ -3081,9 +3238,12 @@ void scheduler_tick(void) sched_clock_tick(); raw_spin_lock(&rq->lock); + walt_set_window_start(rq); update_rq_clock(rq); curr->sched_class->task_tick(rq, curr, 0); cpu_load_update_active(rq); + walt_update_task_ravg(rq->curr, rq, TASK_UPDATE, + walt_ktime_clock(), 0); calc_global_load_tick(rq); raw_spin_unlock(&rq->lock); @@ -3094,6 +3254,8 @@ void scheduler_tick(void) trigger_load_balance(rq); #endif rq_last_tick_reset(rq); + + sched_freq_tick(cpu); } #ifdef CONFIG_NO_HZ_FULL @@ -3337,6 +3499,7 @@ static void __sched notrace __schedule(bool preempt) struct pin_cookie cookie; struct rq *rq; int cpu; + u64 wallclock; cpu = smp_processor_id(); rq = cpu_rq(cpu); @@ -3389,6 +3552,9 @@ static void __sched notrace __schedule(bool preempt) update_rq_clock(rq); next = pick_next_task(rq, prev, cookie); + wallclock = walt_ktime_clock(); + walt_update_task_ravg(prev, rq, PUT_PREV_TASK, wallclock, 0); + walt_update_task_ravg(next, rq, PICK_NEXT_TASK, wallclock, 0); clear_tsk_need_resched(prev); clear_preempt_need_resched(); rq->clock_skip_update = 0; @@ -5707,7 +5873,7 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, printk(KERN_CONT " %*pbl", cpumask_pr_args(sched_group_cpus(group))); if (group->sgc->capacity != SCHED_CAPACITY_SCALE) { - printk(KERN_CONT " (cpu_capacity = %d)", + printk(KERN_CONT " (cpu_capacity = %lu)", group->sgc->capacity); } @@ -5771,7 +5937,8 @@ static int sd_degenerate(struct sched_domain *sd) SD_SHARE_CPUCAPACITY | SD_ASYM_CPUCAPACITY | SD_SHARE_PKG_RESOURCES | - SD_SHARE_POWERDOMAIN)) { + SD_SHARE_POWERDOMAIN | + SD_SHARE_CAP_STATES)) { if (sd->groups != sd->groups->next) return 0; } @@ -5804,7 +5971,8 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent) SD_SHARE_CPUCAPACITY | SD_SHARE_PKG_RESOURCES | SD_PREFER_SIBLING | - SD_SHARE_POWERDOMAIN); + SD_SHARE_POWERDOMAIN | + SD_SHARE_CAP_STATES); if (nr_node_ids == 1) pflags &= ~SD_SERIALIZE; } @@ -5883,6 +6051,8 @@ static int init_rootdomain(struct root_domain *rd) if (cpupri_init(&rd->cpupri) != 0) goto free_rto_mask; + + init_max_cpu_capacity(&rd->max_cpu_capacity); return 0; free_rto_mask: @@ -5994,11 +6164,14 @@ DEFINE_PER_CPU(int, sd_llc_id); DEFINE_PER_CPU(struct sched_domain_shared *, sd_llc_shared); DEFINE_PER_CPU(struct sched_domain *, sd_numa); DEFINE_PER_CPU(struct sched_domain *, sd_asym); +DEFINE_PER_CPU(struct sched_domain *, sd_ea); +DEFINE_PER_CPU(struct sched_domain *, sd_scs); static void update_top_cache_domain(int cpu) { struct sched_domain_shared *sds = NULL; struct sched_domain *sd; + struct sched_domain *ea_sd = NULL; int id = cpu; int size = 1; @@ -6019,6 +6192,17 @@ static void update_top_cache_domain(int cpu) sd = highest_flag_domain(cpu, SD_ASYM_PACKING); rcu_assign_pointer(per_cpu(sd_asym, cpu), sd); + + for_each_domain(cpu, sd) { + if (sd->groups->sge) + ea_sd = sd; + else + break; + } + rcu_assign_pointer(per_cpu(sd_ea, cpu), ea_sd); + + sd = highest_flag_domain(cpu, SD_SHARE_CAP_STATES); + rcu_assign_pointer(per_cpu(sd_scs, cpu), sd); } /* @@ -6184,6 +6368,7 @@ build_overlap_sched_groups(struct sched_domain *sd, int cpu) * die on a /0 trap. */ sg->sgc->capacity = SCHED_CAPACITY_SCALE * cpumask_weight(sg_span); + sg->sgc->max_capacity = SCHED_CAPACITY_SCALE; /* * Make sure the first group of this domain contains the @@ -6311,6 +6496,66 @@ static void init_sched_groups_capacity(int cpu, struct sched_domain *sd) update_group_capacity(sd, cpu); } +/* + * Check that the per-cpu provided sd energy data is consistent for all cpus + * within the mask. + */ +static inline void check_sched_energy_data(int cpu, sched_domain_energy_f fn, + const struct cpumask *cpumask) +{ + const struct sched_group_energy * const sge = fn(cpu); + struct cpumask mask; + int i; + + if (cpumask_weight(cpumask) <= 1) + return; + + cpumask_xor(&mask, cpumask, get_cpu_mask(cpu)); + + for_each_cpu(i, &mask) { + const struct sched_group_energy * const e = fn(i); + int y; + + BUG_ON(e->nr_idle_states != sge->nr_idle_states); + + for (y = 0; y < (e->nr_idle_states); y++) { + BUG_ON(e->idle_states[y].power != + sge->idle_states[y].power); + } + + BUG_ON(e->nr_cap_states != sge->nr_cap_states); + + for (y = 0; y < (e->nr_cap_states); y++) { + BUG_ON(e->cap_states[y].cap != sge->cap_states[y].cap); + BUG_ON(e->cap_states[y].power != + sge->cap_states[y].power); + } + } +} + +static void init_sched_energy(int cpu, struct sched_domain *sd, + sched_domain_energy_f fn) +{ + if (!(fn && fn(cpu))) + return; + + if (cpu != group_balance_cpu(sd->groups)) + return; + + if (sd->child && !sd->child->groups->sge) { + pr_err("BUG: EAS setup broken for CPU%d\n", cpu); +#ifdef CONFIG_SCHED_DEBUG + pr_err(" energy data on %s but not on %s domain\n", + sd->name, sd->child->name); +#endif + return; + } + + check_sched_energy_data(cpu, fn, sched_group_cpus(sd->groups)); + + sd->groups->sge = fn(cpu); +} + /* * Initializers for schedule domains * Non-inlined to reduce accumulated stack pressure in build_sched_domains() @@ -6427,6 +6672,7 @@ static int sched_domains_curr_level; * SD_NUMA - describes NUMA topologies * SD_SHARE_POWERDOMAIN - describes shared power domain * SD_ASYM_CPUCAPACITY - describes mixed capacity topologies + * SD_SHARE_CAP_STATES - describes shared capacity states * * Odd one out, which beside describing the topology has a quirk also * prescribes the desired behaviour that goes along with it: @@ -6439,7 +6685,8 @@ static int sched_domains_curr_level; SD_NUMA | \ SD_ASYM_PACKING | \ SD_ASYM_CPUCAPACITY | \ - SD_SHARE_POWERDOMAIN) + SD_SHARE_POWERDOMAIN | \ + SD_SHARE_CAP_STATES) static struct sched_domain * sd_init(struct sched_domain_topology_level *tl, @@ -7036,10 +7283,13 @@ static int build_sched_domains(const struct cpumask *cpu_map, /* Calculate CPU capacity for physical packages and nodes */ for (i = nr_cpumask_bits-1; i >= 0; i--) { + struct sched_domain_topology_level *tl = sched_domain_topology; + if (!cpumask_test_cpu(i, cpu_map)) continue; - for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) { + for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent, tl++) { + init_sched_energy(i, sd, tl->energy); claim_allocations(i, sd); init_sched_groups_capacity(i, sd); } @@ -7050,20 +7300,10 @@ static int build_sched_domains(const struct cpumask *cpu_map, for_each_cpu(i, cpu_map) { rq = cpu_rq(i); sd = *per_cpu_ptr(d.sd, i); - - /* Use READ_ONCE()/WRITE_ONCE() to avoid load/store tearing: */ - if (rq->cpu_capacity_orig > READ_ONCE(d.rd->max_cpu_capacity)) - WRITE_ONCE(d.rd->max_cpu_capacity, rq->cpu_capacity_orig); - cpu_attach_domain(sd, d.rd, i); } rcu_read_unlock(); - if (rq && sched_debug_enabled) { - pr_info("span: %*pbl (max cpu_capacity = %lu)\n", - cpumask_pr_args(cpu_map), rq->rd->max_cpu_capacity); - } - ret = 0; error: __free_domain_allocs(&d, alloc_state, cpu_map); @@ -7407,6 +7647,9 @@ int sched_cpu_dying(unsigned int cpu) /* Handle pending wakeups and then migrate everything off */ sched_ttwu_pending(); raw_spin_lock_irqsave(&rq->lock, flags); + + walt_migrate_sync_cpu(cpu); + if (rq->rd) { BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); set_rq_offline(rq); @@ -7442,6 +7685,7 @@ void __init sched_init_smp(void) { cpumask_var_t non_isolated_cpus; + walt_init_cpu_efficiency(); alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL); alloc_cpumask_var(&fallback_doms, GFP_KERNEL); @@ -7646,6 +7890,11 @@ void __init sched_init(void) rq->idle_stamp = 0; rq->avg_idle = 2*sysctl_sched_migration_cost; rq->max_idle_balance_cost = sysctl_sched_migration_cost; +#ifdef CONFIG_SCHED_WALT + rq->cur_irqload = 0; + rq->avg_irqload = 0; + rq->irqload_ts = 0; +#endif INIT_LIST_HEAD(&rq->cfs_tasks); @@ -7703,6 +7952,14 @@ static inline int preempt_count_equals(int preempt_offset) return (nested == preempt_offset); } +static int __might_sleep_init_called; +int __init __might_sleep_init(void) +{ + __might_sleep_init_called = 1; + return 0; +} +early_initcall(__might_sleep_init); + void __might_sleep(const char *file, int line, int preempt_offset) { /* @@ -7728,8 +7985,10 @@ void ___might_sleep(const char *file, int line, int preempt_offset) rcu_sleep_check(); /* WARN_ON_ONCE() by default, no rate limit reqd. */ if ((preempt_count_equals(preempt_offset) && !irqs_disabled() && - !is_idle_task(current)) || - system_state != SYSTEM_RUNNING || oops_in_progress) + !is_idle_task(current)) || oops_in_progress) + return; + if (system_state != SYSTEM_RUNNING && + (!__might_sleep_init_called || system_state != SYSTEM_BOOTING)) return; if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) return; diff --git a/kernel/sched/cpufreq_sched.c b/kernel/sched/cpufreq_sched.c new file mode 100644 index 00000000000000..1d471d5334149e --- /dev/null +++ b/kernel/sched/cpufreq_sched.c @@ -0,0 +1,485 @@ +/* + * Copyright (C) 2015 Michael Turquette + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include +#include +#include +#include +#include + +#define CREATE_TRACE_POINTS +#include + +#include "sched.h" + +#define THROTTLE_DOWN_NSEC 50000000 /* 50ms default */ +#define THROTTLE_UP_NSEC 500000 /* 500us default */ + +struct static_key __read_mostly __sched_freq = STATIC_KEY_INIT_FALSE; +static bool __read_mostly cpufreq_driver_slow; + +#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_SCHED +static struct cpufreq_governor cpufreq_gov_sched; +#endif + +static DEFINE_PER_CPU(unsigned long, enabled); +DEFINE_PER_CPU(struct sched_capacity_reqs, cpu_sched_capacity_reqs); + +/** + * gov_data - per-policy data internal to the governor + * @up_throttle: next throttling period expiry if increasing OPP + * @down_throttle: next throttling period expiry if decreasing OPP + * @up_throttle_nsec: throttle period length in nanoseconds if increasing OPP + * @down_throttle_nsec: throttle period length in nanoseconds if decreasing OPP + * @task: worker thread for dvfs transition that may block/sleep + * @irq_work: callback used to wake up worker thread + * @requested_freq: last frequency requested by the sched governor + * + * struct gov_data is the per-policy cpufreq_sched-specific data structure. A + * per-policy instance of it is created when the cpufreq_sched governor receives + * the CPUFREQ_GOV_START condition and a pointer to it exists in the gov_data + * member of struct cpufreq_policy. + * + * Readers of this data must call down_read(policy->rwsem). Writers must + * call down_write(policy->rwsem). + */ +struct gov_data { + ktime_t up_throttle; + ktime_t down_throttle; + unsigned int up_throttle_nsec; + unsigned int down_throttle_nsec; + struct task_struct *task; + struct irq_work irq_work; + unsigned int requested_freq; +}; + +static void cpufreq_sched_try_driver_target(struct cpufreq_policy *policy, + unsigned int freq) +{ + struct gov_data *gd = policy->governor_data; + + /* avoid race with cpufreq_sched_stop */ + if (!down_write_trylock(&policy->rwsem)) + return; + + __cpufreq_driver_target(policy, freq, CPUFREQ_RELATION_L); + + gd->up_throttle = ktime_add_ns(ktime_get(), gd->up_throttle_nsec); + gd->down_throttle = ktime_add_ns(ktime_get(), gd->down_throttle_nsec); + up_write(&policy->rwsem); +} + +static bool finish_last_request(struct gov_data *gd, unsigned int cur_freq) +{ + ktime_t now = ktime_get(); + + ktime_t throttle = gd->requested_freq < cur_freq ? + gd->down_throttle : gd->up_throttle; + + if (ktime_after(now, throttle)) + return false; + + while (1) { + int usec_left = ktime_to_ns(ktime_sub(throttle, now)); + + usec_left /= NSEC_PER_USEC; + trace_cpufreq_sched_throttled(usec_left); + usleep_range(usec_left, usec_left + 100); + now = ktime_get(); + if (ktime_after(now, throttle)) + return true; + } +} + +/* + * we pass in struct cpufreq_policy. This is safe because changing out the + * policy requires a call to __cpufreq_governor(policy, CPUFREQ_GOV_STOP), + * which tears down all of the data structures and __cpufreq_governor(policy, + * CPUFREQ_GOV_START) will do a full rebuild, including this kthread with the + * new policy pointer + */ +static int cpufreq_sched_thread(void *data) +{ + struct sched_param param; + struct cpufreq_policy *policy; + struct gov_data *gd; + unsigned int new_request = 0; + unsigned int last_request = 0; + int ret; + + policy = (struct cpufreq_policy *) data; + gd = policy->governor_data; + + param.sched_priority = 50; + ret = sched_setscheduler_nocheck(gd->task, SCHED_FIFO, ¶m); + if (ret) { + pr_warn("%s: failed to set SCHED_FIFO\n", __func__); + do_exit(-EINVAL); + } else { + pr_debug("%s: kthread (%d) set to SCHED_FIFO\n", + __func__, gd->task->pid); + } + + do { + new_request = gd->requested_freq; + if (new_request == last_request) { + set_current_state(TASK_INTERRUPTIBLE); + if (kthread_should_stop()) + break; + schedule(); + } else { + /* + * if the frequency thread sleeps while waiting to be + * unthrottled, start over to check for a newer request + */ + if (finish_last_request(gd, policy->cur)) + continue; + last_request = new_request; + cpufreq_sched_try_driver_target(policy, new_request); + } + } while (!kthread_should_stop()); + + return 0; +} + +static void cpufreq_sched_irq_work(struct irq_work *irq_work) +{ + struct gov_data *gd; + + gd = container_of(irq_work, struct gov_data, irq_work); + if (!gd) + return; + + wake_up_process(gd->task); +} + +static void update_fdomain_capacity_request(int cpu) +{ + unsigned int freq_new, index_new, cpu_tmp; + struct cpufreq_policy *policy; + struct gov_data *gd; + unsigned long capacity = 0; + + /* + * Avoid grabbing the policy if possible. A test is still + * required after locking the CPU's policy to avoid racing + * with the governor changing. + */ + if (!per_cpu(enabled, cpu)) + return; + + policy = cpufreq_cpu_get(cpu); + if (IS_ERR_OR_NULL(policy)) + return; + + if (policy->governor != &cpufreq_gov_sched || + !policy->governor_data) + goto out; + + gd = policy->governor_data; + + /* find max capacity requested by cpus in this policy */ + for_each_cpu(cpu_tmp, policy->cpus) { + struct sched_capacity_reqs *scr; + + scr = &per_cpu(cpu_sched_capacity_reqs, cpu_tmp); + capacity = max(capacity, scr->total); + } + + /* Convert the new maximum capacity request into a cpu frequency */ + freq_new = capacity * policy->max >> SCHED_CAPACITY_SHIFT; + index_new = cpufreq_frequency_table_target(policy, freq_new, CPUFREQ_RELATION_L); + freq_new = policy->freq_table[index_new].frequency; + + if (freq_new > policy->max) + freq_new = policy->max; + + if (freq_new < policy->min) + freq_new = policy->min; + + trace_cpufreq_sched_request_opp(cpu, capacity, freq_new, + gd->requested_freq); + if (freq_new == gd->requested_freq) + goto out; + + gd->requested_freq = freq_new; + + /* + * Throttling is not yet supported on platforms with fast cpufreq + * drivers. + */ + if (cpufreq_driver_slow) + irq_work_queue_on(&gd->irq_work, cpu); + else + cpufreq_sched_try_driver_target(policy, freq_new); + +out: + cpufreq_cpu_put(policy); +} + +void update_cpu_capacity_request(int cpu, bool request) +{ + unsigned long new_capacity; + struct sched_capacity_reqs *scr; + + /* The rq lock serializes access to the CPU's sched_capacity_reqs. */ + lockdep_assert_held(&cpu_rq(cpu)->lock); + + scr = &per_cpu(cpu_sched_capacity_reqs, cpu); + + new_capacity = scr->cfs + scr->rt; + new_capacity = new_capacity * capacity_margin + / SCHED_CAPACITY_SCALE; + new_capacity += scr->dl; + + if (new_capacity == scr->total) + return; + + trace_cpufreq_sched_update_capacity(cpu, request, scr, new_capacity); + + scr->total = new_capacity; + if (request) + update_fdomain_capacity_request(cpu); +} + +static inline void set_sched_freq(void) +{ + static_key_slow_inc(&__sched_freq); +} + +static inline void clear_sched_freq(void) +{ + static_key_slow_dec(&__sched_freq); +} + +static struct attribute_group sched_attr_group_gov_pol; +static struct attribute_group *get_sysfs_attr(void) +{ + return &sched_attr_group_gov_pol; +} + +static int cpufreq_sched_policy_init(struct cpufreq_policy *policy) +{ + struct gov_data *gd; + int cpu; + int rc; + + for_each_cpu(cpu, policy->cpus) + memset(&per_cpu(cpu_sched_capacity_reqs, cpu), 0, + sizeof(struct sched_capacity_reqs)); + + gd = kzalloc(sizeof(*gd), GFP_KERNEL); + if (!gd) + return -ENOMEM; + + gd->up_throttle_nsec = policy->cpuinfo.transition_latency ? + policy->cpuinfo.transition_latency : + THROTTLE_UP_NSEC; + gd->down_throttle_nsec = THROTTLE_DOWN_NSEC; + pr_debug("%s: throttle threshold = %u [ns]\n", + __func__, gd->up_throttle_nsec); + + rc = sysfs_create_group(&policy->kobj, get_sysfs_attr()); + if (rc) { + pr_err("%s: couldn't create sysfs attributes: %d\n", __func__, rc); + goto err; + } + + policy->governor_data = gd; + if (cpufreq_driver_is_slow()) { + cpufreq_driver_slow = true; + gd->task = kthread_create(cpufreq_sched_thread, policy, + "kschedfreq:%d", + cpumask_first(policy->related_cpus)); + if (IS_ERR_OR_NULL(gd->task)) { + pr_err("%s: failed to create kschedfreq thread\n", + __func__); + goto err; + } + get_task_struct(gd->task); + kthread_bind_mask(gd->task, policy->related_cpus); + wake_up_process(gd->task); + init_irq_work(&gd->irq_work, cpufreq_sched_irq_work); + } + + set_sched_freq(); + + return 0; + +err: + policy->governor_data = NULL; + kfree(gd); + return -ENOMEM; +} + +static void cpufreq_sched_policy_exit(struct cpufreq_policy *policy) +{ + struct gov_data *gd = policy->governor_data; + + clear_sched_freq(); + if (cpufreq_driver_slow) { + kthread_stop(gd->task); + put_task_struct(gd->task); + } + + sysfs_remove_group(&policy->kobj, get_sysfs_attr()); + + policy->governor_data = NULL; + + kfree(gd); +} + +static int cpufreq_sched_start(struct cpufreq_policy *policy) +{ + int cpu; + + for_each_cpu(cpu, policy->cpus) + per_cpu(enabled, cpu) = 1; + + return 0; +} + +static void cpufreq_sched_limits(struct cpufreq_policy *policy) +{ + unsigned int clamp_freq; + struct gov_data *gd = policy->governor_data;; + + pr_debug("limit event for cpu %u: %u - %u kHz, currently %u kHz\n", + policy->cpu, policy->min, policy->max, + policy->cur); + + clamp_freq = clamp(gd->requested_freq, policy->min, policy->max); + + if (policy->cur != clamp_freq) + __cpufreq_driver_target(policy, clamp_freq, CPUFREQ_RELATION_L); +} + +static void cpufreq_sched_stop(struct cpufreq_policy *policy) +{ + int cpu; + + for_each_cpu(cpu, policy->cpus) + per_cpu(enabled, cpu) = 0; +} + +/* Tunables */ +static ssize_t show_up_throttle_nsec(struct gov_data *gd, char *buf) +{ + return sprintf(buf, "%u\n", gd->up_throttle_nsec); +} + +static ssize_t store_up_throttle_nsec(struct gov_data *gd, + const char *buf, size_t count) +{ + int ret; + long unsigned int val; + + ret = kstrtoul(buf, 0, &val); + if (ret < 0) + return ret; + gd->up_throttle_nsec = val; + return count; +} + +static ssize_t show_down_throttle_nsec(struct gov_data *gd, char *buf) +{ + return sprintf(buf, "%u\n", gd->down_throttle_nsec); +} + +static ssize_t store_down_throttle_nsec(struct gov_data *gd, + const char *buf, size_t count) +{ + int ret; + long unsigned int val; + + ret = kstrtoul(buf, 0, &val); + if (ret < 0) + return ret; + gd->down_throttle_nsec = val; + return count; +} + +/* + * Create show/store routines + * - sys: One governor instance for complete SYSTEM + * - pol: One governor instance per struct cpufreq_policy + */ +#define show_gov_pol_sys(file_name) \ +static ssize_t show_##file_name##_gov_pol \ +(struct cpufreq_policy *policy, char *buf) \ +{ \ + return show_##file_name(policy->governor_data, buf); \ +} + +#define store_gov_pol_sys(file_name) \ +static ssize_t store_##file_name##_gov_pol \ +(struct cpufreq_policy *policy, const char *buf, size_t count) \ +{ \ + return store_##file_name(policy->governor_data, buf, count); \ +} + +#define gov_pol_attr_rw(_name) \ + static struct freq_attr _name##_gov_pol = \ + __ATTR(_name, 0644, show_##_name##_gov_pol, store_##_name##_gov_pol) + +#define show_store_gov_pol_sys(file_name) \ + show_gov_pol_sys(file_name); \ + store_gov_pol_sys(file_name) +#define tunable_handlers(file_name) \ + show_gov_pol_sys(file_name); \ + store_gov_pol_sys(file_name); \ + gov_pol_attr_rw(file_name) + +tunable_handlers(down_throttle_nsec); +tunable_handlers(up_throttle_nsec); + +/* Per policy governor instance */ +static struct attribute *sched_attributes_gov_pol[] = { + &up_throttle_nsec_gov_pol.attr, + &down_throttle_nsec_gov_pol.attr, + NULL, +}; + +static struct attribute_group sched_attr_group_gov_pol = { + .attrs = sched_attributes_gov_pol, + .name = "sched", +}; + +#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_SCHED +static +#endif +struct cpufreq_governor cpufreq_gov_sched = { + .name = "sched", + .init = cpufreq_sched_policy_init, + .exit = cpufreq_sched_policy_exit, + .start = cpufreq_sched_start, + .stop = cpufreq_sched_stop, + .limits = cpufreq_sched_limits, + .owner = THIS_MODULE, +}; + +static int __init cpufreq_sched_init(void) +{ + int cpu; + + for_each_cpu(cpu, cpu_possible_mask) + per_cpu(enabled, cpu) = 0; + return cpufreq_register_governor(&cpufreq_gov_sched); +} + +#ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_SCHED +struct cpufreq_governor *cpufreq_default_governor(void) +{ + return &cpufreq_gov_sched; +} +#endif + +/* Try to make this the default governor */ +fs_initcall(cpufreq_sched_init); diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c deleted file mode 100644 index 69e06898997d39..00000000000000 --- a/kernel/sched/cpufreq_schedutil.c +++ /dev/null @@ -1,576 +0,0 @@ -/* - * CPUFreq governor based on scheduler-provided CPU utilization data. - * - * Copyright (C) 2016, Intel Corporation - * Author: Rafael J. Wysocki - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include -#include -#include - -#include "sched.h" - -struct sugov_tunables { - struct gov_attr_set attr_set; - unsigned int rate_limit_us; -}; - -struct sugov_policy { - struct cpufreq_policy *policy; - - struct sugov_tunables *tunables; - struct list_head tunables_hook; - - raw_spinlock_t update_lock; /* For shared policies */ - u64 last_freq_update_time; - s64 freq_update_delay_ns; - unsigned int next_freq; - - /* The next fields are only needed if fast switch cannot be used. */ - struct irq_work irq_work; - struct work_struct work; - struct mutex work_lock; - bool work_in_progress; - - bool need_freq_update; -}; - -struct sugov_cpu { - struct update_util_data update_util; - struct sugov_policy *sg_policy; - - unsigned int cached_raw_freq; - unsigned long iowait_boost; - unsigned long iowait_boost_max; - u64 last_update; - - /* The fields below are only needed when sharing a policy. */ - unsigned long util; - unsigned long max; - unsigned int flags; -}; - -static DEFINE_PER_CPU(struct sugov_cpu, sugov_cpu); - -/************************ Governor internals ***********************/ - -static bool sugov_should_update_freq(struct sugov_policy *sg_policy, u64 time) -{ - s64 delta_ns; - - if (sg_policy->work_in_progress) - return false; - - if (unlikely(sg_policy->need_freq_update)) { - sg_policy->need_freq_update = false; - /* - * This happens when limits change, so forget the previous - * next_freq value and force an update. - */ - sg_policy->next_freq = UINT_MAX; - return true; - } - - delta_ns = time - sg_policy->last_freq_update_time; - return delta_ns >= sg_policy->freq_update_delay_ns; -} - -static void sugov_update_commit(struct sugov_policy *sg_policy, u64 time, - unsigned int next_freq) -{ - struct cpufreq_policy *policy = sg_policy->policy; - - sg_policy->last_freq_update_time = time; - - if (policy->fast_switch_enabled) { - if (sg_policy->next_freq == next_freq) { - trace_cpu_frequency(policy->cur, smp_processor_id()); - return; - } - sg_policy->next_freq = next_freq; - next_freq = cpufreq_driver_fast_switch(policy, next_freq); - if (next_freq == CPUFREQ_ENTRY_INVALID) - return; - - policy->cur = next_freq; - trace_cpu_frequency(next_freq, smp_processor_id()); - } else if (sg_policy->next_freq != next_freq) { - sg_policy->next_freq = next_freq; - sg_policy->work_in_progress = true; - irq_work_queue(&sg_policy->irq_work); - } -} - -/** - * get_next_freq - Compute a new frequency for a given cpufreq policy. - * @sg_cpu: schedutil cpu object to compute the new frequency for. - * @util: Current CPU utilization. - * @max: CPU capacity. - * - * If the utilization is frequency-invariant, choose the new frequency to be - * proportional to it, that is - * - * next_freq = C * max_freq * util / max - * - * Otherwise, approximate the would-be frequency-invariant utilization by - * util_raw * (curr_freq / max_freq) which leads to - * - * next_freq = C * curr_freq * util_raw / max - * - * Take C = 1.25 for the frequency tipping point at (util / max) = 0.8. - * - * The lowest driver-supported frequency which is equal or greater than the raw - * next_freq (as calculated above) is returned, subject to policy min/max and - * cpufreq driver limitations. - */ -static unsigned int get_next_freq(struct sugov_cpu *sg_cpu, unsigned long util, - unsigned long max) -{ - struct sugov_policy *sg_policy = sg_cpu->sg_policy; - struct cpufreq_policy *policy = sg_policy->policy; - unsigned int freq = arch_scale_freq_invariant() ? - policy->cpuinfo.max_freq : policy->cur; - - freq = (freq + (freq >> 2)) * util / max; - - if (freq == sg_cpu->cached_raw_freq && sg_policy->next_freq != UINT_MAX) - return sg_policy->next_freq; - sg_cpu->cached_raw_freq = freq; - return cpufreq_driver_resolve_freq(policy, freq); -} - -static void sugov_get_util(unsigned long *util, unsigned long *max) -{ - struct rq *rq = this_rq(); - unsigned long cfs_max; - - cfs_max = arch_scale_cpu_capacity(NULL, smp_processor_id()); - - *util = min(rq->cfs.avg.util_avg, cfs_max); - *max = cfs_max; -} - -static void sugov_set_iowait_boost(struct sugov_cpu *sg_cpu, u64 time, - unsigned int flags) -{ - if (flags & SCHED_CPUFREQ_IOWAIT) { - sg_cpu->iowait_boost = sg_cpu->iowait_boost_max; - } else if (sg_cpu->iowait_boost) { - s64 delta_ns = time - sg_cpu->last_update; - - /* Clear iowait_boost if the CPU apprears to have been idle. */ - if (delta_ns > TICK_NSEC) - sg_cpu->iowait_boost = 0; - } -} - -static void sugov_iowait_boost(struct sugov_cpu *sg_cpu, unsigned long *util, - unsigned long *max) -{ - unsigned long boost_util = sg_cpu->iowait_boost; - unsigned long boost_max = sg_cpu->iowait_boost_max; - - if (!boost_util) - return; - - if (*util * boost_max < *max * boost_util) { - *util = boost_util; - *max = boost_max; - } - sg_cpu->iowait_boost >>= 1; -} - -static void sugov_update_single(struct update_util_data *hook, u64 time, - unsigned int flags) -{ - struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util); - struct sugov_policy *sg_policy = sg_cpu->sg_policy; - struct cpufreq_policy *policy = sg_policy->policy; - unsigned long util, max; - unsigned int next_f; - - sugov_set_iowait_boost(sg_cpu, time, flags); - sg_cpu->last_update = time; - - if (!sugov_should_update_freq(sg_policy, time)) - return; - - if (flags & SCHED_CPUFREQ_RT_DL) { - next_f = policy->cpuinfo.max_freq; - } else { - sugov_get_util(&util, &max); - sugov_iowait_boost(sg_cpu, &util, &max); - next_f = get_next_freq(sg_cpu, util, max); - } - sugov_update_commit(sg_policy, time, next_f); -} - -static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu, - unsigned long util, unsigned long max, - unsigned int flags) -{ - struct sugov_policy *sg_policy = sg_cpu->sg_policy; - struct cpufreq_policy *policy = sg_policy->policy; - unsigned int max_f = policy->cpuinfo.max_freq; - u64 last_freq_update_time = sg_policy->last_freq_update_time; - unsigned int j; - - if (flags & SCHED_CPUFREQ_RT_DL) - return max_f; - - sugov_iowait_boost(sg_cpu, &util, &max); - - for_each_cpu(j, policy->cpus) { - struct sugov_cpu *j_sg_cpu; - unsigned long j_util, j_max; - s64 delta_ns; - - if (j == smp_processor_id()) - continue; - - j_sg_cpu = &per_cpu(sugov_cpu, j); - /* - * If the CPU utilization was last updated before the previous - * frequency update and the time elapsed between the last update - * of the CPU utilization and the last frequency update is long - * enough, don't take the CPU into account as it probably is - * idle now (and clear iowait_boost for it). - */ - delta_ns = last_freq_update_time - j_sg_cpu->last_update; - if (delta_ns > TICK_NSEC) { - j_sg_cpu->iowait_boost = 0; - continue; - } - if (j_sg_cpu->flags & SCHED_CPUFREQ_RT_DL) - return max_f; - - j_util = j_sg_cpu->util; - j_max = j_sg_cpu->max; - if (j_util * max > j_max * util) { - util = j_util; - max = j_max; - } - - sugov_iowait_boost(j_sg_cpu, &util, &max); - } - - return get_next_freq(sg_cpu, util, max); -} - -static void sugov_update_shared(struct update_util_data *hook, u64 time, - unsigned int flags) -{ - struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util); - struct sugov_policy *sg_policy = sg_cpu->sg_policy; - unsigned long util, max; - unsigned int next_f; - - sugov_get_util(&util, &max); - - raw_spin_lock(&sg_policy->update_lock); - - sg_cpu->util = util; - sg_cpu->max = max; - sg_cpu->flags = flags; - - sugov_set_iowait_boost(sg_cpu, time, flags); - sg_cpu->last_update = time; - - if (sugov_should_update_freq(sg_policy, time)) { - next_f = sugov_next_freq_shared(sg_cpu, util, max, flags); - sugov_update_commit(sg_policy, time, next_f); - } - - raw_spin_unlock(&sg_policy->update_lock); -} - -static void sugov_work(struct work_struct *work) -{ - struct sugov_policy *sg_policy = container_of(work, struct sugov_policy, work); - - mutex_lock(&sg_policy->work_lock); - __cpufreq_driver_target(sg_policy->policy, sg_policy->next_freq, - CPUFREQ_RELATION_L); - mutex_unlock(&sg_policy->work_lock); - - sg_policy->work_in_progress = false; -} - -static void sugov_irq_work(struct irq_work *irq_work) -{ - struct sugov_policy *sg_policy; - - sg_policy = container_of(irq_work, struct sugov_policy, irq_work); - schedule_work_on(smp_processor_id(), &sg_policy->work); -} - -/************************** sysfs interface ************************/ - -static struct sugov_tunables *global_tunables; -static DEFINE_MUTEX(global_tunables_lock); - -static inline struct sugov_tunables *to_sugov_tunables(struct gov_attr_set *attr_set) -{ - return container_of(attr_set, struct sugov_tunables, attr_set); -} - -static ssize_t rate_limit_us_show(struct gov_attr_set *attr_set, char *buf) -{ - struct sugov_tunables *tunables = to_sugov_tunables(attr_set); - - return sprintf(buf, "%u\n", tunables->rate_limit_us); -} - -static ssize_t rate_limit_us_store(struct gov_attr_set *attr_set, const char *buf, - size_t count) -{ - struct sugov_tunables *tunables = to_sugov_tunables(attr_set); - struct sugov_policy *sg_policy; - unsigned int rate_limit_us; - - if (kstrtouint(buf, 10, &rate_limit_us)) - return -EINVAL; - - tunables->rate_limit_us = rate_limit_us; - - list_for_each_entry(sg_policy, &attr_set->policy_list, tunables_hook) - sg_policy->freq_update_delay_ns = rate_limit_us * NSEC_PER_USEC; - - return count; -} - -static struct governor_attr rate_limit_us = __ATTR_RW(rate_limit_us); - -static struct attribute *sugov_attributes[] = { - &rate_limit_us.attr, - NULL -}; - -static struct kobj_type sugov_tunables_ktype = { - .default_attrs = sugov_attributes, - .sysfs_ops = &governor_sysfs_ops, -}; - -/********************** cpufreq governor interface *********************/ - -static struct cpufreq_governor schedutil_gov; - -static struct sugov_policy *sugov_policy_alloc(struct cpufreq_policy *policy) -{ - struct sugov_policy *sg_policy; - - sg_policy = kzalloc(sizeof(*sg_policy), GFP_KERNEL); - if (!sg_policy) - return NULL; - - sg_policy->policy = policy; - init_irq_work(&sg_policy->irq_work, sugov_irq_work); - INIT_WORK(&sg_policy->work, sugov_work); - mutex_init(&sg_policy->work_lock); - raw_spin_lock_init(&sg_policy->update_lock); - return sg_policy; -} - -static void sugov_policy_free(struct sugov_policy *sg_policy) -{ - mutex_destroy(&sg_policy->work_lock); - kfree(sg_policy); -} - -static struct sugov_tunables *sugov_tunables_alloc(struct sugov_policy *sg_policy) -{ - struct sugov_tunables *tunables; - - tunables = kzalloc(sizeof(*tunables), GFP_KERNEL); - if (tunables) { - gov_attr_set_init(&tunables->attr_set, &sg_policy->tunables_hook); - if (!have_governor_per_policy()) - global_tunables = tunables; - } - return tunables; -} - -static void sugov_tunables_free(struct sugov_tunables *tunables) -{ - if (!have_governor_per_policy()) - global_tunables = NULL; - - kfree(tunables); -} - -static int sugov_init(struct cpufreq_policy *policy) -{ - struct sugov_policy *sg_policy; - struct sugov_tunables *tunables; - unsigned int lat; - int ret = 0; - - /* State should be equivalent to EXIT */ - if (policy->governor_data) - return -EBUSY; - - sg_policy = sugov_policy_alloc(policy); - if (!sg_policy) - return -ENOMEM; - - mutex_lock(&global_tunables_lock); - - if (global_tunables) { - if (WARN_ON(have_governor_per_policy())) { - ret = -EINVAL; - goto free_sg_policy; - } - policy->governor_data = sg_policy; - sg_policy->tunables = global_tunables; - - gov_attr_set_get(&global_tunables->attr_set, &sg_policy->tunables_hook); - goto out; - } - - tunables = sugov_tunables_alloc(sg_policy); - if (!tunables) { - ret = -ENOMEM; - goto free_sg_policy; - } - - tunables->rate_limit_us = LATENCY_MULTIPLIER; - lat = policy->cpuinfo.transition_latency / NSEC_PER_USEC; - if (lat) - tunables->rate_limit_us *= lat; - - policy->governor_data = sg_policy; - sg_policy->tunables = tunables; - - ret = kobject_init_and_add(&tunables->attr_set.kobj, &sugov_tunables_ktype, - get_governor_parent_kobj(policy), "%s", - schedutil_gov.name); - if (ret) - goto fail; - - out: - mutex_unlock(&global_tunables_lock); - - cpufreq_enable_fast_switch(policy); - return 0; - - fail: - policy->governor_data = NULL; - sugov_tunables_free(tunables); - - free_sg_policy: - mutex_unlock(&global_tunables_lock); - - sugov_policy_free(sg_policy); - pr_err("initialization failed (error %d)\n", ret); - return ret; -} - -static void sugov_exit(struct cpufreq_policy *policy) -{ - struct sugov_policy *sg_policy = policy->governor_data; - struct sugov_tunables *tunables = sg_policy->tunables; - unsigned int count; - - cpufreq_disable_fast_switch(policy); - - mutex_lock(&global_tunables_lock); - - count = gov_attr_set_put(&tunables->attr_set, &sg_policy->tunables_hook); - policy->governor_data = NULL; - if (!count) - sugov_tunables_free(tunables); - - mutex_unlock(&global_tunables_lock); - - sugov_policy_free(sg_policy); -} - -static int sugov_start(struct cpufreq_policy *policy) -{ - struct sugov_policy *sg_policy = policy->governor_data; - unsigned int cpu; - - sg_policy->freq_update_delay_ns = sg_policy->tunables->rate_limit_us * NSEC_PER_USEC; - sg_policy->last_freq_update_time = 0; - sg_policy->next_freq = UINT_MAX; - sg_policy->work_in_progress = false; - sg_policy->need_freq_update = false; - - for_each_cpu(cpu, policy->cpus) { - struct sugov_cpu *sg_cpu = &per_cpu(sugov_cpu, cpu); - - sg_cpu->sg_policy = sg_policy; - if (policy_is_shared(policy)) { - sg_cpu->util = 0; - sg_cpu->max = 0; - sg_cpu->flags = SCHED_CPUFREQ_RT; - sg_cpu->last_update = 0; - sg_cpu->cached_raw_freq = 0; - sg_cpu->iowait_boost = 0; - sg_cpu->iowait_boost_max = policy->cpuinfo.max_freq; - cpufreq_add_update_util_hook(cpu, &sg_cpu->update_util, - sugov_update_shared); - } else { - cpufreq_add_update_util_hook(cpu, &sg_cpu->update_util, - sugov_update_single); - } - } - return 0; -} - -static void sugov_stop(struct cpufreq_policy *policy) -{ - struct sugov_policy *sg_policy = policy->governor_data; - unsigned int cpu; - - for_each_cpu(cpu, policy->cpus) - cpufreq_remove_update_util_hook(cpu); - - synchronize_sched(); - - irq_work_sync(&sg_policy->irq_work); - cancel_work_sync(&sg_policy->work); -} - -static void sugov_limits(struct cpufreq_policy *policy) -{ - struct sugov_policy *sg_policy = policy->governor_data; - - if (!policy->fast_switch_enabled) { - mutex_lock(&sg_policy->work_lock); - cpufreq_policy_apply_limits(policy); - mutex_unlock(&sg_policy->work_lock); - } - - sg_policy->need_freq_update = true; -} - -static struct cpufreq_governor schedutil_gov = { - .name = "schedutil", - .owner = THIS_MODULE, - .init = sugov_init, - .exit = sugov_exit, - .start = sugov_start, - .stop = sugov_stop, - .limits = sugov_limits, -}; - -#ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_SCHEDUTIL -struct cpufreq_governor *cpufreq_default_governor(void) -{ - return &schedutil_gov; -} -#endif - -static int __init sugov_register(void) -{ - return cpufreq_register_governor(&schedutil_gov); -} -fs_initcall(sugov_register); diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index 5ebee3164e6442..5e9422b66f5f08 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -8,7 +8,7 @@ #ifdef CONFIG_PARAVIRT #include #endif - +#include "walt.h" #ifdef CONFIG_IRQ_TIME_ACCOUNTING @@ -46,11 +46,18 @@ void irqtime_account_irq(struct task_struct *curr) struct irqtime *irqtime = this_cpu_ptr(&cpu_irqtime); s64 delta; int cpu; +#ifdef CONFIG_SCHED_WALT + u64 wallclock; + bool account = true; +#endif if (!sched_clock_irqtime) return; cpu = smp_processor_id(); +#ifdef CONFIG_SCHED_WALT + wallclock = sched_clock_cpu(cpu); +#endif delta = sched_clock_cpu(cpu) - irqtime->irq_start_time; irqtime->irq_start_time += delta; @@ -65,8 +72,16 @@ void irqtime_account_irq(struct task_struct *curr) irqtime->hardirq_time += delta; else if (in_serving_softirq() && curr != this_cpu_ksoftirqd()) irqtime->softirq_time += delta; +#ifdef CONFIG_SCHED_WALT + else + account = false; +#endif u64_stats_update_end(&irqtime->sync); +#ifdef CONFIG_SCHED_WALT + if (account) + walt_account_irqtime(cpu, curr, delta, wallclock); +#endif } EXPORT_SYMBOL_GPL(irqtime_account_irq); diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index fa178b62ea79b5..48a621c257d1c7 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -260,10 +260,61 @@ set_table_entry(struct ctl_table *entry, } } +static struct ctl_table * +sd_alloc_ctl_energy_table(struct sched_group_energy *sge) +{ + struct ctl_table *table = sd_alloc_ctl_entry(5); + + if (table == NULL) + return NULL; + + set_table_entry(&table[0], "nr_idle_states", &sge->nr_idle_states, + sizeof(int), 0644, proc_dointvec_minmax, false); + set_table_entry(&table[1], "idle_states", &sge->idle_states[0].power, + sge->nr_idle_states*sizeof(struct idle_state), 0644, + proc_doulongvec_minmax, false); + set_table_entry(&table[2], "nr_cap_states", &sge->nr_cap_states, + sizeof(int), 0644, proc_dointvec_minmax, false); + set_table_entry(&table[3], "cap_states", &sge->cap_states[0].cap, + sge->nr_cap_states*sizeof(struct capacity_state), 0644, + proc_doulongvec_minmax, false); + + return table; +} + +static struct ctl_table * +sd_alloc_ctl_group_table(struct sched_group *sg) +{ + struct ctl_table *table = sd_alloc_ctl_entry(2); + + if (table == NULL) + return NULL; + + table->procname = kstrdup("energy", GFP_KERNEL); + table->mode = 0555; + table->child = sd_alloc_ctl_energy_table((struct sched_group_energy *)sg->sge); + + return table; +} + static struct ctl_table * sd_alloc_ctl_domain_table(struct sched_domain *sd) { - struct ctl_table *table = sd_alloc_ctl_entry(14); + struct ctl_table *table; + unsigned int nr_entries = 14; + + int i = 0; + struct sched_group *sg = sd->groups; + + if (sg->sge) { + int nr_sgs = 0; + + do {} while (nr_sgs++, sg = sg->next, sg != sd->groups); + + nr_entries += nr_sgs; + } + + table = sd_alloc_ctl_entry(nr_entries); if (table == NULL) return NULL; @@ -296,7 +347,19 @@ sd_alloc_ctl_domain_table(struct sched_domain *sd) sizeof(long), 0644, proc_doulongvec_minmax, false); set_table_entry(&table[12], "name", sd->name, CORENAME_MAX_SIZE, 0444, proc_dostring, false); - /* &table[13] is terminator */ + sg = sd->groups; + if (sg->sge) { + char buf[32]; + struct ctl_table *entry = &table[13]; + + do { + snprintf(buf, 32, "group%d", i); + entry->procname = kstrdup(buf, GFP_KERNEL); + entry->mode = 0555; + entry->child = sd_alloc_ctl_group_table(sg); + } while (entry++, i++, sg = sg->next, sg != sd->groups); + } + /* &table[nr_entries-1] is terminator */ return table; } diff --git a/kernel/sched/energy.c b/kernel/sched/energy.c new file mode 100644 index 00000000000000..b0656b7a93e337 --- /dev/null +++ b/kernel/sched/energy.c @@ -0,0 +1,124 @@ +/* + * Obtain energy cost data from DT and populate relevant scheduler data + * structures. + * + * Copyright (C) 2015 ARM Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#define pr_fmt(fmt) "sched-energy: " fmt + +#define DEBUG + +#include +#include +#include +#include +#include +#include + +struct sched_group_energy *sge_array[NR_CPUS][NR_SD_LEVELS]; + +static void free_resources(void) +{ + int cpu, sd_level; + struct sched_group_energy *sge; + + for_each_possible_cpu(cpu) { + for_each_possible_sd_level(sd_level) { + sge = sge_array[cpu][sd_level]; + if (sge) { + kfree(sge->cap_states); + kfree(sge->idle_states); + kfree(sge); + } + } + } +} + +void init_sched_energy_costs(void) +{ + struct device_node *cn, *cp; + struct capacity_state *cap_states; + struct idle_state *idle_states; + struct sched_group_energy *sge; + const struct property *prop; + int sd_level, i, nstates, cpu; + const __be32 *val; + + for_each_possible_cpu(cpu) { + cn = of_get_cpu_node(cpu, NULL); + if (!cn) { + pr_warn("CPU device node missing for CPU %d\n", cpu); + return; + } + + if (!of_find_property(cn, "sched-energy-costs", NULL)) { + pr_warn("CPU device node has no sched-energy-costs\n"); + return; + } + + for_each_possible_sd_level(sd_level) { + cp = of_parse_phandle(cn, "sched-energy-costs", sd_level); + if (!cp) + break; + + prop = of_find_property(cp, "busy-cost-data", NULL); + if (!prop || !prop->value) { + pr_warn("No busy-cost data, skipping sched_energy init\n"); + goto out; + } + + sge = kcalloc(1, sizeof(struct sched_group_energy), + GFP_NOWAIT); + + nstates = (prop->length / sizeof(u32)) / 2; + cap_states = kcalloc(nstates, + sizeof(struct capacity_state), + GFP_NOWAIT); + + for (i = 0, val = prop->value; i < nstates; i++) { + cap_states[i].cap = be32_to_cpup(val++); + cap_states[i].power = be32_to_cpup(val++); + } + + sge->nr_cap_states = nstates; + sge->cap_states = cap_states; + + prop = of_find_property(cp, "idle-cost-data", NULL); + if (!prop || !prop->value) { + pr_warn("No idle-cost data, skipping sched_energy init\n"); + goto out; + } + + nstates = (prop->length / sizeof(u32)); + idle_states = kcalloc(nstates, + sizeof(struct idle_state), + GFP_NOWAIT); + + for (i = 0, val = prop->value; i < nstates; i++) + idle_states[i].power = be32_to_cpup(val++); + + sge->nr_idle_states = nstates; + sge->idle_states = idle_states; + + sge_array[cpu][sd_level] = sge; + } + } + + pr_info("Sched-energy-costs installed from DT\n"); + return; + +out: + free_resources(); +} diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index c242944f5cbd56..0771ad827e3b98 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -30,10 +30,13 @@ #include #include #include +#include #include #include "sched.h" +#include "tune.h" +#include "walt.h" /* * Targeted preemption latency for CPU-bound tasks: @@ -50,6 +53,17 @@ unsigned int sysctl_sched_latency = 6000000ULL; unsigned int normalized_sysctl_sched_latency = 6000000ULL; +unsigned int sysctl_sched_is_big_little = 0; +unsigned int sysctl_sched_sync_hint_enable = 1; +unsigned int sysctl_sched_initial_task_util = 0; +unsigned int sysctl_sched_cstate_aware = 1; + +#ifdef CONFIG_SCHED_WALT +unsigned int sysctl_sched_use_walt_cpu_util = 1; +unsigned int sysctl_sched_use_walt_task_util = 1; +__read_mostly unsigned int sysctl_sched_walt_cpu_high_irqload = + (10 * NSEC_PER_MSEC); +#endif /* * The initial- and re-scaling of tunables is configurable * (default SCHED_TUNABLESCALING_LOG = *(1+ilog(ncpus)) @@ -699,10 +713,13 @@ void init_entity_runnable_average(struct sched_entity *se) if (entity_is_task(se)) sa->load_avg = scale_load_down(se->load.weight); sa->load_sum = sa->load_avg * LOAD_AVG_MAX; + /* * At this point, util_avg won't be used in select_task_rq_fair anyway */ - sa->util_avg = 0; + sa->util_avg = sched_freq() ? + sysctl_sched_initial_task_util : + 0; sa->util_sum = 0; /* when this task enqueue'ed, it will contribute to its cfs_rq's load_avg */ } @@ -937,6 +954,7 @@ update_stats_enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se) } trace_sched_stat_blocked(tsk, delta); + trace_sched_blocked_reason(tsk); /* * Blocking time is in units of nanosecs, so shift by @@ -2815,6 +2833,7 @@ __update_load_avg(u64 now, int cpu, struct sched_avg *sa, scale_freq = arch_scale_freq_capacity(NULL, cpu); scale_cpu = arch_scale_cpu_capacity(NULL, cpu); + trace_sched_contrib_scale_f(cpu, scale_freq, scale_cpu); /* delta_w is the amount already accumulated against our next period */ delta_w = sa->period_contrib; @@ -3082,6 +3101,10 @@ static inline void update_load_avg(struct sched_entity *se, int update_tg) if (update_cfs_rq_load_avg(now, cfs_rq, true) && update_tg) update_tg_load_avg(cfs_rq, 0); + + if (entity_is_task(se)) + trace_sched_load_avg_task(task_of(se), &se->avg); + trace_sched_load_avg_cpu(cpu, cfs_rq); } /** @@ -4527,6 +4550,30 @@ static inline void hrtick_update(struct rq *rq) } #endif +#ifdef CONFIG_SMP +static bool cpu_overutilized(int cpu); +static unsigned long capacity_orig_of(int cpu); +static unsigned long cpu_util(int cpu); +static inline unsigned long boosted_cpu_util(int cpu); +#else +#define boosted_cpu_util(cpu) cpu_util(cpu) +#endif + +#ifdef CONFIG_SMP +static void update_capacity_of(int cpu) +{ + unsigned long req_cap; + + if (!sched_freq()) + return; + + /* Convert scale-invariant capacity to cpu. */ + req_cap = boosted_cpu_util(cpu); + req_cap = req_cap * SCHED_CAPACITY_SCALE / capacity_orig_of(cpu); + set_cfs_cpu_capacity(cpu, true, req_cap); +} +#endif + /* * The enqueue_task method is called before nr_running is * increased. Here we update the fair scheduling stats and @@ -4537,6 +4584,10 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) { struct cfs_rq *cfs_rq; struct sched_entity *se = &p->se; +#ifdef CONFIG_SMP + int task_new = flags & ENQUEUE_WAKEUP_NEW; + int task_wakeup = flags & ENQUEUE_WAKEUP; +#endif /* * If in_iowait is set, the code below may not trigger any cpufreq @@ -4561,6 +4612,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) if (cfs_rq_throttled(cfs_rq)) break; cfs_rq->h_nr_running++; + walt_inc_cfs_cumulative_runnable_avg(cfs_rq, p); flags = ENQUEUE_WAKEUP; } @@ -4568,6 +4620,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) for_each_sched_entity(se) { cfs_rq = cfs_rq_of(se); cfs_rq->h_nr_running++; + walt_inc_cfs_cumulative_runnable_avg(cfs_rq, p); if (cfs_rq_throttled(cfs_rq)) break; @@ -4579,6 +4632,31 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) if (!se) add_nr_running(rq, 1); +#ifdef CONFIG_SMP + + if (!se) { + walt_inc_cumulative_runnable_avg(rq, p); + if (!task_new && !rq->rd->overutilized && + cpu_overutilized(rq->cpu)) { + rq->rd->overutilized = true; + trace_sched_overutilized(true); + } + + /* + * We want to potentially trigger a freq switch + * request only for tasks that are waking up; this is + * because we get here also during load balancing, but + * in these cases it seems wise to trigger as single + * request after load balancing is done. + */ + if (task_new || task_wakeup) + update_capacity_of(cpu_of(rq)); + } + + /* Update SchedTune accouting */ + schedtune_enqueue_task(p, cpu_of(rq)); + +#endif /* CONFIG_SMP */ hrtick_update(rq); } @@ -4608,6 +4686,7 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) if (cfs_rq_throttled(cfs_rq)) break; cfs_rq->h_nr_running--; + walt_dec_cfs_cumulative_runnable_avg(cfs_rq, p); /* Don't dequeue parent if it has other entities besides us */ if (cfs_rq->load.weight) { @@ -4627,6 +4706,7 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) for_each_sched_entity(se) { cfs_rq = cfs_rq_of(se); cfs_rq->h_nr_running--; + walt_dec_cfs_cumulative_runnable_avg(cfs_rq, p); if (cfs_rq_throttled(cfs_rq)) break; @@ -4638,6 +4718,32 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) if (!se) sub_nr_running(rq, 1); +#ifdef CONFIG_SMP + + if (!se) { + walt_dec_cumulative_runnable_avg(rq, p); + + /* + * We want to potentially trigger a freq switch + * request only for tasks that are going to sleep; + * this is because we get here also during load + * balancing, but in these cases it seems wise to + * trigger as single request after load balancing is + * done. + */ + if (task_sleep) { + if (rq->cfs.nr_running) + update_capacity_of(cpu_of(rq)); + else if (sched_freq()) + set_cfs_cpu_capacity(cpu_of(rq), false, 0); + } + } + + /* Update SchedTune accouting */ + schedtune_dequeue_task(p, cpu_of(rq)); + +#endif /* CONFIG_SMP */ + hrtick_update(rq); } @@ -4944,15 +5050,6 @@ static unsigned long target_load(int cpu, int type) return max(rq->cpu_load[type-1], total); } -static unsigned long capacity_of(int cpu) -{ - return cpu_rq(cpu)->cpu_capacity; -} - -static unsigned long capacity_orig_of(int cpu) -{ - return cpu_rq(cpu)->cpu_capacity_orig; -} static unsigned long cpu_avg_load_per_task(int cpu) { @@ -5103,6 +5200,397 @@ static void record_wakee(struct task_struct *p) } } +/* + * Returns the current capacity of cpu after applying both + * cpu and freq scaling. + */ +unsigned long capacity_curr_of(int cpu) +{ + return cpu_rq(cpu)->cpu_capacity_orig * + arch_scale_freq_capacity(NULL, cpu) + >> SCHED_CAPACITY_SHIFT; +} + +static inline bool energy_aware(void) +{ + return sched_feat(ENERGY_AWARE); +} + +struct energy_env { + struct sched_group *sg_top; + struct sched_group *sg_cap; + int cap_idx; + int util_delta; + int src_cpu; + int dst_cpu; + int energy; + int payoff; + struct task_struct *task; + struct { + int before; + int after; + int delta; + int diff; + } nrg; + struct { + int before; + int after; + int delta; + } cap; +}; + +/* + * __cpu_norm_util() returns the cpu util relative to a specific capacity, + * i.e. it's busy ratio, in the range [0..SCHED_LOAD_SCALE] which is useful for + * energy calculations. Using the scale-invariant util returned by + * cpu_util() and approximating scale-invariant util by: + * + * util ~ (curr_freq/max_freq)*1024 * capacity_orig/1024 * running_time/time + * + * the normalized util can be found using the specific capacity. + * + * capacity = capacity_orig * curr_freq/max_freq + * + * norm_util = running_time/time ~ util/capacity + */ +static unsigned long __cpu_norm_util(int cpu, unsigned long capacity, int delta) +{ + int util = __cpu_util(cpu, delta); + + if (util >= capacity) + return SCHED_CAPACITY_SCALE; + + return (util << SCHED_CAPACITY_SHIFT)/capacity; +} + +static int calc_util_delta(struct energy_env *eenv, int cpu) +{ + if (cpu == eenv->src_cpu) + return -eenv->util_delta; + if (cpu == eenv->dst_cpu) + return eenv->util_delta; + return 0; +} + +static +unsigned long group_max_util(struct energy_env *eenv) +{ + int i, delta; + unsigned long max_util = 0; + + for_each_cpu(i, sched_group_cpus(eenv->sg_cap)) { + delta = calc_util_delta(eenv, i); + max_util = max(max_util, __cpu_util(i, delta)); + } + + return max_util; +} + +/* + * group_norm_util() returns the approximated group util relative to it's + * current capacity (busy ratio) in the range [0..SCHED_LOAD_SCALE] for use in + * energy calculations. Since task executions may or may not overlap in time in + * the group the true normalized util is between max(cpu_norm_util(i)) and + * sum(cpu_norm_util(i)) when iterating over all cpus in the group, i. The + * latter is used as the estimate as it leads to a more pessimistic energy + * estimate (more busy). + */ +static unsigned +long group_norm_util(struct energy_env *eenv, struct sched_group *sg) +{ + int i, delta; + unsigned long util_sum = 0; + unsigned long capacity = sg->sge->cap_states[eenv->cap_idx].cap; + + for_each_cpu(i, sched_group_cpus(sg)) { + delta = calc_util_delta(eenv, i); + util_sum += __cpu_norm_util(i, capacity, delta); + } + + if (util_sum > SCHED_CAPACITY_SCALE) + return SCHED_CAPACITY_SCALE; + return util_sum; +} + +static int find_new_capacity(struct energy_env *eenv, + const struct sched_group_energy const *sge) +{ + int idx; + unsigned long util = group_max_util(eenv); + + for (idx = 0; idx < sge->nr_cap_states; idx++) { + if (sge->cap_states[idx].cap >= util) + break; + } + + eenv->cap_idx = idx; + + return idx; +} + +static int group_idle_state(struct sched_group *sg) +{ + int i, state = INT_MAX; + + /* Find the shallowest idle state in the sched group. */ + for_each_cpu(i, sched_group_cpus(sg)) + state = min(state, idle_get_state_idx(cpu_rq(i))); + + /* Take non-cpuidle idling into account (active idle/arch_cpu_idle()) */ + state++; + + return state; +} + +/* + * sched_group_energy(): Computes the absolute energy consumption of cpus + * belonging to the sched_group including shared resources shared only by + * members of the group. Iterates over all cpus in the hierarchy below the + * sched_group starting from the bottom working it's way up before going to + * the next cpu until all cpus are covered at all levels. The current + * implementation is likely to gather the same util statistics multiple times. + * This can probably be done in a faster but more complex way. + * Note: sched_group_energy() may fail when racing with sched_domain updates. + */ +static int sched_group_energy(struct energy_env *eenv) +{ + struct sched_domain *sd; + int cpu, total_energy = 0; + struct cpumask visit_cpus; + struct sched_group *sg; + + WARN_ON(!eenv->sg_top->sge); + + cpumask_copy(&visit_cpus, sched_group_cpus(eenv->sg_top)); + + while (!cpumask_empty(&visit_cpus)) { + struct sched_group *sg_shared_cap = NULL; + + cpu = cpumask_first(&visit_cpus); + + /* + * Is the group utilization affected by cpus outside this + * sched_group? + */ + sd = rcu_dereference(per_cpu(sd_scs, cpu)); + + if (!sd) + /* + * We most probably raced with hotplug; returning a + * wrong energy estimation is better than entering an + * infinite loop. + */ + return -EINVAL; + + if (sd->parent) + sg_shared_cap = sd->parent->groups; + + for_each_domain(cpu, sd) { + sg = sd->groups; + + /* Has this sched_domain already been visited? */ + if (sd->child && group_first_cpu(sg) != cpu) + break; + + do { + unsigned long group_util; + int sg_busy_energy, sg_idle_energy; + int cap_idx, idle_idx; + + if (sg_shared_cap && sg_shared_cap->group_weight >= sg->group_weight) + eenv->sg_cap = sg_shared_cap; + else + eenv->sg_cap = sg; + + cap_idx = find_new_capacity(eenv, sg->sge); + + if (sg->group_weight == 1) { + /* Remove capacity of src CPU (before task move) */ + if (eenv->util_delta == 0 && + cpumask_test_cpu(eenv->src_cpu, sched_group_cpus(sg))) { + eenv->cap.before = sg->sge->cap_states[cap_idx].cap; + eenv->cap.delta -= eenv->cap.before; + } + /* Add capacity of dst CPU (after task move) */ + if (eenv->util_delta != 0 && + cpumask_test_cpu(eenv->dst_cpu, sched_group_cpus(sg))) { + eenv->cap.after = sg->sge->cap_states[cap_idx].cap; + eenv->cap.delta += eenv->cap.after; + } + } + + idle_idx = group_idle_state(sg); + group_util = group_norm_util(eenv, sg); + sg_busy_energy = (group_util * sg->sge->cap_states[cap_idx].power) + >> SCHED_CAPACITY_SHIFT; + sg_idle_energy = ((SCHED_CAPACITY_SCALE-group_util) + * sg->sge->idle_states[idle_idx].power) + >> SCHED_CAPACITY_SHIFT; + + total_energy += sg_busy_energy + sg_idle_energy; + + if (!sd->child) + cpumask_xor(&visit_cpus, &visit_cpus, sched_group_cpus(sg)); + + if (cpumask_equal(sched_group_cpus(sg), sched_group_cpus(eenv->sg_top))) + goto next_cpu; + + } while (sg = sg->next, sg != sd->groups); + } +next_cpu: + cpumask_clear_cpu(cpu, &visit_cpus); + continue; + } + + eenv->energy = total_energy; + return 0; +} + +static inline bool cpu_in_sg(struct sched_group *sg, int cpu) +{ + return cpu != -1 && cpumask_test_cpu(cpu, sched_group_cpus(sg)); +} + +/* + * energy_diff(): Estimate the energy impact of changing the utilization + * distribution. eenv specifies the change: utilisation amount, source, and + * destination cpu. Source or destination cpu may be -1 in which case the + * utilization is removed from or added to the system (e.g. task wake-up). If + * both are specified, the utilization is migrated. + */ +static inline int __energy_diff(struct energy_env *eenv) +{ + struct sched_domain *sd; + struct sched_group *sg; + int sd_cpu = -1, energy_before = 0, energy_after = 0; + + struct energy_env eenv_before = { + .util_delta = 0, + .src_cpu = eenv->src_cpu, + .dst_cpu = eenv->dst_cpu, + .nrg = { 0, 0, 0, 0}, + .cap = { 0, 0, 0 }, + }; + + if (eenv->src_cpu == eenv->dst_cpu) + return 0; + + sd_cpu = (eenv->src_cpu != -1) ? eenv->src_cpu : eenv->dst_cpu; + sd = rcu_dereference(per_cpu(sd_ea, sd_cpu)); + + if (!sd) + return 0; /* Error */ + + sg = sd->groups; + + do { + if (cpu_in_sg(sg, eenv->src_cpu) || cpu_in_sg(sg, eenv->dst_cpu)) { + eenv_before.sg_top = eenv->sg_top = sg; + + if (sched_group_energy(&eenv_before)) + return 0; /* Invalid result abort */ + energy_before += eenv_before.energy; + + /* Keep track of SRC cpu (before) capacity */ + eenv->cap.before = eenv_before.cap.before; + eenv->cap.delta = eenv_before.cap.delta; + + if (sched_group_energy(eenv)) + return 0; /* Invalid result abort */ + energy_after += eenv->energy; + } + } while (sg = sg->next, sg != sd->groups); + + eenv->nrg.before = energy_before; + eenv->nrg.after = energy_after; + eenv->nrg.diff = eenv->nrg.after - eenv->nrg.before; + eenv->payoff = 0; + + trace_sched_energy_diff(eenv->task, + eenv->src_cpu, eenv->dst_cpu, eenv->util_delta, + eenv->nrg.before, eenv->nrg.after, eenv->nrg.diff, + eenv->cap.before, eenv->cap.after, eenv->cap.delta, + eenv->nrg.delta, eenv->payoff); + + return eenv->nrg.diff; +} + +#ifdef CONFIG_SCHED_TUNE + +struct target_nrg schedtune_target_nrg; + +/* + * System energy normalization + * Returns the normalized value, in the range [0..SCHED_LOAD_SCALE], + * corresponding to the specified energy variation. + */ +static inline int +normalize_energy(int energy_diff) +{ + u32 normalized_nrg; +#ifdef CONFIG_SCHED_DEBUG + int max_delta; + + /* Check for boundaries */ + max_delta = schedtune_target_nrg.max_power; + max_delta -= schedtune_target_nrg.min_power; + WARN_ON(abs(energy_diff) >= max_delta); +#endif + + /* Do scaling using positive numbers to increase the range */ + normalized_nrg = (energy_diff < 0) ? -energy_diff : energy_diff; + + /* Scale by energy magnitude */ + normalized_nrg <<= SCHED_CAPACITY_SHIFT; + + /* Normalize on max energy for target platform */ + normalized_nrg = reciprocal_divide( + normalized_nrg, schedtune_target_nrg.rdiv); + + return (energy_diff < 0) ? -normalized_nrg : normalized_nrg; +} + +static inline int +energy_diff(struct energy_env *eenv) +{ + unsigned int boost; + int nrg_delta; + + /* Conpute "absolute" energy diff */ + __energy_diff(eenv); + + /* Return energy diff when boost margin is 0 */ +#ifdef CONFIG_CGROUP_SCHEDTUNE + boost = schedtune_task_boost(eenv->task); +#else + boost = get_sysctl_sched_cfs_boost(); +#endif + if (boost == 0) + return eenv->nrg.diff; + + /* Compute normalized energy diff */ + nrg_delta = normalize_energy(eenv->nrg.diff); + eenv->nrg.delta = nrg_delta; + + eenv->payoff = schedtune_accept_deltas( + eenv->nrg.delta, + eenv->cap.delta, + eenv->task); + + /* + * When SchedTune is enabled, the energy_diff() function will return + * the computed energy payoff value. Since the energy_diff() return + * value is expected to be negative by its callers, this evaluation + * function return a negative value each time the evaluation return a + * positive payoff, which is the condition for the acceptance of + * a scheduling decision + */ + return -eenv->payoff; +} +#else /* CONFIG_SCHED_TUNE */ +#define energy_diff(eenv) __energy_diff(eenv) +#endif + /* * Detect M:N waker/wakee relationships via a switching-frequency heuristic. * @@ -5199,6 +5687,169 @@ static int wake_affine(struct sched_domain *sd, struct task_struct *p, return 1; } +static inline int task_util(struct task_struct *p) +{ +#ifdef CONFIG_SCHED_WALT + if (!walt_disabled && sysctl_sched_use_walt_task_util) { + unsigned long demand = p->ravg.demand; + return (demand << 10) / walt_ravg_window; + } +#endif + return p->se.avg.util_avg; +} + +static inline unsigned long boosted_task_util(struct task_struct *task); + +static inline bool __task_fits(struct task_struct *p, int cpu, int util) +{ + unsigned long capacity = capacity_of(cpu); + + util += boosted_task_util(p); + + return (capacity * 1024) > (util * capacity_margin); +} + +static inline bool task_fits_max(struct task_struct *p, int cpu) +{ + unsigned long capacity = capacity_of(cpu); + unsigned long max_capacity = cpu_rq(cpu)->rd->max_cpu_capacity.val; + + if (capacity == max_capacity) + return true; + + if (capacity * capacity_margin > max_capacity * 1024) + return true; + + return __task_fits(p, cpu, 0); +} + +static inline bool task_fits_spare(struct task_struct *p, int cpu) +{ + return __task_fits(p, cpu, cpu_util(cpu)); +} + +static bool cpu_overutilized(int cpu) +{ + return (capacity_of(cpu) * 1024) < (cpu_util(cpu) * capacity_margin); +} + +#ifdef CONFIG_SCHED_TUNE + +static long +schedtune_margin(unsigned long signal, long boost) +{ + long long margin = 0; + + /* + * Signal proportional compensation (SPC) + * + * The Boost (B) value is used to compute a Margin (M) which is + * proportional to the complement of the original Signal (S): + * M = B * (SCHED_LOAD_SCALE - S), if B is positive + * M = B * S, if B is negative + * The obtained M could be used by the caller to "boost" S. + */ + + if (boost >= 0) { + margin = SCHED_CAPACITY_SCALE - signal; + margin *= boost; + } else + margin = -signal * boost; + /* + * Fast integer division by constant: + * Constant : (C) = 100 + * Precision : 0.1% (P) = 0.1 + * Reference : C * 100 / P (R) = 100000 + * + * Thus: + * Shift bits : ceil(log(R,2)) (S) = 17 + * Mult const : round(2^S/C) (M) = 1311 + * + * + */ + margin *= 1311; + margin >>= 17; + + if (boost < 0) + margin *= -1; + return margin; +} + +static inline int +schedtune_cpu_margin(unsigned long util, int cpu) +{ + int boost; + +#ifdef CONFIG_CGROUP_SCHEDTUNE + boost = schedtune_cpu_boost(cpu); +#else + boost = get_sysctl_sched_cfs_boost(); +#endif + if (boost == 0) + return 0; + + return schedtune_margin(util, boost); +} + +static inline long +schedtune_task_margin(struct task_struct *task) +{ + int boost; + unsigned long util; + long margin; + +#ifdef CONFIG_CGROUP_SCHEDTUNE + boost = schedtune_task_boost(task); +#else + boost = get_sysctl_sched_cfs_boost(); +#endif + if (boost == 0) + return 0; + + util = task_util(task); + margin = schedtune_margin(util, boost); + + return margin; +} + +#else /* CONFIG_SCHED_TUNE */ + +static inline int +schedtune_cpu_margin(unsigned long util, int cpu) +{ + return 0; +} + +static inline int +schedtune_task_margin(struct task_struct *task) +{ + return 0; +} + +#endif /* CONFIG_SCHED_TUNE */ + +static inline unsigned long +boosted_cpu_util(int cpu) +{ + unsigned long util = cpu_util(cpu); + long margin = schedtune_cpu_margin(util, cpu); + + trace_sched_boost_cpu(cpu, util, margin); + + return util + margin; +} + +static inline unsigned long +boosted_task_util(struct task_struct *task) +{ + unsigned long util = task_util(task); + long margin = schedtune_task_margin(task); + + trace_sched_boost_task(task, util, margin); + + return util + margin; +} + /* * find_idlest_group finds and returns the least busy CPU group within the * domain. @@ -5208,7 +5859,10 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu, int sd_flag) { struct sched_group *idlest = NULL, *group = sd->groups; + struct sched_group *fit_group = NULL, *spare_group = NULL; unsigned long min_load = ULONG_MAX, this_load = 0; + unsigned long fit_capacity = ULONG_MAX; + unsigned long max_spare_capacity = capacity_margin - SCHED_CAPACITY_SCALE; int load_idx = sd->forkexec_idx; int imbalance = 100 + (sd->imbalance_pct-100)/2; @@ -5216,7 +5870,7 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, load_idx = sd->wake_idx; do { - unsigned long load, avg_load; + unsigned long load, avg_load, spare_capacity; int local_group; int i; @@ -5239,6 +5893,25 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, load = target_load(i, load_idx); avg_load += load; + + /* + * Look for most energy-efficient group that can fit + * that can fit the task. + */ + if (capacity_of(i) < fit_capacity && task_fits_spare(p, i)) { + fit_capacity = capacity_of(i); + fit_group = group; + } + + /* + * Look for group which has most spare capacity on a + * single cpu. + */ + spare_capacity = capacity_of(i) - cpu_util(i); + if (spare_capacity > max_spare_capacity) { + max_spare_capacity = spare_capacity; + spare_group = group; + } } /* Adjust by relative CPU capacity of the group */ @@ -5252,6 +5925,12 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, } } while (group = group->next, group != sd->groups); + if (fit_group) + return fit_group; + + if (spare_group) + return spare_group; + if (!idlest || 100*this_load < imbalance*min_load) return NULL; return idlest; @@ -5276,7 +5955,7 @@ find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu) /* Traverse only the allowed CPUs */ for_each_cpu_and(i, sched_group_cpus(group), tsk_cpus_allowed(p)) { - if (idle_cpu(i)) { + if (task_fits_spare(p, i)) { struct rq *rq = cpu_rq(i); struct cpuidle_state *idle = idle_get_state(rq); if (idle && idle->exit_latency < min_exit_latency) { @@ -5288,7 +5967,8 @@ find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu) min_exit_latency = idle->exit_latency; latest_idle_timestamp = rq->idle_stamp; shallowest_idle_cpu = i; - } else if ((!idle || idle->exit_latency == min_exit_latency) && + } else if (idle_cpu(i) && + (!idle || idle->exit_latency == min_exit_latency) && rq->idle_stamp > latest_idle_timestamp) { /* * If equal or no active idle state, then @@ -5297,6 +5977,13 @@ find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu) */ latest_idle_timestamp = rq->idle_stamp; shallowest_idle_cpu = i; + } else if (shallowest_idle_cpu == -1) { + /* + * If we haven't found an idle CPU yet + * pick a non-idle one that can fit the task as + * fallback. + */ + shallowest_idle_cpu = i; } } else if (shallowest_idle_cpu == -1) { load = weighted_cpuload(i); @@ -5520,94 +6207,305 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t static int select_idle_sibling(struct task_struct *p, int prev, int target) { struct sched_domain *sd; - int i; + struct sched_group *sg; + int i = task_cpu(p); + int best_idle = -1; + int best_idle_cstate = -1; + int best_idle_capacity = INT_MAX; - if (idle_cpu(target)) - return target; + if (!sysctl_sched_cstate_aware) { + if (idle_cpu(target)) + return target; - /* - * If the previous cpu is cache affine and idle, don't be stupid. - */ - if (prev != target && cpus_share_cache(prev, target) && idle_cpu(prev)) - return prev; + /* + * If the prevous cpu is cache affine and idle, don't be stupid. + */ + if (i != target && cpus_share_cache(i, target) && idle_cpu(i)) + return i; - sd = rcu_dereference(per_cpu(sd_llc, target)); - if (!sd) - return target; + sd = rcu_dereference(per_cpu(sd_llc, target)); + if (!sd) + return target; - i = select_idle_core(p, sd, target); - if ((unsigned)i < nr_cpumask_bits) - return i; + i = select_idle_core(p, sd, target); + if ((unsigned)i < nr_cpumask_bits) + return i; - i = select_idle_cpu(p, sd, target); - if ((unsigned)i < nr_cpumask_bits) - return i; + i = select_idle_cpu(p, sd, target); + if ((unsigned)i < nr_cpumask_bits) + return i; - i = select_idle_smt(p, sd, target); - if ((unsigned)i < nr_cpumask_bits) - return i; + i = select_idle_smt(p, sd, target); + if ((unsigned)i < nr_cpumask_bits) + return i; + } - return target; -} + /* + * Otherwise, iterate the domains and find an elegible idle cpu. + */ + sd = rcu_dereference(per_cpu(sd_llc, target)); + for_each_lower_domain(sd) { + sg = sd->groups; + do { + if (!cpumask_intersects(sched_group_cpus(sg), + tsk_cpus_allowed(p))) + goto next; + + + if (sysctl_sched_cstate_aware) { + for_each_cpu_and(i, tsk_cpus_allowed(p), sched_group_cpus(sg)) { + struct rq *rq = cpu_rq(i); + int idle_idx = idle_get_state_idx(rq); + unsigned long new_usage = boosted_task_util(p); + unsigned long capacity_orig = capacity_orig_of(i); + if (new_usage > capacity_orig || !idle_cpu(i)) + goto next; + + if (i == target && new_usage <= capacity_curr_of(target)) + return target; + + if (best_idle < 0 || (idle_idx < best_idle_cstate && capacity_orig <= best_idle_capacity)) { + best_idle = i; + best_idle_cstate = idle_idx; + best_idle_capacity = capacity_orig; + } + } + } else { + for_each_cpu(i, sched_group_cpus(sg)) { + if (i == target || !idle_cpu(i)) + goto next; + } -/* - * cpu_util returns the amount of capacity of a CPU that is used by CFS - * tasks. The unit of the return value must be the one of capacity so we can - * compare the utilization with the capacity of the CPU that is available for - * CFS task (ie cpu_capacity). - * - * cfs_rq.avg.util_avg is the sum of running time of runnable tasks plus the - * recent utilization of currently non-runnable tasks on a CPU. It represents - * the amount of utilization of a CPU in the range [0..capacity_orig] where - * capacity_orig is the cpu_capacity available at the highest frequency - * (arch_scale_freq_capacity()). - * The utilization of a CPU converges towards a sum equal to or less than the - * current capacity (capacity_curr <= capacity_orig) of the CPU because it is - * the running time on this CPU scaled by capacity_curr. - * - * Nevertheless, cfs_rq.avg.util_avg can be higher than capacity_curr or even - * higher than capacity_orig because of unfortunate rounding in - * cfs.avg.util_avg or just after migrating tasks and new task wakeups until - * the average stabilizes with the new running time. We need to check that the - * utilization stays within the range of [0..capacity_orig] and cap it if - * necessary. Without utilization capping, a group could be seen as overloaded - * (CPU0 utilization at 121% + CPU1 utilization at 80%) whereas CPU1 has 20% of - * available capacity. We allow utilization to overshoot capacity_curr (but not - * capacity_orig) as it useful for predicting the capacity required after task - * migrations (scheduler-driven DVFS). - */ -static int cpu_util(int cpu) -{ - unsigned long util = cpu_rq(cpu)->cfs.avg.util_avg; - unsigned long capacity = capacity_orig_of(cpu); - - return (util >= capacity) ? capacity : util; + target = cpumask_first_and(sched_group_cpus(sg), + tsk_cpus_allowed(p)); + goto done; + } +next: + sg = sg->next; + } while (sg != sd->groups); + } + if (best_idle > 0) + target = best_idle; + +done: + return target; } -static inline int task_util(struct task_struct *p) +static inline int find_best_target(struct task_struct *p, bool boosted, bool prefer_idle) { - return p->se.avg.util_avg; + int iter_cpu; + int target_cpu = -1; + int target_util = 0; + int backup_capacity = 0; + int best_idle_cpu = -1; + int best_idle_cstate = INT_MAX; + int backup_cpu = -1; + unsigned long task_util_boosted, new_util; + + task_util_boosted = boosted_task_util(p); + for (iter_cpu = 0; iter_cpu < NR_CPUS; iter_cpu++) { + int cur_capacity; + struct rq *rq; + int idle_idx; + + /* + * Iterate from higher cpus for boosted tasks. + */ + int i = boosted ? NR_CPUS-iter_cpu-1 : iter_cpu; + + if (!cpu_online(i) || !cpumask_test_cpu(i, tsk_cpus_allowed(p))) + continue; + + /* + * p's blocked utilization is still accounted for on prev_cpu + * so prev_cpu will receive a negative bias due to the double + * accounting. However, the blocked utilization may be zero. + */ + new_util = cpu_util(i) + task_util_boosted; + + /* + * Ensure minimum capacity to grant the required boost. + * The target CPU can be already at a capacity level higher + * than the one required to boost the task. + */ + if (new_util > capacity_orig_of(i)) + continue; + +#ifdef CONFIG_SCHED_WALT + if (walt_cpu_high_irqload(i)) + continue; +#endif + /* + * Unconditionally favoring tasks that prefer idle cpus to + * improve latency. + */ + if (idle_cpu(i) && prefer_idle) { + if (best_idle_cpu < 0) + best_idle_cpu = i; + continue; + } + + cur_capacity = capacity_curr_of(i); + rq = cpu_rq(i); + idle_idx = idle_get_state_idx(rq); + + if (new_util < cur_capacity) { + if (cpu_rq(i)->nr_running) { + if(prefer_idle) { + // Find a target cpu with lowest + // utilization. + if (target_util == 0 || + target_util < new_util) { + target_cpu = i; + target_util = new_util; + } + } else { + // Find a target cpu with highest + // utilization. + if (target_util == 0 || + target_util > new_util) { + target_cpu = i; + target_util = new_util; + } + } + } else if (!prefer_idle) { + if (best_idle_cpu < 0 || + (sysctl_sched_cstate_aware && + best_idle_cstate > idle_idx)) { + best_idle_cstate = idle_idx; + best_idle_cpu = i; + } + } + } else if (backup_capacity == 0 || + backup_capacity > cur_capacity) { + // Find a backup cpu with least capacity. + backup_capacity = cur_capacity; + backup_cpu = i; + } + } + + if (prefer_idle && best_idle_cpu >= 0) + target_cpu = best_idle_cpu; + else if (target_cpu < 0) + target_cpu = best_idle_cpu >= 0 ? best_idle_cpu : backup_cpu; + + return target_cpu; } -/* - * Disable WAKE_AFFINE in the case where task @p doesn't fit in the - * capacity of either the waking CPU @cpu or the previous CPU @prev_cpu. - * - * In that case WAKE_AFFINE doesn't make sense and we'll let - * BALANCE_WAKE sort things out. - */ -static int wake_cap(struct task_struct *p, int cpu, int prev_cpu) +static int energy_aware_wake_cpu(struct task_struct *p, int target, int sync) { - long min_cap, max_cap; + struct sched_domain *sd; + struct sched_group *sg, *sg_target; + int target_max_cap = INT_MAX; + int target_cpu = task_cpu(p); + unsigned long task_util_boosted, new_util; + int i; - min_cap = min(capacity_orig_of(prev_cpu), capacity_orig_of(cpu)); - max_cap = cpu_rq(cpu)->rd->max_cpu_capacity; + if (sysctl_sched_sync_hint_enable && sync) { + int cpu = smp_processor_id(); + cpumask_t search_cpus; + cpumask_and(&search_cpus, tsk_cpus_allowed(p), cpu_online_mask); + if (cpumask_test_cpu(cpu, &search_cpus)) + return cpu; + } - /* Minimum capacity is close to max, no need to abort wake_affine */ - if (max_cap - min_cap < max_cap >> 3) - return 0; + sd = rcu_dereference(per_cpu(sd_ea, task_cpu(p))); + + if (!sd) + return target; + + sg = sd->groups; + sg_target = sg; - return min_cap * 1024 < task_util(p) * capacity_margin; + if (sysctl_sched_is_big_little) { + + /* + * Find group with sufficient capacity. We only get here if no cpu is + * overutilized. We may end up overutilizing a cpu by adding the task, + * but that should not be any worse than select_idle_sibling(). + * load_balance() should sort it out later as we get above the tipping + * point. + */ + do { + /* Assuming all cpus are the same in group */ + int max_cap_cpu = group_first_cpu(sg); + + /* + * Assume smaller max capacity means more energy-efficient. + * Ideally we should query the energy model for the right + * answer but it easily ends up in an exhaustive search. + */ + if (capacity_of(max_cap_cpu) < target_max_cap && + task_fits_max(p, max_cap_cpu)) { + sg_target = sg; + target_max_cap = capacity_of(max_cap_cpu); + } + } while (sg = sg->next, sg != sd->groups); + + task_util_boosted = boosted_task_util(p); + /* Find cpu with sufficient capacity */ + for_each_cpu_and(i, tsk_cpus_allowed(p), sched_group_cpus(sg_target)) { + /* + * p's blocked utilization is still accounted for on prev_cpu + * so prev_cpu will receive a negative bias due to the double + * accounting. However, the blocked utilization may be zero. + */ + new_util = cpu_util(i) + task_util_boosted; + + /* + * Ensure minimum capacity to grant the required boost. + * The target CPU can be already at a capacity level higher + * than the one required to boost the task. + */ + if (new_util > capacity_orig_of(i)) + continue; + + if (new_util < capacity_curr_of(i)) { + target_cpu = i; + if (cpu_rq(i)->nr_running) + break; + } + + /* cpu has capacity at higher OPP, keep it as fallback */ + if (target_cpu == task_cpu(p)) + target_cpu = i; + } + } else { + /* + * Find a cpu with sufficient capacity + */ +#ifdef CONFIG_CGROUP_SCHEDTUNE + bool boosted = schedtune_task_boost(p) > 0; + bool prefer_idle = schedtune_prefer_idle(p) > 0; +#else + bool boosted = 0; + bool prefer_idle = 0; +#endif + int tmp_target = find_best_target(p, boosted, prefer_idle); + if (tmp_target >= 0) { + target_cpu = tmp_target; + if ((boosted || prefer_idle) && idle_cpu(target_cpu)) + return target_cpu; + } + } + + if (target_cpu != task_cpu(p)) { + struct energy_env eenv = { + .util_delta = task_util(p), + .src_cpu = task_cpu(p), + .dst_cpu = target_cpu, + .task = p, + }; + + /* Not enough spare capacity on previous cpu */ + if (cpu_overutilized(task_cpu(p))) + return target_cpu; + + if (energy_diff(&eenv) >= 0) + return task_cpu(p); + } + + return target_cpu; } /* @@ -5633,8 +6531,9 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_f if (sd_flag & SD_BALANCE_WAKE) { record_wakee(p); - want_affine = !wake_wide(p) && !wake_cap(p, cpu, prev_cpu) - && cpumask_test_cpu(cpu, tsk_cpus_allowed(p)); + want_affine = (!wake_wide(p) && task_fits_max(p, cpu) && + cpumask_test_cpu(cpu, tsk_cpus_allowed(p))) || + energy_aware(); } rcu_read_lock(); @@ -5665,7 +6564,9 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_f } if (!sd) { - if (sd_flag & SD_BALANCE_WAKE) /* XXX always ? */ + if (energy_aware() && !cpu_rq(cpu)->rd->overutilized) + new_cpu = energy_aware_wake_cpu(p, prev_cpu, sync); + else if (sd_flag & SD_BALANCE_WAKE) /* XXX always ? */ new_cpu = select_idle_sibling(p, prev_cpu, new_cpu); } else while (sd) { @@ -5760,6 +6661,8 @@ static void task_dead_fair(struct task_struct *p) { remove_entity_load_avg(&p->se); } +#else +#define task_fits_max(p, cpu) true #endif /* CONFIG_SMP */ static unsigned long @@ -6006,6 +6909,8 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct pin_cookie c if (hrtick_enabled(rq)) hrtick_start_fair(rq, p); + rq->misfit_task = !task_fits_max(p, rq->cpu); + return p; simple: cfs_rq = &rq->cfs; @@ -6027,9 +6932,12 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct pin_cookie c if (hrtick_enabled(rq)) hrtick_start_fair(rq, p); + rq->misfit_task = !task_fits_max(p, rq->cpu); + return p; idle: + rq->misfit_task = 0; /* * This is OK, because current is on_cpu, which avoids it being picked * for load-balance and preemption/IRQs are still disabled avoiding @@ -6242,6 +7150,13 @@ static unsigned long __read_mostly max_load_balance_interval = HZ/10; enum fbq_type { regular, remote, all }; +enum group_type { + group_other = 0, + group_misfit_task, + group_imbalanced, + group_overloaded, +}; + #define LBF_ALL_PINNED 0x01 #define LBF_NEED_BREAK 0x02 #define LBF_DST_PINNED 0x04 @@ -6260,6 +7175,7 @@ struct lb_env { int new_dst_cpu; enum cpu_idle_type idle; long imbalance; + unsigned int src_grp_nr_running; /* The set of CPUs under consideration for load-balancing */ struct cpumask *cpus; @@ -6596,6 +7512,10 @@ static void attach_one_task(struct rq *rq, struct task_struct *p) { raw_spin_lock(&rq->lock); attach_task(rq, p); + /* + * We want to potentially raise target_cpu's OPP. + */ + update_capacity_of(cpu_of(rq)); raw_spin_unlock(&rq->lock); } @@ -6617,6 +7537,11 @@ static void attach_tasks(struct lb_env *env) attach_task(env->dst_rq, p); } + /* + * We want to potentially raise env.dst_cpu's OPP. + */ + update_capacity_of(env->dst_cpu); + raw_spin_unlock(&env->dst_rq->lock); } @@ -6712,12 +7637,6 @@ static unsigned long task_h_load(struct task_struct *p) /********** Helpers for find_busiest_group ************************/ -enum group_type { - group_other = 0, - group_imbalanced, - group_overloaded, -}; - /* * sg_lb_stats - stats of a sched_group required for load_balancing */ @@ -6733,6 +7652,7 @@ struct sg_lb_stats { unsigned int group_weight; enum group_type group_type; int group_no_capacity; + int group_misfit_task; /* A cpu has a task too big for its capacity */ #ifdef CONFIG_NUMA_BALANCING unsigned int nr_numa_running; unsigned int nr_preferred_running; @@ -6830,13 +7750,43 @@ static unsigned long scale_rt_capacity(int cpu) return 1; } +void init_max_cpu_capacity(struct max_cpu_capacity *mcc) +{ + raw_spin_lock_init(&mcc->lock); + mcc->val = 0; + mcc->cpu = -1; +} + static void update_cpu_capacity(struct sched_domain *sd, int cpu) { unsigned long capacity = arch_scale_cpu_capacity(sd, cpu); struct sched_group *sdg = sd->groups; + struct max_cpu_capacity *mcc; + unsigned long max_capacity; + int max_cap_cpu; + unsigned long flags; cpu_rq(cpu)->cpu_capacity_orig = capacity; + mcc = &cpu_rq(cpu)->rd->max_cpu_capacity; + + raw_spin_lock_irqsave(&mcc->lock, flags); + max_capacity = mcc->val; + max_cap_cpu = mcc->cpu; + + if ((max_capacity > capacity && max_cap_cpu == cpu) || + (max_capacity < capacity)) { + mcc->val = capacity; + mcc->cpu = cpu; +#ifdef CONFIG_SCHED_DEBUG + raw_spin_unlock_irqrestore(&mcc->lock, flags); + pr_info("CPU%d: update max cpu_capacity %lu\n", cpu, capacity); + goto skip_unlock; +#endif + } + raw_spin_unlock_irqrestore(&mcc->lock, flags); + +skip_unlock: __attribute__ ((unused)); capacity *= scale_rt_capacity(cpu); capacity >>= SCHED_CAPACITY_SHIFT; @@ -6845,13 +7795,14 @@ static void update_cpu_capacity(struct sched_domain *sd, int cpu) cpu_rq(cpu)->cpu_capacity = capacity; sdg->sgc->capacity = capacity; + sdg->sgc->max_capacity = capacity; } void update_group_capacity(struct sched_domain *sd, int cpu) { struct sched_domain *child = sd->child; struct sched_group *group, *sdg = sd->groups; - unsigned long capacity; + unsigned long capacity, max_capacity; unsigned long interval; interval = msecs_to_jiffies(sd->balance_interval); @@ -6864,6 +7815,7 @@ void update_group_capacity(struct sched_domain *sd, int cpu) } capacity = 0; + max_capacity = 0; if (child->flags & SD_OVERLAP) { /* @@ -6888,11 +7840,12 @@ void update_group_capacity(struct sched_domain *sd, int cpu) */ if (unlikely(!rq->sd)) { capacity += capacity_of(cpu); - continue; + } else { + sgc = rq->sd->groups->sgc; + capacity += sgc->capacity; } - sgc = rq->sd->groups->sgc; - capacity += sgc->capacity; + max_capacity = max(capacity, max_capacity); } } else { /* @@ -6902,12 +7855,16 @@ void update_group_capacity(struct sched_domain *sd, int cpu) group = child->groups; do { - capacity += group->sgc->capacity; + struct sched_group_capacity *sgc = group->sgc; + + capacity += sgc->capacity; + max_capacity = max(sgc->max_capacity, max_capacity); group = group->next; } while (group != child->groups); } sdg->sgc->capacity = capacity; + sdg->sgc->max_capacity = max_capacity; } /* @@ -7012,6 +7969,9 @@ group_type group_classify(struct sched_group *group, if (sg_imbalanced(group)) return group_imbalanced; + if (sgs->group_misfit_task) + return group_misfit_task; + return group_other; } @@ -7023,11 +7983,12 @@ group_type group_classify(struct sched_group *group, * @local_group: Does group contain this_cpu. * @sgs: variable to hold the statistics for this group. * @overload: Indicate more than one runnable task for any CPU. + * @overutilized: Indicate overutilization for any CPU. */ static inline void update_sg_lb_stats(struct lb_env *env, struct sched_group *group, int load_idx, int local_group, struct sg_lb_stats *sgs, - bool *overload) + bool *overload, bool *overutilized) { unsigned long load; int i, nr_running; @@ -7061,6 +8022,12 @@ static inline void update_sg_lb_stats(struct lb_env *env, */ if (!nr_running && idle_cpu(i)) sgs->idle_cpus++; + + if (cpu_overutilized(i)) { + *overutilized = true; + if (!sgs->group_misfit_task && rq->misfit_task) + sgs->group_misfit_task = capacity_of(i); + } } /* Adjust by relative CPU capacity of the group */ @@ -7170,7 +8137,7 @@ static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sd struct sched_group *sg = env->sd->groups; struct sg_lb_stats tmp_sgs; int load_idx, prefer_sibling = 0; - bool overload = false; + bool overload = false, overutilized = false; if (child && child->flags & SD_PREFER_SIBLING) prefer_sibling = 1; @@ -7192,7 +8159,7 @@ static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sd } update_sg_lb_stats(env, sg, load_idx, local_group, sgs, - &overload); + &overload, &overutilized); if (local_group) goto next_group; @@ -7230,10 +8197,23 @@ static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sd if (env->sd->flags & SD_NUMA) env->fbq_type = fbq_classify_group(&sds->busiest_stat); + env->src_grp_nr_running = sds->busiest_stat.sum_nr_running; + if (!env->sd->parent) { /* update overload indicator if we are at root domain */ if (env->dst_rq->rd->overload != overload) env->dst_rq->rd->overload = overload; + + /* Update over-utilization (tipping point, U >= 0) indicator */ + if (env->dst_rq->rd->overutilized != overutilized) { + env->dst_rq->rd->overutilized = overutilized; + trace_sched_overutilized(overutilized); + } + } else { + if (!env->dst_rq->rd->overutilized && overutilized) { + env->dst_rq->rd->overutilized = true; + trace_sched_overutilized(true); + } } } @@ -7454,6 +8434,10 @@ static struct sched_group *find_busiest_group(struct lb_env *env) * this level. */ update_sd_lb_stats(env, &sds); + + if (energy_aware() && !env->dst_rq->rd->overutilized) + goto out_balanced; + local = &sds.local_stat; busiest = &sds.busiest_stat; @@ -7633,6 +8617,13 @@ static int need_active_balance(struct lb_env *env) return 1; } + if ((capacity_of(env->src_cpu) < capacity_of(env->dst_cpu)) && + env->src_rq->cfs.h_nr_running == 1 && + cpu_overutilized(env->src_cpu) && + !cpu_overutilized(env->dst_cpu)) { + return 1; + } + return unlikely(sd->nr_balance_failed > sd->cache_nice_tries+2); } @@ -7754,6 +8745,11 @@ static int load_balance(int this_cpu, struct rq *this_rq, * ld_moved - cumulative load moved across iterations */ cur_ld_moved = detach_tasks(&env); + /* + * We want to potentially lower env.src_cpu's OPP. + */ + if (cur_ld_moved) + update_capacity_of(env.src_cpu); /* * We've detached some tasks from busiest_rq. Every @@ -7845,7 +8841,8 @@ static int load_balance(int this_cpu, struct rq *this_rq, * excessive cache_hot migrations and active balances. */ if (idle != CPU_NEWLY_IDLE) - sd->nr_balance_failed++; + if (env.src_grp_nr_running > 1) + sd->nr_balance_failed++; if (need_active_balance(&env)) { raw_spin_lock_irqsave(&busiest->lock, flags); @@ -7975,6 +8972,7 @@ static int idle_balance(struct rq *this_rq) struct sched_domain *sd; int pulled_task = 0; u64 curr_cost = 0; + long removed_util=0; /* * We must set idle_stamp _before_ calling idle_balance(), such that we @@ -7982,8 +8980,9 @@ static int idle_balance(struct rq *this_rq) */ this_rq->idle_stamp = rq_clock(this_rq); - if (this_rq->avg_idle < sysctl_sched_migration_cost || - !this_rq->rd->overload) { + if (!energy_aware() && + (this_rq->avg_idle < sysctl_sched_migration_cost || + !this_rq->rd->overload)) { rcu_read_lock(); sd = rcu_dereference_check_sched_domain(this_rq->sd); if (sd) @@ -7995,6 +8994,17 @@ static int idle_balance(struct rq *this_rq) raw_spin_unlock(&this_rq->lock); + /* + * If removed_util_avg is !0 we most probably migrated some task away + * from this_cpu. In this case we might be willing to trigger an OPP + * update, but we want to do so if we don't find anybody else to pull + * here (we will trigger an OPP update with the pulled task's enqueue + * anyway). + * + * Record removed_util before calling update_blocked_averages, and use + * it below (before returning) to see if an OPP update is required. + */ + removed_util = atomic_long_read(&(this_rq->cfs).removed_util_avg); update_blocked_averages(this_cpu); rcu_read_lock(); for_each_domain(this_cpu, sd) { @@ -8058,6 +9068,13 @@ static int idle_balance(struct rq *this_rq) if (pulled_task) this_rq->idle_stamp = 0; + else if (removed_util) { + /* + * No task pulled and someone has been migrated away. + * Good case to trigger an OPP update. + */ + update_capacity_of(this_cpu); + } return pulled_task; } @@ -8118,6 +9135,10 @@ static int active_load_balance_cpu_stop(void *data) p = detach_one_task(&env); if (p) { schedstat_inc(sd->alb_pushed); + /* + * We want to potentially lower env.src_cpu's OPP. + */ + update_capacity_of(env.src_cpu); /* Active balancing done, reset the failure counter. */ sd->nr_balance_failed = 0; } else { @@ -8488,12 +9509,13 @@ static inline bool nohz_kick_needed(struct rq *rq) if (time_before(now, nohz.next_balance)) return false; - if (rq->nr_running >= 2) + if (rq->nr_running >= 2 && + (!energy_aware() || cpu_overutilized(cpu))) return true; rcu_read_lock(); sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); - if (sds) { + if (sds && !energy_aware()) { /* * XXX: write a coherent comment on why we do this. * See also: http://lkml.kernel.org/r/20111202010832.602203411@sbsiddha-desk.sc.intel.com @@ -8601,6 +9623,16 @@ static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued) if (static_branch_unlikely(&sched_numa_balancing)) task_tick_numa(rq, curr); + +#ifdef CONFIG_SMP + if (!rq->rd->overutilized && cpu_overutilized(task_cpu(curr))) { + rq->rd->overutilized = true; + trace_sched_overutilized(true); + } + + rq->misfit_task = !task_fits_max(curr, rq->cpu); +#endif + } /* diff --git a/kernel/sched/features.h b/kernel/sched/features.h index 69631fa46c2f84..55e46105533297 100644 --- a/kernel/sched/features.h +++ b/kernel/sched/features.h @@ -69,3 +69,12 @@ SCHED_FEAT(RT_RUNTIME_SHARE, true) SCHED_FEAT(LB_MIN, false) SCHED_FEAT(ATTACH_AGE_LOAD, true) +/* + * Energy aware scheduling. Use platform energy model to guide scheduling + * decisions optimizing for energy efficiency. + */ +#ifdef CONFIG_DEFAULT_USE_ENERGY_AWARE +SCHED_FEAT(ENERGY_AWARE, true) +#else +SCHED_FEAT(ENERGY_AWARE, false) +#endif diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index 1d8718d5300d81..cf75f00f703770 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -23,9 +23,10 @@ extern char __cpuidle_text_start[], __cpuidle_text_end[]; * sched_idle_set_state - Record idle state for the current CPU. * @idle_state: State to record. */ -void sched_idle_set_state(struct cpuidle_state *idle_state) +void sched_idle_set_state(struct cpuidle_state *idle_state, int index) { idle_set_state(this_rq(), idle_state); + idle_set_state_idx(this_rq(), index); } static int __read_mostly cpu_idle_force_poll; diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index 2516b8df6dbbd8..f8006b20d3883f 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -8,6 +8,8 @@ #include #include +#include "walt.h" + int sched_rr_timeslice = RR_TIMESLICE; static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun); @@ -897,6 +899,51 @@ static inline int rt_se_prio(struct sched_rt_entity *rt_se) return rt_task_of(rt_se)->prio; } +static void dump_throttled_rt_tasks(struct rt_rq *rt_rq) +{ + struct rt_prio_array *array = &rt_rq->active; + struct sched_rt_entity *rt_se; + char buf[500]; + char *pos = buf; + char *end = buf + sizeof(buf); + int idx; + + pos += snprintf(pos, sizeof(buf), + "sched: RT throttling activated for rt_rq %p (cpu %d)\n", + rt_rq, cpu_of(rq_of_rt_rq(rt_rq))); + + if (bitmap_empty(array->bitmap, MAX_RT_PRIO)) + goto out; + + pos += snprintf(pos, end - pos, "potential CPU hogs:\n"); + idx = sched_find_first_bit(array->bitmap); + while (idx < MAX_RT_PRIO) { + list_for_each_entry(rt_se, array->queue + idx, run_list) { + struct task_struct *p; + + if (!rt_entity_is_task(rt_se)) + continue; + + p = rt_task_of(rt_se); + if (pos < end) + pos += snprintf(pos, end - pos, "\t%s (%d)\n", + p->comm, p->pid); + } + idx = find_next_bit(array->bitmap, MAX_RT_PRIO, idx + 1); + } +out: +#ifdef CONFIG_PANIC_ON_RT_THROTTLING + /* + * Use pr_err() in the BUG() case since printk_sched() will + * not get flushed and deadlock is not a concern. + */ + pr_err("%s", buf); + BUG(); +#else + printk_deferred("%s", buf); +#endif +} + static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq) { u64 runtime = sched_rt_runtime(rt_rq); @@ -920,8 +967,14 @@ static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq) * but accrue some time due to boosting. */ if (likely(rt_b->rt_runtime)) { + static bool once = false; + rt_rq->rt_throttled = 1; - printk_deferred_once("sched: RT throttling activated\n"); + + if (!once) { + once = true; + dump_throttled_rt_tasks(rt_rq); + } } else { /* * In case we did anyway, make it go away, @@ -1322,6 +1375,7 @@ enqueue_task_rt(struct rq *rq, struct task_struct *p, int flags) rt_se->timeout = 0; enqueue_rt_entity(rt_se, flags); + walt_inc_cumulative_runnable_avg(rq, p); if (!task_current(rq, p) && tsk_nr_cpus_allowed(p) > 1) enqueue_pushable_task(rq, p); @@ -1333,6 +1387,7 @@ static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int flags) update_curr_rt(rq); dequeue_rt_entity(rt_se, flags); + walt_dec_cumulative_runnable_avg(rq, p); dequeue_pushable_task(rq, p); } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 055f935d442183..b6839c2fdb50d8 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -438,6 +438,10 @@ struct cfs_rq { struct list_head leaf_cfs_rq_list; struct task_group *tg; /* group that "owns" this runqueue */ +#ifdef CONFIG_SCHED_WALT + u64 cumulative_runnable_avg; +#endif + #ifdef CONFIG_CFS_BANDWIDTH int runtime_enabled; u64 runtime_expires; @@ -539,6 +543,12 @@ struct dl_rq { #ifdef CONFIG_SMP +struct max_cpu_capacity { + raw_spinlock_t lock; + unsigned long val; + int cpu; +}; + /* * We add the notion of a root-domain which will be used to define per-domain * variables. Each exclusive cpuset essentially defines an island domain by @@ -557,6 +567,9 @@ struct root_domain { /* Indicate more than one runnable task for any CPU */ bool overload; + /* Indicate one or more cpus over-utilized (tipping point) */ + bool overutilized; + /* * The bit corresponding to a CPU gets set here if such CPU has more * than one runnable -deadline task (as it is below for RT tasks). @@ -573,7 +586,8 @@ struct root_domain { cpumask_var_t rto_mask; struct cpupri cpupri; - unsigned long max_cpu_capacity; + /* Maximum cpu capacity in the system. */ + struct max_cpu_capacity max_cpu_capacity; }; extern struct root_domain def_root_domain; @@ -602,6 +616,7 @@ struct rq { #endif #define CPU_LOAD_IDX_MAX 5 unsigned long cpu_load[CPU_LOAD_IDX_MAX]; + unsigned int misfit_task; #ifdef CONFIG_NO_HZ_COMMON #ifdef CONFIG_SMP unsigned long last_load_update_tick; @@ -611,6 +626,14 @@ struct rq { #ifdef CONFIG_NO_HZ_FULL unsigned long last_sched_tick; #endif + +#ifdef CONFIG_CPU_QUIET + /* time-based average load */ + u64 nr_last_stamp; + u64 nr_running_integral; + seqcount_t ave_seqcnt; +#endif + /* capture load from *all* tasks on this cpu: */ struct load_weight load; unsigned long nr_load_updates; @@ -672,6 +695,30 @@ struct rq { u64 max_idle_balance_cost; #endif +#ifdef CONFIG_SCHED_WALT + /* + * max_freq = user or thermal defined maximum + * max_possible_freq = maximum supported by hardware + */ + unsigned int cur_freq, max_freq, min_freq, max_possible_freq; + struct cpumask freq_domain_cpumask; + + u64 cumulative_runnable_avg; + int efficiency; /* Differentiate cpus with different IPC capability */ + int load_scale_factor; + int capacity; + int max_possible_capacity; + u64 window_start; + u64 curr_runnable_sum; + u64 prev_runnable_sum; + u64 nt_curr_runnable_sum; + u64 nt_prev_runnable_sum; + u64 cur_irqload; + u64 avg_irqload; + u64 irqload_ts; +#endif /* CONFIG_SCHED_WALT */ + + #ifdef CONFIG_IRQ_TIME_ACCOUNTING u64 prev_irq_time; #endif @@ -719,6 +766,7 @@ struct rq { #ifdef CONFIG_CPU_IDLE /* Must be inspected within a rcu lock section */ struct cpuidle_state *idle_state; + int idle_state_idx; #endif }; @@ -885,6 +933,8 @@ DECLARE_PER_CPU(int, sd_llc_id); DECLARE_PER_CPU(struct sched_domain_shared *, sd_llc_shared); DECLARE_PER_CPU(struct sched_domain *, sd_numa); DECLARE_PER_CPU(struct sched_domain *, sd_asym); +DECLARE_PER_CPU(struct sched_domain *, sd_ea); +DECLARE_PER_CPU(struct sched_domain *, sd_scs); struct sched_group_capacity { atomic_t ref; @@ -892,7 +942,8 @@ struct sched_group_capacity { * CPU capacity of this group, SCHED_CAPACITY_SCALE being max capacity * for a single CPU. */ - unsigned int capacity; + unsigned long capacity; + unsigned long max_capacity; /* Max per-cpu capacity in group */ unsigned long next_update; int imbalance; /* XXX unrelated to capacity but shared group state */ @@ -905,6 +956,7 @@ struct sched_group { unsigned int group_weight; struct sched_group_capacity *sgc; + const struct sched_group_energy const *sge; /* * The CPUs this group covers. @@ -1213,6 +1265,7 @@ extern const u32 sched_prio_to_wmult[40]; #else #define ENQUEUE_MIGRATED 0x00 #endif +#define ENQUEUE_WAKEUP_NEW 0x40 #define RETRY_TASK ((void *)-1UL) @@ -1303,6 +1356,7 @@ extern const struct sched_class idle_sched_class; #ifdef CONFIG_SMP +extern void init_max_cpu_capacity(struct max_cpu_capacity *mcc); extern void update_group_capacity(struct sched_domain *sd, int cpu); extern void trigger_load_balance(struct rq *rq); @@ -1323,6 +1377,17 @@ static inline struct cpuidle_state *idle_get_state(struct rq *rq) SCHED_WARN_ON(!rcu_read_lock_held()); return rq->idle_state; } + +static inline void idle_set_state_idx(struct rq *rq, int idle_state_idx) +{ + rq->idle_state_idx = idle_state_idx; +} + +static inline int idle_get_state_idx(struct rq *rq) +{ + WARN_ON(!rcu_read_lock_held()); + return rq->idle_state_idx; +} #else static inline void idle_set_state(struct rq *rq, struct cpuidle_state *idle_state) @@ -1333,6 +1398,15 @@ static inline struct cpuidle_state *idle_get_state(struct rq *rq) { return NULL; } + +static inline void idle_set_state_idx(struct rq *rq, int idle_state_idx) +{ +} + +static inline int idle_get_state_idx(struct rq *rq) +{ + return -1; +} #endif extern void sysrq_sched_debug_show(void); @@ -1387,7 +1461,7 @@ static inline void sched_update_tick_dependency(struct rq *rq) static inline void sched_update_tick_dependency(struct rq *rq) { } #endif -static inline void add_nr_running(struct rq *rq, unsigned count) +static inline void __add_nr_running(struct rq *rq, unsigned count) { unsigned prev_nr = rq->nr_running; @@ -1403,13 +1477,50 @@ static inline void add_nr_running(struct rq *rq, unsigned count) sched_update_tick_dependency(rq); } -static inline void sub_nr_running(struct rq *rq, unsigned count) +static inline void __sub_nr_running(struct rq *rq, unsigned count) { rq->nr_running -= count; /* Check if we still need preemption */ sched_update_tick_dependency(rq); } +#ifdef CONFIG_CPU_QUIET +#define NR_AVE_SCALE(x) ((x) << FSHIFT) +static inline u64 do_nr_running_integral(struct rq *rq) +{ + s64 nr, deltax; + u64 nr_running_integral = rq->nr_running_integral; + + deltax = rq->clock_task - rq->nr_last_stamp; + nr = NR_AVE_SCALE(rq->nr_running); + + nr_running_integral += nr * deltax; + + return nr_running_integral; +} + +static inline void add_nr_running(struct rq *rq, unsigned count) +{ + write_seqcount_begin(&rq->ave_seqcnt); + rq->nr_running_integral = do_nr_running_integral(rq); + rq->nr_last_stamp = rq->clock_task; + __add_nr_running(rq, count); + write_seqcount_end(&rq->ave_seqcnt); +} + +static inline void sub_nr_running(struct rq *rq, unsigned count) +{ + write_seqcount_begin(&rq->ave_seqcnt); + rq->nr_running_integral = do_nr_running_integral(rq); + rq->nr_last_stamp = rq->clock_task; + __sub_nr_running(rq, count); + write_seqcount_end(&rq->ave_seqcnt); +} +#else +#define add_nr_running __add_nr_running +#define sub_nr_running __sub_nr_running +#endif + static inline void rq_last_tick_reset(struct rq *rq) { #ifdef CONFIG_NO_HZ_FULL @@ -1482,10 +1593,146 @@ unsigned long arch_scale_cpu_capacity(struct sched_domain *sd, int cpu) } #endif +#ifdef CONFIG_SMP +static inline unsigned long capacity_of(int cpu) +{ + return cpu_rq(cpu)->cpu_capacity; +} + +static inline unsigned long capacity_orig_of(int cpu) +{ + return cpu_rq(cpu)->cpu_capacity_orig; +} + +extern unsigned int sysctl_sched_use_walt_cpu_util; +extern unsigned int walt_ravg_window; +extern unsigned int walt_disabled; + +/* + * cpu_util returns the amount of capacity of a CPU that is used by CFS + * tasks. The unit of the return value must be the one of capacity so we can + * compare the utilization with the capacity of the CPU that is available for + * CFS task (ie cpu_capacity). + * + * cfs_rq.avg.util_avg is the sum of running time of runnable tasks plus the + * recent utilization of currently non-runnable tasks on a CPU. It represents + * the amount of utilization of a CPU in the range [0..capacity_orig] where + * capacity_orig is the cpu_capacity available at the highest frequency + * (arch_scale_freq_capacity()). + * The utilization of a CPU converges towards a sum equal to or less than the + * current capacity (capacity_curr <= capacity_orig) of the CPU because it is + * the running time on this CPU scaled by capacity_curr. + * + * Nevertheless, cfs_rq.avg.util_avg can be higher than capacity_curr or even + * higher than capacity_orig because of unfortunate rounding in + * cfs.avg.util_avg or just after migrating tasks and new task wakeups until + * the average stabilizes with the new running time. We need to check that the + * utilization stays within the range of [0..capacity_orig] and cap it if + * necessary. Without utilization capping, a group could be seen as overloaded + * (CPU0 utilization at 121% + CPU1 utilization at 80%) whereas CPU1 has 20% of + * available capacity. We allow utilization to overshoot capacity_curr (but not + * capacity_orig) as it useful for predicting the capacity required after task + * migrations (scheduler-driven DVFS). + */ +static inline unsigned long __cpu_util(int cpu, int delta) +{ + unsigned long util = cpu_rq(cpu)->cfs.avg.util_avg; + unsigned long capacity = capacity_orig_of(cpu); + +#ifdef CONFIG_SCHED_WALT + if (!walt_disabled && sysctl_sched_use_walt_cpu_util) { + util = cpu_rq(cpu)->prev_runnable_sum << SCHED_CAPACITY_SHIFT; + util = div_u64(util, walt_ravg_window); + } +#endif + delta += util; + if (delta < 0) + return 0; + + return (delta >= capacity) ? capacity : delta; +} + +static inline unsigned long cpu_util(int cpu) +{ + return __cpu_util(cpu, 0); +} + +#endif + +#ifdef CONFIG_CPU_FREQ_GOV_SCHED +#define capacity_max SCHED_CAPACITY_SCALE +extern unsigned int capacity_margin; +extern struct static_key __sched_freq; + +static inline bool sched_freq(void) +{ + return static_key_false(&__sched_freq); +} + +DECLARE_PER_CPU(struct sched_capacity_reqs, cpu_sched_capacity_reqs); +void update_cpu_capacity_request(int cpu, bool request); + +static inline void set_cfs_cpu_capacity(int cpu, bool request, + unsigned long capacity) +{ + struct sched_capacity_reqs *scr = &per_cpu(cpu_sched_capacity_reqs, cpu); + +#ifdef CONFIG_SCHED_WALT + if (!walt_disabled && sysctl_sched_use_walt_cpu_util) { + int rtdl = scr->rt + scr->dl; + /* + * WALT tracks the utilization of a CPU considering the load + * generated by all the scheduling classes. + * Since the following call to: + * update_cpu_capacity + * is already adding the RT and DL utilizations let's remove + * these contributions from the WALT signal. + */ + if (capacity > rtdl) + capacity -= rtdl; + else + capacity = 0; + } +#endif + if (scr->cfs != capacity) { + scr->cfs = capacity; + update_cpu_capacity_request(cpu, request); + } +} + +static inline void set_rt_cpu_capacity(int cpu, bool request, + unsigned long capacity) +{ + if (per_cpu(cpu_sched_capacity_reqs, cpu).rt != capacity) { + per_cpu(cpu_sched_capacity_reqs, cpu).rt = capacity; + update_cpu_capacity_request(cpu, request); + } +} + +static inline void set_dl_cpu_capacity(int cpu, bool request, + unsigned long capacity) +{ + if (per_cpu(cpu_sched_capacity_reqs, cpu).dl != capacity) { + per_cpu(cpu_sched_capacity_reqs, cpu).dl = capacity; + update_cpu_capacity_request(cpu, request); + } +} +#else +static inline bool sched_freq(void) { return false; } +static inline void set_cfs_cpu_capacity(int cpu, bool request, + unsigned long capacity) +{ } +static inline void set_rt_cpu_capacity(int cpu, bool request, + unsigned long capacity) +{ } +static inline void set_dl_cpu_capacity(int cpu, bool request, + unsigned long capacity) +{ } +#endif + static inline void sched_rt_avg_update(struct rq *rq, u64 rt_delta) { rq->rt_avg += rt_delta * arch_scale_freq_capacity(NULL, cpu_of(rq)); - sched_avg_update(rq); } #else static inline void sched_rt_avg_update(struct rq *rq, u64 rt_delta) { } @@ -1520,6 +1767,9 @@ task_rq_unlock(struct rq *rq, struct task_struct *p, struct rq_flags *rf) raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags); } +extern struct rq *lock_rq_of(struct task_struct *p, struct rq_flags *flags); +extern void unlock_rq_of(struct rq *rq, struct task_struct *p, struct rq_flags *flags); + #ifdef CONFIG_SMP #ifdef CONFIG_PREEMPT @@ -1592,7 +1842,8 @@ static inline int double_lock_balance(struct rq *this_rq, struct rq *busiest) static inline void double_unlock_balance(struct rq *this_rq, struct rq *busiest) __releases(busiest->lock) { - raw_spin_unlock(&busiest->lock); + if (this_rq != busiest) + raw_spin_unlock(&busiest->lock); lock_set_subclass(&this_rq->lock.dep_map, 0, _RET_IP_); } diff --git a/kernel/sched/stop_task.c b/kernel/sched/stop_task.c index 604297a08b3ae3..836a3894cf577d 100644 --- a/kernel/sched/stop_task.c +++ b/kernel/sched/stop_task.c @@ -1,4 +1,5 @@ #include "sched.h" +#include "walt.h" /* * stop-task scheduling class. @@ -42,12 +43,14 @@ static void enqueue_task_stop(struct rq *rq, struct task_struct *p, int flags) { add_nr_running(rq, 1); + walt_inc_cumulative_runnable_avg(rq, p); } static void dequeue_task_stop(struct rq *rq, struct task_struct *p, int flags) { sub_nr_running(rq, 1); + walt_dec_cumulative_runnable_avg(rq, p); } static void yield_task_stop(struct rq *rq) diff --git a/kernel/sched/tune.c b/kernel/sched/tune.c new file mode 100644 index 00000000000000..d1fe64c91e5256 --- /dev/null +++ b/kernel/sched/tune.c @@ -0,0 +1,945 @@ +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "sched.h" +#include "tune.h" + +#ifdef CONFIG_CGROUP_SCHEDTUNE +static bool schedtune_initialized = false; +#endif + +unsigned int sysctl_sched_cfs_boost __read_mostly; + +extern struct target_nrg schedtune_target_nrg; + +/* Performance Boost region (B) threshold params */ +static int perf_boost_idx; + +/* Performance Constraint region (C) threshold params */ +static int perf_constrain_idx; + +/** + * Performance-Energy (P-E) Space thresholds constants + */ +struct threshold_params { + int nrg_gain; + int cap_gain; +}; + +/* + * System specific P-E space thresholds constants + */ +static struct threshold_params +threshold_gains[] = { + { 0, 5 }, /* < 10% */ + { 1, 5 }, /* < 20% */ + { 2, 5 }, /* < 30% */ + { 3, 5 }, /* < 40% */ + { 4, 5 }, /* < 50% */ + { 5, 4 }, /* < 60% */ + { 5, 3 }, /* < 70% */ + { 5, 2 }, /* < 80% */ + { 5, 1 }, /* < 90% */ + { 5, 0 } /* <= 100% */ +}; + +static int +__schedtune_accept_deltas(int nrg_delta, int cap_delta, + int perf_boost_idx, int perf_constrain_idx) +{ + int payoff = -INT_MAX; + int gain_idx = -1; + + /* Performance Boost (B) region */ + if (nrg_delta >= 0 && cap_delta > 0) + gain_idx = perf_boost_idx; + /* Performance Constraint (C) region */ + else if (nrg_delta < 0 && cap_delta <= 0) + gain_idx = perf_constrain_idx; + + /* Default: reject schedule candidate */ + if (gain_idx == -1) + return payoff; + + /* + * Evaluate "Performance Boost" vs "Energy Increase" + * + * - Performance Boost (B) region + * + * Condition: nrg_delta > 0 && cap_delta > 0 + * Payoff criteria: + * cap_gain / nrg_gain < cap_delta / nrg_delta = + * cap_gain * nrg_delta < cap_delta * nrg_gain + * Note that since both nrg_gain and nrg_delta are positive, the + * inequality does not change. Thus: + * + * payoff = (cap_delta * nrg_gain) - (cap_gain * nrg_delta) + * + * - Performance Constraint (C) region + * + * Condition: nrg_delta < 0 && cap_delta < 0 + * payoff criteria: + * cap_gain / nrg_gain > cap_delta / nrg_delta = + * cap_gain * nrg_delta < cap_delta * nrg_gain + * Note that since nrg_gain > 0 while nrg_delta < 0, the + * inequality change. Thus: + * + * payoff = (cap_delta * nrg_gain) - (cap_gain * nrg_delta) + * + * This means that, in case of same positive defined {cap,nrg}_gain + * for both the B and C regions, we can use the same payoff formula + * where a positive value represents the accept condition. + */ + payoff = cap_delta * threshold_gains[gain_idx].nrg_gain; + payoff -= nrg_delta * threshold_gains[gain_idx].cap_gain; + + return payoff; +} + +#ifdef CONFIG_CGROUP_SCHEDTUNE + +/* + * EAS scheduler tunables for task groups. + */ + +/* SchdTune tunables for a group of tasks */ +struct schedtune { + /* SchedTune CGroup subsystem */ + struct cgroup_subsys_state css; + + /* Boost group allocated ID */ + int idx; + + /* Boost value for tasks on that SchedTune CGroup */ + int boost; + + /* Performance Boost (B) region threshold params */ + int perf_boost_idx; + + /* Performance Constraint (C) region threshold params */ + int perf_constrain_idx; + + /* Hint to bias scheduling of tasks on that SchedTune CGroup + * towards idle CPUs */ + int prefer_idle; +}; + +static inline struct schedtune *css_st(struct cgroup_subsys_state *css) +{ + return css ? container_of(css, struct schedtune, css) : NULL; +} + +static inline struct schedtune *task_schedtune(struct task_struct *tsk) +{ + return css_st(task_css(tsk, schedtune_cgrp_id)); +} + +static inline struct schedtune *parent_st(struct schedtune *st) +{ + return css_st(st->css.parent); +} + +/* + * SchedTune root control group + * The root control group is used to defined a system-wide boosting tuning, + * which is applied to all tasks in the system. + * Task specific boost tuning could be specified by creating and + * configuring a child control group under the root one. + * By default, system-wide boosting is disabled, i.e. no boosting is applied + * to tasks which are not into a child control group. + */ +static struct schedtune +root_schedtune = { + .boost = 0, + .perf_boost_idx = 0, + .perf_constrain_idx = 0, + .prefer_idle = 0, +}; + +int +schedtune_accept_deltas(int nrg_delta, int cap_delta, + struct task_struct *task) +{ + struct schedtune *ct; + int perf_boost_idx; + int perf_constrain_idx; + + /* Optimal (O) region */ + if (nrg_delta < 0 && cap_delta > 0) { + trace_sched_tune_filter(nrg_delta, cap_delta, 0, 0, 1, 0); + return INT_MAX; + } + + /* Suboptimal (S) region */ + if (nrg_delta > 0 && cap_delta < 0) { + trace_sched_tune_filter(nrg_delta, cap_delta, 0, 0, -1, 5); + return -INT_MAX; + } + + /* Get task specific perf Boost/Constraints indexes */ + rcu_read_lock(); + ct = task_schedtune(task); + perf_boost_idx = ct->perf_boost_idx; + perf_constrain_idx = ct->perf_constrain_idx; + rcu_read_unlock(); + + return __schedtune_accept_deltas(nrg_delta, cap_delta, + perf_boost_idx, perf_constrain_idx); +} + +/* + * Maximum number of boost groups to support + * When per-task boosting is used we still allow only limited number of + * boost groups for two main reasons: + * 1. on a real system we usually have only few classes of workloads which + * make sense to boost with different values (e.g. background vs foreground + * tasks, interactive vs low-priority tasks) + * 2. a limited number allows for a simpler and more memory/time efficient + * implementation especially for the computation of the per-CPU boost + * value + */ +#define BOOSTGROUPS_COUNT 4 + +/* Array of configured boostgroups */ +static struct schedtune *allocated_group[BOOSTGROUPS_COUNT] = { + &root_schedtune, + NULL, +}; + +/* SchedTune boost groups + * Keep track of all the boost groups which impact on CPU, for example when a + * CPU has two RUNNABLE tasks belonging to two different boost groups and thus + * likely with different boost values. + * Since on each system we expect only a limited number of boost groups, here + * we use a simple array to keep track of the metrics required to compute the + * maximum per-CPU boosting value. + */ +struct boost_groups { + /* Maximum boost value for all RUNNABLE tasks on a CPU */ + bool idle; + int boost_max; + struct { + /* The boost for tasks on that boost group */ + int boost; + /* Count of RUNNABLE tasks on that boost group */ + unsigned tasks; + } group[BOOSTGROUPS_COUNT]; + /* CPU's boost group locking */ + raw_spinlock_t lock; +}; + +/* Boost groups affecting each CPU in the system */ +DEFINE_PER_CPU(struct boost_groups, cpu_boost_groups); + +static void +schedtune_cpu_update(int cpu) +{ + struct boost_groups *bg; + int boost_max; + int idx; + + bg = &per_cpu(cpu_boost_groups, cpu); + + /* The root boost group is always active */ + boost_max = bg->group[0].boost; + for (idx = 1; idx < BOOSTGROUPS_COUNT; ++idx) { + /* + * A boost group affects a CPU only if it has + * RUNNABLE tasks on that CPU + */ + if (bg->group[idx].tasks == 0) + continue; + + boost_max = max(boost_max, bg->group[idx].boost); + } + /* Ensures boost_max is non-negative when all cgroup boost values + * are neagtive. Avoids under-accounting of cpu capacity which may cause + * task stacking and frequency spikes.*/ + boost_max = max(boost_max, 0); + bg->boost_max = boost_max; +} + +static int +schedtune_boostgroup_update(int idx, int boost) +{ + struct boost_groups *bg; + int cur_boost_max; + int old_boost; + int cpu; + + /* Update per CPU boost groups */ + for_each_possible_cpu(cpu) { + bg = &per_cpu(cpu_boost_groups, cpu); + + /* + * Keep track of current boost values to compute the per CPU + * maximum only when it has been affected by the new value of + * the updated boost group + */ + cur_boost_max = bg->boost_max; + old_boost = bg->group[idx].boost; + + /* Update the boost value of this boost group */ + bg->group[idx].boost = boost; + + /* Check if this update increase current max */ + if (boost > cur_boost_max && bg->group[idx].tasks) { + bg->boost_max = boost; + trace_sched_tune_boostgroup_update(cpu, 1, bg->boost_max); + continue; + } + + /* Check if this update has decreased current max */ + if (cur_boost_max == old_boost && old_boost > boost) { + schedtune_cpu_update(cpu); + trace_sched_tune_boostgroup_update(cpu, -1, bg->boost_max); + continue; + } + + trace_sched_tune_boostgroup_update(cpu, 0, bg->boost_max); + } + + return 0; +} + +#define ENQUEUE_TASK 1 +#define DEQUEUE_TASK -1 + +static inline void +schedtune_tasks_update(struct task_struct *p, int cpu, int idx, int task_count) +{ + struct boost_groups *bg = &per_cpu(cpu_boost_groups, cpu); + int tasks = bg->group[idx].tasks + task_count; + + /* Update boosted tasks count while avoiding to make it negative */ + bg->group[idx].tasks = max(0, tasks); + + trace_sched_tune_tasks_update(p, cpu, tasks, idx, + bg->group[idx].boost, bg->boost_max); + + /* Boost group activation or deactivation on that RQ */ + if (tasks == 1 || tasks == 0) + schedtune_cpu_update(cpu); +} + +/* + * NOTE: This function must be called while holding the lock on the CPU RQ + */ +void schedtune_enqueue_task(struct task_struct *p, int cpu) +{ + struct boost_groups *bg = &per_cpu(cpu_boost_groups, cpu); + unsigned long irq_flags; + struct schedtune *st; + int idx; + + if (!unlikely(schedtune_initialized)) + return; + + /* + * When a task is marked PF_EXITING by do_exit() it's going to be + * dequeued and enqueued multiple times in the exit path. + * Thus we avoid any further update, since we do not want to change + * CPU boosting while the task is exiting. + */ + if (p->flags & PF_EXITING) + return; + + /* + * Boost group accouting is protected by a per-cpu lock and requires + * interrupt to be disabled to avoid race conditions for example on + * do_exit()::cgroup_exit() and task migration. + */ + raw_spin_lock_irqsave(&bg->lock, irq_flags); + rcu_read_lock(); + + st = task_schedtune(p); + idx = st->idx; + + schedtune_tasks_update(p, cpu, idx, ENQUEUE_TASK); + + rcu_read_unlock(); + raw_spin_unlock_irqrestore(&bg->lock, irq_flags); +} + +int schedtune_can_attach(struct cgroup_taskset *tset) +{ + struct task_struct *task; + struct cgroup_subsys_state *css; + struct boost_groups *bg; + struct rq_flags irq_flags; + unsigned int cpu; + struct rq *rq; + int src_bg; /* Source boost group index */ + int dst_bg; /* Destination boost group index */ + int tasks; + + if (!unlikely(schedtune_initialized)) + return 0; + + + cgroup_taskset_for_each(task, css, tset) { + + /* + * Lock the CPU's RQ the task is enqueued to avoid race + * conditions with migration code while the task is being + * accounted + */ + rq = lock_rq_of(task, &irq_flags); + + if (!task->on_rq) { + unlock_rq_of(rq, task, &irq_flags); + continue; + } + + /* + * Boost group accouting is protected by a per-cpu lock and requires + * interrupt to be disabled to avoid race conditions on... + */ + cpu = cpu_of(rq); + bg = &per_cpu(cpu_boost_groups, cpu); + raw_spin_lock(&bg->lock); + + dst_bg = css_st(css)->idx; + src_bg = task_schedtune(task)->idx; + + /* + * Current task is not changing boostgroup, which can + * happen when the new hierarchy is in use. + */ + if (unlikely(dst_bg == src_bg)) { + raw_spin_unlock(&bg->lock); + unlock_rq_of(rq, task, &irq_flags); + continue; + } + + /* + * This is the case of a RUNNABLE task which is switching its + * current boost group. + */ + + /* Move task from src to dst boost group */ + tasks = bg->group[src_bg].tasks - 1; + bg->group[src_bg].tasks = max(0, tasks); + bg->group[dst_bg].tasks += 1; + + raw_spin_unlock(&bg->lock); + unlock_rq_of(rq, task, &irq_flags); + + /* Update CPU boost group */ + if (bg->group[src_bg].tasks == 0 || bg->group[dst_bg].tasks == 1) + schedtune_cpu_update(task_cpu(task)); + + } + + return 0; +} + +void schedtune_cancel_attach(struct cgroup_taskset *tset) +{ + /* This can happen only if SchedTune controller is mounted with + * other hierarchies ane one of them fails. Since usually SchedTune is + * mouted on its own hierarcy, for the time being we do not implement + * a proper rollback mechanism */ + WARN(1, "SchedTune cancel attach not implemented"); +} + +/* + * NOTE: This function must be called while holding the lock on the CPU RQ + */ +void schedtune_dequeue_task(struct task_struct *p, int cpu) +{ + struct boost_groups *bg = &per_cpu(cpu_boost_groups, cpu); + unsigned long irq_flags; + struct schedtune *st; + int idx; + + if (!unlikely(schedtune_initialized)) + return; + + /* + * When a task is marked PF_EXITING by do_exit() it's going to be + * dequeued and enqueued multiple times in the exit path. + * Thus we avoid any further update, since we do not want to change + * CPU boosting while the task is exiting. + * The last dequeue is already enforce by the do_exit() code path + * via schedtune_exit_task(). + */ + if (p->flags & PF_EXITING) + return; + + /* + * Boost group accouting is protected by a per-cpu lock and requires + * interrupt to be disabled to avoid race conditions on... + */ + raw_spin_lock_irqsave(&bg->lock, irq_flags); + rcu_read_lock(); + + st = task_schedtune(p); + idx = st->idx; + + schedtune_tasks_update(p, cpu, idx, DEQUEUE_TASK); + + rcu_read_unlock(); + raw_spin_unlock_irqrestore(&bg->lock, irq_flags); +} + +void schedtune_exit_task(struct task_struct *tsk) +{ + struct schedtune *st; + struct rq_flags irq_flags; + unsigned int cpu; + struct rq *rq; + int idx; + + if (!unlikely(schedtune_initialized)) + return; + + rq = lock_rq_of(tsk, &irq_flags); + rcu_read_lock(); + + cpu = cpu_of(rq); + st = task_schedtune(tsk); + idx = st->idx; + schedtune_tasks_update(tsk, cpu, idx, DEQUEUE_TASK); + + rcu_read_unlock(); + unlock_rq_of(rq, tsk, &irq_flags); +} + +int schedtune_cpu_boost(int cpu) +{ + struct boost_groups *bg; + + bg = &per_cpu(cpu_boost_groups, cpu); + return bg->boost_max; +} + +int schedtune_task_boost(struct task_struct *p) +{ + struct schedtune *st; + int task_boost; + + /* Get task boost value */ + rcu_read_lock(); + st = task_schedtune(p); + task_boost = st->boost; + rcu_read_unlock(); + + return task_boost; +} + +int schedtune_prefer_idle(struct task_struct *p) +{ + struct schedtune *st; + int prefer_idle; + + /* Get prefer_idle value */ + rcu_read_lock(); + st = task_schedtune(p); + prefer_idle = st->prefer_idle; + rcu_read_unlock(); + + return prefer_idle; +} + +static u64 +prefer_idle_read(struct cgroup_subsys_state *css, struct cftype *cft) +{ + struct schedtune *st = css_st(css); + + return st->prefer_idle; +} + +static int +prefer_idle_write(struct cgroup_subsys_state *css, struct cftype *cft, + u64 prefer_idle) +{ + struct schedtune *st = css_st(css); + st->prefer_idle = prefer_idle; + + return 0; +} + +static s64 +boost_read(struct cgroup_subsys_state *css, struct cftype *cft) +{ + struct schedtune *st = css_st(css); + + return st->boost; +} + +static int +boost_write(struct cgroup_subsys_state *css, struct cftype *cft, + s64 boost) +{ + struct schedtune *st = css_st(css); + unsigned threshold_idx; + int boost_pct; + + if (boost < -100 || boost > 100) + return -EINVAL; + boost_pct = boost; + + /* + * Update threshold params for Performance Boost (B) + * and Performance Constraint (C) regions. + * The current implementatio uses the same cuts for both + * B and C regions. + */ + threshold_idx = clamp(boost_pct, 0, 99) / 10; + st->perf_boost_idx = threshold_idx; + st->perf_constrain_idx = threshold_idx; + + st->boost = boost; + if (css == &root_schedtune.css) { + sysctl_sched_cfs_boost = boost; + perf_boost_idx = threshold_idx; + perf_constrain_idx = threshold_idx; + } + + /* Update CPU boost */ + schedtune_boostgroup_update(st->idx, st->boost); + + trace_sched_tune_config(st->boost); + + return 0; +} + +static struct cftype files[] = { + { + .name = "boost", + .read_s64 = boost_read, + .write_s64 = boost_write, + }, + { + .name = "prefer_idle", + .read_u64 = prefer_idle_read, + .write_u64 = prefer_idle_write, + }, + { } /* terminate */ +}; + +static int +schedtune_boostgroup_init(struct schedtune *st) +{ + struct boost_groups *bg; + int cpu; + + /* Keep track of allocated boost groups */ + allocated_group[st->idx] = st; + + /* Initialize the per CPU boost groups */ + for_each_possible_cpu(cpu) { + bg = &per_cpu(cpu_boost_groups, cpu); + bg->group[st->idx].boost = 0; + bg->group[st->idx].tasks = 0; + } + + return 0; +} + +static struct cgroup_subsys_state * +schedtune_css_alloc(struct cgroup_subsys_state *parent_css) +{ + struct schedtune *st; + int idx; + + if (!parent_css) + return &root_schedtune.css; + + /* Allow only single level hierachies */ + if (parent_css != &root_schedtune.css) { + pr_err("Nested SchedTune boosting groups not allowed\n"); + return ERR_PTR(-ENOMEM); + } + + /* Allow only a limited number of boosting groups */ + for (idx = 1; idx < BOOSTGROUPS_COUNT; ++idx) + if (!allocated_group[idx]) + break; + if (idx == BOOSTGROUPS_COUNT) { + pr_err("Trying to create more than %d SchedTune boosting groups\n", + BOOSTGROUPS_COUNT); + return ERR_PTR(-ENOSPC); + } + + st = kzalloc(sizeof(*st), GFP_KERNEL); + if (!st) + goto out; + + /* Initialize per CPUs boost group support */ + st->idx = idx; + if (schedtune_boostgroup_init(st)) + goto release; + + return &st->css; + +release: + kfree(st); +out: + return ERR_PTR(-ENOMEM); +} + +static void +schedtune_boostgroup_release(struct schedtune *st) +{ + /* Reset this boost group */ + schedtune_boostgroup_update(st->idx, 0); + + /* Keep track of allocated boost groups */ + allocated_group[st->idx] = NULL; +} + +static void +schedtune_css_free(struct cgroup_subsys_state *css) +{ + struct schedtune *st = css_st(css); + + schedtune_boostgroup_release(st); + kfree(st); +} + +struct cgroup_subsys schedtune_cgrp_subsys = { + .css_alloc = schedtune_css_alloc, + .css_free = schedtune_css_free, + .can_attach = schedtune_can_attach, + .cancel_attach = schedtune_cancel_attach, + .legacy_cftypes = files, + .early_init = 1, +}; + +static inline void +schedtune_init_cgroups(void) +{ + struct boost_groups *bg; + int cpu; + + /* Initialize the per CPU boost groups */ + for_each_possible_cpu(cpu) { + bg = &per_cpu(cpu_boost_groups, cpu); + memset(bg, 0, sizeof(struct boost_groups)); + raw_spin_lock_init(&bg->lock); + } + + pr_info("schedtune: configured to support %d boost groups\n", + BOOSTGROUPS_COUNT); + + schedtune_initialized = true; +} + +#else /* CONFIG_CGROUP_SCHEDTUNE */ + +int +schedtune_accept_deltas(int nrg_delta, int cap_delta, + struct task_struct *task) +{ + /* Optimal (O) region */ + if (nrg_delta < 0 && cap_delta > 0) { + trace_sched_tune_filter(nrg_delta, cap_delta, 0, 0, 1, 0); + return INT_MAX; + } + + /* Suboptimal (S) region */ + if (nrg_delta > 0 && cap_delta < 0) { + trace_sched_tune_filter(nrg_delta, cap_delta, 0, 0, -1, 5); + return -INT_MAX; + } + + return __schedtune_accept_deltas(nrg_delta, cap_delta, + perf_boost_idx, perf_constrain_idx); +} + +#endif /* CONFIG_CGROUP_SCHEDTUNE */ + +int +sysctl_sched_cfs_boost_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos) +{ + int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + unsigned threshold_idx; + int boost_pct; + + if (ret || !write) + return ret; + + if (sysctl_sched_cfs_boost < -100 || sysctl_sched_cfs_boost > 100) + return -EINVAL; + boost_pct = sysctl_sched_cfs_boost; + + /* + * Update threshold params for Performance Boost (B) + * and Performance Constraint (C) regions. + * The current implementatio uses the same cuts for both + * B and C regions. + */ + threshold_idx = clamp(boost_pct, 0, 99) / 10; + perf_boost_idx = threshold_idx; + perf_constrain_idx = threshold_idx; + + return 0; +} + +#ifdef CONFIG_SCHED_DEBUG +static void +schedtune_test_nrg(unsigned long delta_pwr) +{ + unsigned long test_delta_pwr; + unsigned long test_norm_pwr; + int idx; + + /* + * Check normalization constants using some constant system + * energy values + */ + pr_info("schedtune: verify normalization constants...\n"); + for (idx = 0; idx < 6; ++idx) { + test_delta_pwr = delta_pwr >> idx; + + /* Normalize on max energy for target platform */ + test_norm_pwr = reciprocal_divide( + test_delta_pwr << SCHED_CAPACITY_SHIFT, + schedtune_target_nrg.rdiv); + + pr_info("schedtune: max_pwr/2^%d: %4lu => norm_pwr: %5lu\n", + idx, test_delta_pwr, test_norm_pwr); + } +} +#else +#define schedtune_test_nrg(delta_pwr) +#endif + +/* + * Compute the min/max power consumption of a cluster and all its CPUs + */ +static void +schedtune_add_cluster_nrg( + struct sched_domain *sd, + struct sched_group *sg, + struct target_nrg *ste) +{ + struct sched_domain *sd2; + struct sched_group *sg2; + + struct cpumask *cluster_cpus; + char str[32]; + + unsigned long min_pwr; + unsigned long max_pwr; + int cpu; + + /* Get Cluster energy using EM data for the first CPU */ + cluster_cpus = sched_group_cpus(sg); + snprintf(str, 32, "CLUSTER[%*pbl]", + cpumask_pr_args(cluster_cpus)); + + min_pwr = sg->sge->idle_states[sg->sge->nr_idle_states - 1].power; + max_pwr = sg->sge->cap_states[sg->sge->nr_cap_states - 1].power; + pr_info("schedtune: %-17s min_pwr: %5lu max_pwr: %5lu\n", + str, min_pwr, max_pwr); + + /* + * Keep track of this cluster's energy in the computation of the + * overall system energy + */ + ste->min_power += min_pwr; + ste->max_power += max_pwr; + + /* Get CPU energy using EM data for each CPU in the group */ + for_each_cpu(cpu, cluster_cpus) { + /* Get a SD view for the specific CPU */ + for_each_domain(cpu, sd2) { + /* Get the CPU group */ + sg2 = sd2->groups; + min_pwr = sg2->sge->idle_states[sg2->sge->nr_idle_states - 1].power; + max_pwr = sg2->sge->cap_states[sg2->sge->nr_cap_states - 1].power; + + ste->min_power += min_pwr; + ste->max_power += max_pwr; + + snprintf(str, 32, "CPU[%d]", cpu); + pr_info("schedtune: %-17s min_pwr: %5lu max_pwr: %5lu\n", + str, min_pwr, max_pwr); + + /* + * Assume we have EM data only at the CPU and + * the upper CLUSTER level + */ + BUG_ON(!cpumask_equal( + sched_group_cpus(sg), + sched_group_cpus(sd2->parent->groups) + )); + break; + } + } +} + +/* + * Initialize the constants required to compute normalized energy. + * The values of these constants depends on the EM data for the specific + * target system and topology. + * Thus, this function is expected to be called by the code + * that bind the EM to the topology information. + */ +static int +schedtune_init(void) +{ + struct target_nrg *ste = &schedtune_target_nrg; + unsigned long delta_pwr = 0; + struct sched_domain *sd; + struct sched_group *sg; + + pr_info("schedtune: init normalization constants...\n"); + ste->max_power = 0; + ste->min_power = 0; + + rcu_read_lock(); + + /* + * When EAS is in use, we always have a pointer to the highest SD + * which provides EM data. + */ + sd = rcu_dereference(per_cpu(sd_ea, cpumask_first(cpu_online_mask))); + if (!sd) { + pr_info("schedtune: no energy model data\n"); + goto nodata; + } + + sg = sd->groups; + do { + schedtune_add_cluster_nrg(sd, sg, ste); + } while (sg = sg->next, sg != sd->groups); + + rcu_read_unlock(); + + pr_info("schedtune: %-17s min_pwr: %5lu max_pwr: %5lu\n", + "SYSTEM", ste->min_power, ste->max_power); + + /* Compute normalization constants */ + delta_pwr = ste->max_power - ste->min_power; + ste->rdiv = reciprocal_value(delta_pwr); + pr_info("schedtune: using normalization constants mul: %u sh1: %u sh2: %u\n", + ste->rdiv.m, ste->rdiv.sh1, ste->rdiv.sh2); + + schedtune_test_nrg(delta_pwr); + +#ifdef CONFIG_CGROUP_SCHEDTUNE + schedtune_init_cgroups(); +#else + pr_info("schedtune: configured to support global boosting only\n"); +#endif + + return 0; + +nodata: + rcu_read_unlock(); + return -EINVAL; +} +postcore_initcall(schedtune_init); diff --git a/kernel/sched/tune.h b/kernel/sched/tune.h new file mode 100644 index 00000000000000..4f6441771e4c9e --- /dev/null +++ b/kernel/sched/tune.h @@ -0,0 +1,55 @@ + +#ifdef CONFIG_SCHED_TUNE + +#include + +/* + * System energy normalization constants + */ +struct target_nrg { + unsigned long min_power; + unsigned long max_power; + struct reciprocal_value rdiv; +}; + +#ifdef CONFIG_CGROUP_SCHEDTUNE + +int schedtune_cpu_boost(int cpu); +int schedtune_task_boost(struct task_struct *tsk); + +int schedtune_prefer_idle(struct task_struct *tsk); + +void schedtune_exit_task(struct task_struct *tsk); + +void schedtune_enqueue_task(struct task_struct *p, int cpu); +void schedtune_dequeue_task(struct task_struct *p, int cpu); + +#else /* CONFIG_CGROUP_SCHEDTUNE */ + +#define schedtune_cpu_boost(cpu) get_sysctl_sched_cfs_boost() +#define schedtune_task_boost(tsk) get_sysctl_sched_cfs_boost() + +#define schedtune_exit_task(task) do { } while (0) + +#define schedtune_enqueue_task(task, cpu) do { } while (0) +#define schedtune_dequeue_task(task, cpu) do { } while (0) + +#endif /* CONFIG_CGROUP_SCHEDTUNE */ + +int schedtune_normalize_energy(int energy); +int schedtune_accept_deltas(int nrg_delta, int cap_delta, + struct task_struct *task); + +#else /* CONFIG_SCHED_TUNE */ + +#define schedtune_cpu_boost(cpu) 0 +#define schedtune_task_boost(tsk) 0 + +#define schedtune_exit_task(task) do { } while (0) + +#define schedtune_enqueue_task(task, cpu) do { } while (0) +#define schedtune_dequeue_task(task, cpu) do { } while (0) + +#define schedtune_accept_deltas(nrg_delta, cap_delta, task) nrg_delta + +#endif /* CONFIG_SCHED_TUNE */ diff --git a/kernel/sched/walt.c b/kernel/sched/walt.c new file mode 100644 index 00000000000000..7f686ffccbf072 --- /dev/null +++ b/kernel/sched/walt.c @@ -0,0 +1,1168 @@ +/* + * Copyright (c) 2016, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * + * Window Assisted Load Tracking (WALT) implementation credits: + * Srivatsa Vaddagiri, Steve Muckle, Syed Rameez Mustafa, Joonwoo Park, + * Pavan Kumar Kondeti, Olav Haugan + * + * 2016-03-06: Integration with EAS/refactoring by Vikram Mulukutla + * and Todd Kjos + */ + +#include +#include +#include +#include "sched.h" +#include "walt.h" + +#define WINDOW_STATS_RECENT 0 +#define WINDOW_STATS_MAX 1 +#define WINDOW_STATS_MAX_RECENT_AVG 2 +#define WINDOW_STATS_AVG 3 +#define WINDOW_STATS_INVALID_POLICY 4 + +#define EXITING_TASK_MARKER 0xdeaddead + +static __read_mostly unsigned int walt_ravg_hist_size = 5; +static __read_mostly unsigned int walt_window_stats_policy = + WINDOW_STATS_MAX_RECENT_AVG; +static __read_mostly unsigned int walt_account_wait_time = 1; +static __read_mostly unsigned int walt_freq_account_wait_time = 0; +static __read_mostly unsigned int walt_io_is_busy = 0; + +unsigned int sysctl_sched_walt_init_task_load_pct = 15; + +/* 1 -> use PELT based load stats, 0 -> use window-based load stats */ +unsigned int __read_mostly walt_disabled = 0; + +static unsigned int max_possible_efficiency = 1024; +static unsigned int min_possible_efficiency = 1024; + +/* + * Maximum possible frequency across all cpus. Task demand and cpu + * capacity (cpu_power) metrics are scaled in reference to it. + */ +static unsigned int max_possible_freq = 1; + +/* + * Minimum possible max_freq across all cpus. This will be same as + * max_possible_freq on homogeneous systems and could be different from + * max_possible_freq on heterogenous systems. min_max_freq is used to derive + * capacity (cpu_power) of cpus. + */ +static unsigned int min_max_freq = 1; + +static unsigned int max_capacity = 1024; +static unsigned int min_capacity = 1024; +static unsigned int max_load_scale_factor = 1024; +static unsigned int max_possible_capacity = 1024; + +/* Mask of all CPUs that have max_possible_capacity */ +static cpumask_t mpc_mask = CPU_MASK_ALL; + +/* Window size (in ns) */ +__read_mostly unsigned int walt_ravg_window = 20000000; + +/* Min window size (in ns) = 10ms */ +#define MIN_SCHED_RAVG_WINDOW 10000000 + +/* Max window size (in ns) = 1s */ +#define MAX_SCHED_RAVG_WINDOW 1000000000 + +static unsigned int sync_cpu; +static ktime_t ktime_last; +static bool walt_ktime_suspended; + +static unsigned int task_load(struct task_struct *p) +{ + return p->ravg.demand; +} + +void +walt_inc_cumulative_runnable_avg(struct rq *rq, + struct task_struct *p) +{ + rq->cumulative_runnable_avg += p->ravg.demand; +} + +void +walt_dec_cumulative_runnable_avg(struct rq *rq, + struct task_struct *p) +{ + rq->cumulative_runnable_avg -= p->ravg.demand; + BUG_ON((s64)rq->cumulative_runnable_avg < 0); +} + +static void +fixup_cumulative_runnable_avg(struct rq *rq, + struct task_struct *p, s64 task_load_delta) +{ + rq->cumulative_runnable_avg += task_load_delta; + if ((s64)rq->cumulative_runnable_avg < 0) + panic("cra less than zero: tld: %lld, task_load(p) = %u\n", + task_load_delta, task_load(p)); +} + +u64 walt_ktime_clock(void) +{ + if (unlikely(walt_ktime_suspended)) + return ktime_to_ns(ktime_last); + return ktime_get_ns(); +} + +static void walt_resume(void) +{ + walt_ktime_suspended = false; +} + +static int walt_suspend(void) +{ + ktime_last = ktime_get(); + walt_ktime_suspended = true; + return 0; +} + +static struct syscore_ops walt_syscore_ops = { + .resume = walt_resume, + .suspend = walt_suspend +}; + +static int __init walt_init_ops(void) +{ + register_syscore_ops(&walt_syscore_ops); + return 0; +} +late_initcall(walt_init_ops); + +void walt_inc_cfs_cumulative_runnable_avg(struct cfs_rq *cfs_rq, + struct task_struct *p) +{ + cfs_rq->cumulative_runnable_avg += p->ravg.demand; +} + +void walt_dec_cfs_cumulative_runnable_avg(struct cfs_rq *cfs_rq, + struct task_struct *p) +{ + cfs_rq->cumulative_runnable_avg -= p->ravg.demand; +} + +static int exiting_task(struct task_struct *p) +{ + if (p->flags & PF_EXITING) { + if (p->ravg.sum_history[0] != EXITING_TASK_MARKER) { + p->ravg.sum_history[0] = EXITING_TASK_MARKER; + } + return 1; + } + return 0; +} + +static int __init set_walt_ravg_window(char *str) +{ + get_option(&str, &walt_ravg_window); + + walt_disabled = (walt_ravg_window < MIN_SCHED_RAVG_WINDOW || + walt_ravg_window > MAX_SCHED_RAVG_WINDOW); + return 0; +} + +early_param("walt_ravg_window", set_walt_ravg_window); + +static void +update_window_start(struct rq *rq, u64 wallclock) +{ + s64 delta; + int nr_windows; + + delta = wallclock - rq->window_start; + /* If the MPM global timer is cleared, set delta as 0 to avoid kernel BUG happening */ + if (delta < 0) { + delta = 0; + WARN_ONCE(1, "WALT wallclock appears to have gone backwards or reset\n"); + } + + if (delta < walt_ravg_window) + return; + + nr_windows = div64_u64(delta, walt_ravg_window); + rq->window_start += (u64)nr_windows * (u64)walt_ravg_window; +} + +static u64 scale_exec_time(u64 delta, struct rq *rq) +{ + unsigned int cur_freq = rq->cur_freq; + int sf; + + if (unlikely(cur_freq > max_possible_freq)) + cur_freq = rq->max_possible_freq; + + /* round up div64 */ + delta = div64_u64(delta * cur_freq + max_possible_freq - 1, + max_possible_freq); + + sf = DIV_ROUND_UP(rq->efficiency * 1024, max_possible_efficiency); + + delta *= sf; + delta >>= 10; + + return delta; +} + +static int cpu_is_waiting_on_io(struct rq *rq) +{ + if (!walt_io_is_busy) + return 0; + + return atomic_read(&rq->nr_iowait); +} + +void walt_account_irqtime(int cpu, struct task_struct *curr, + u64 delta, u64 wallclock) +{ + struct rq *rq = cpu_rq(cpu); + unsigned long flags, nr_windows; + u64 cur_jiffies_ts; + + raw_spin_lock_irqsave(&rq->lock, flags); + + /* + * cputime (wallclock) uses sched_clock so use the same here for + * consistency. + */ + delta += sched_clock() - wallclock; + cur_jiffies_ts = get_jiffies_64(); + + if (is_idle_task(curr)) + walt_update_task_ravg(curr, rq, IRQ_UPDATE, walt_ktime_clock(), + delta); + + nr_windows = cur_jiffies_ts - rq->irqload_ts; + + if (nr_windows) { + if (nr_windows < 10) { + /* Decay CPU's irqload by 3/4 for each window. */ + rq->avg_irqload *= (3 * nr_windows); + rq->avg_irqload = div64_u64(rq->avg_irqload, + 4 * nr_windows); + } else { + rq->avg_irqload = 0; + } + rq->avg_irqload += rq->cur_irqload; + rq->cur_irqload = 0; + } + + rq->cur_irqload += delta; + rq->irqload_ts = cur_jiffies_ts; + raw_spin_unlock_irqrestore(&rq->lock, flags); +} + + +#define WALT_HIGH_IRQ_TIMEOUT 3 + +u64 walt_irqload(int cpu) { + struct rq *rq = cpu_rq(cpu); + s64 delta; + delta = get_jiffies_64() - rq->irqload_ts; + + /* + * Current context can be preempted by irq and rq->irqload_ts can be + * updated by irq context so that delta can be negative. + * But this is okay and we can safely return as this means there + * was recent irq occurrence. + */ + + if (delta < WALT_HIGH_IRQ_TIMEOUT) + return rq->avg_irqload; + else + return 0; +} + +int walt_cpu_high_irqload(int cpu) { + return walt_irqload(cpu) >= sysctl_sched_walt_cpu_high_irqload; +} + +static int account_busy_for_cpu_time(struct rq *rq, struct task_struct *p, + u64 irqtime, int event) +{ + if (is_idle_task(p)) { + /* TASK_WAKE && TASK_MIGRATE is not possible on idle task! */ + if (event == PICK_NEXT_TASK) + return 0; + + /* PUT_PREV_TASK, TASK_UPDATE && IRQ_UPDATE are left */ + return irqtime || cpu_is_waiting_on_io(rq); + } + + if (event == TASK_WAKE) + return 0; + + if (event == PUT_PREV_TASK || event == IRQ_UPDATE || + event == TASK_UPDATE) + return 1; + + /* Only TASK_MIGRATE && PICK_NEXT_TASK left */ + return walt_freq_account_wait_time; +} + +/* + * Account cpu activity in its busy time counters (rq->curr/prev_runnable_sum) + */ +static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, + int event, u64 wallclock, u64 irqtime) +{ + int new_window, nr_full_windows = 0; + int p_is_curr_task = (p == rq->curr); + u64 mark_start = p->ravg.mark_start; + u64 window_start = rq->window_start; + u32 window_size = walt_ravg_window; + u64 delta; + + new_window = mark_start < window_start; + if (new_window) { + nr_full_windows = div64_u64((window_start - mark_start), + window_size); + if (p->ravg.active_windows < USHRT_MAX) + p->ravg.active_windows++; + } + + /* Handle per-task window rollover. We don't care about the idle + * task or exiting tasks. */ + if (new_window && !is_idle_task(p) && !exiting_task(p)) { + u32 curr_window = 0; + + if (!nr_full_windows) + curr_window = p->ravg.curr_window; + + p->ravg.prev_window = curr_window; + p->ravg.curr_window = 0; + } + + if (!account_busy_for_cpu_time(rq, p, irqtime, event)) { + /* account_busy_for_cpu_time() = 0, so no update to the + * task's current window needs to be made. This could be + * for example + * + * - a wakeup event on a task within the current + * window (!new_window below, no action required), + * - switching to a new task from idle (PICK_NEXT_TASK) + * in a new window where irqtime is 0 and we aren't + * waiting on IO */ + + if (!new_window) + return; + + /* A new window has started. The RQ demand must be rolled + * over if p is the current task. */ + if (p_is_curr_task) { + u64 prev_sum = 0; + + /* p is either idle task or an exiting task */ + if (!nr_full_windows) { + prev_sum = rq->curr_runnable_sum; + } + + rq->prev_runnable_sum = prev_sum; + rq->curr_runnable_sum = 0; + } + + return; + } + + if (!new_window) { + /* account_busy_for_cpu_time() = 1 so busy time needs + * to be accounted to the current window. No rollover + * since we didn't start a new window. An example of this is + * when a task starts execution and then sleeps within the + * same window. */ + + if (!irqtime || !is_idle_task(p) || cpu_is_waiting_on_io(rq)) + delta = wallclock - mark_start; + else + delta = irqtime; + delta = scale_exec_time(delta, rq); + rq->curr_runnable_sum += delta; + if (!is_idle_task(p) && !exiting_task(p)) + p->ravg.curr_window += delta; + + return; + } + + if (!p_is_curr_task) { + /* account_busy_for_cpu_time() = 1 so busy time needs + * to be accounted to the current window. A new window + * has also started, but p is not the current task, so the + * window is not rolled over - just split up and account + * as necessary into curr and prev. The window is only + * rolled over when a new window is processed for the current + * task. + * + * Irqtime can't be accounted by a task that isn't the + * currently running task. */ + + if (!nr_full_windows) { + /* A full window hasn't elapsed, account partial + * contribution to previous completed window. */ + delta = scale_exec_time(window_start - mark_start, rq); + if (!exiting_task(p)) + p->ravg.prev_window += delta; + } else { + /* Since at least one full window has elapsed, + * the contribution to the previous window is the + * full window (window_size). */ + delta = scale_exec_time(window_size, rq); + if (!exiting_task(p)) + p->ravg.prev_window = delta; + } + rq->prev_runnable_sum += delta; + + /* Account piece of busy time in the current window. */ + delta = scale_exec_time(wallclock - window_start, rq); + rq->curr_runnable_sum += delta; + if (!exiting_task(p)) + p->ravg.curr_window = delta; + + return; + } + + if (!irqtime || !is_idle_task(p) || cpu_is_waiting_on_io(rq)) { + /* account_busy_for_cpu_time() = 1 so busy time needs + * to be accounted to the current window. A new window + * has started and p is the current task so rollover is + * needed. If any of these three above conditions are true + * then this busy time can't be accounted as irqtime. + * + * Busy time for the idle task or exiting tasks need not + * be accounted. + * + * An example of this would be a task that starts execution + * and then sleeps once a new window has begun. */ + + if (!nr_full_windows) { + /* A full window hasn't elapsed, account partial + * contribution to previous completed window. */ + delta = scale_exec_time(window_start - mark_start, rq); + if (!is_idle_task(p) && !exiting_task(p)) + p->ravg.prev_window += delta; + + delta += rq->curr_runnable_sum; + } else { + /* Since at least one full window has elapsed, + * the contribution to the previous window is the + * full window (window_size). */ + delta = scale_exec_time(window_size, rq); + if (!is_idle_task(p) && !exiting_task(p)) + p->ravg.prev_window = delta; + + } + /* + * Rollover for normal runnable sum is done here by overwriting + * the values in prev_runnable_sum and curr_runnable_sum. + * Rollover for new task runnable sum has completed by previous + * if-else statement. + */ + rq->prev_runnable_sum = delta; + + /* Account piece of busy time in the current window. */ + delta = scale_exec_time(wallclock - window_start, rq); + rq->curr_runnable_sum = delta; + if (!is_idle_task(p) && !exiting_task(p)) + p->ravg.curr_window = delta; + + return; + } + + if (irqtime) { + /* account_busy_for_cpu_time() = 1 so busy time needs + * to be accounted to the current window. A new window + * has started and p is the current task so rollover is + * needed. The current task must be the idle task because + * irqtime is not accounted for any other task. + * + * Irqtime will be accounted each time we process IRQ activity + * after a period of idleness, so we know the IRQ busy time + * started at wallclock - irqtime. */ + + BUG_ON(!is_idle_task(p)); + mark_start = wallclock - irqtime; + + /* Roll window over. If IRQ busy time was just in the current + * window then that is all that need be accounted. */ + rq->prev_runnable_sum = rq->curr_runnable_sum; + if (mark_start > window_start) { + rq->curr_runnable_sum = scale_exec_time(irqtime, rq); + return; + } + + /* The IRQ busy time spanned multiple windows. Process the + * busy time preceding the current window start first. */ + delta = window_start - mark_start; + if (delta > window_size) + delta = window_size; + delta = scale_exec_time(delta, rq); + rq->prev_runnable_sum += delta; + + /* Process the remaining IRQ busy time in the current window. */ + delta = wallclock - window_start; + rq->curr_runnable_sum = scale_exec_time(delta, rq); + + return; + } + + BUG(); +} + +static int account_busy_for_task_demand(struct task_struct *p, int event) +{ + /* No need to bother updating task demand for exiting tasks + * or the idle task. */ + if (exiting_task(p) || is_idle_task(p)) + return 0; + + /* When a task is waking up it is completing a segment of non-busy + * time. Likewise, if wait time is not treated as busy time, then + * when a task begins to run or is migrated, it is not running and + * is completing a segment of non-busy time. */ + if (event == TASK_WAKE || (!walt_account_wait_time && + (event == PICK_NEXT_TASK || event == TASK_MIGRATE))) + return 0; + + return 1; +} + +/* + * Called when new window is starting for a task, to record cpu usage over + * recently concluded window(s). Normally 'samples' should be 1. It can be > 1 + * when, say, a real-time task runs without preemption for several windows at a + * stretch. + */ +static void update_history(struct rq *rq, struct task_struct *p, + u32 runtime, int samples, int event) +{ + u32 *hist = &p->ravg.sum_history[0]; + int ridx, widx; + u32 max = 0, avg, demand; + u64 sum = 0; + + /* Ignore windows where task had no activity */ + if (!runtime || is_idle_task(p) || exiting_task(p) || !samples) + goto done; + + /* Push new 'runtime' value onto stack */ + widx = walt_ravg_hist_size - 1; + ridx = widx - samples; + for (; ridx >= 0; --widx, --ridx) { + hist[widx] = hist[ridx]; + sum += hist[widx]; + if (hist[widx] > max) + max = hist[widx]; + } + + for (widx = 0; widx < samples && widx < walt_ravg_hist_size; widx++) { + hist[widx] = runtime; + sum += hist[widx]; + if (hist[widx] > max) + max = hist[widx]; + } + + p->ravg.sum = 0; + + if (walt_window_stats_policy == WINDOW_STATS_RECENT) { + demand = runtime; + } else if (walt_window_stats_policy == WINDOW_STATS_MAX) { + demand = max; + } else { + avg = div64_u64(sum, walt_ravg_hist_size); + if (walt_window_stats_policy == WINDOW_STATS_AVG) + demand = avg; + else + demand = max(avg, runtime); + } + + /* + * A throttled deadline sched class task gets dequeued without + * changing p->on_rq. Since the dequeue decrements hmp stats + * avoid decrementing it here again. + */ + if (task_on_rq_queued(p) && (!task_has_dl_policy(p) || + !p->dl.dl_throttled)) + fixup_cumulative_runnable_avg(rq, p, demand); + + p->ravg.demand = demand; + +done: + trace_walt_update_history(rq, p, runtime, samples, event); + return; +} + +static void add_to_task_demand(struct rq *rq, struct task_struct *p, + u64 delta) +{ + delta = scale_exec_time(delta, rq); + p->ravg.sum += delta; + if (unlikely(p->ravg.sum > walt_ravg_window)) + p->ravg.sum = walt_ravg_window; +} + +/* + * Account cpu demand of task and/or update task's cpu demand history + * + * ms = p->ravg.mark_start; + * wc = wallclock + * ws = rq->window_start + * + * Three possibilities: + * + * a) Task event is contained within one window. + * window_start < mark_start < wallclock + * + * ws ms wc + * | | | + * V V V + * |---------------| + * + * In this case, p->ravg.sum is updated *iff* event is appropriate + * (ex: event == PUT_PREV_TASK) + * + * b) Task event spans two windows. + * mark_start < window_start < wallclock + * + * ms ws wc + * | | | + * V V V + * -----|------------------- + * + * In this case, p->ravg.sum is updated with (ws - ms) *iff* event + * is appropriate, then a new window sample is recorded followed + * by p->ravg.sum being set to (wc - ws) *iff* event is appropriate. + * + * c) Task event spans more than two windows. + * + * ms ws_tmp ws wc + * | | | | + * V V V V + * ---|-------|-------|-------|-------|------ + * | | + * |<------ nr_full_windows ------>| + * + * In this case, p->ravg.sum is updated with (ws_tmp - ms) first *iff* + * event is appropriate, window sample of p->ravg.sum is recorded, + * 'nr_full_window' samples of window_size is also recorded *iff* + * event is appropriate and finally p->ravg.sum is set to (wc - ws) + * *iff* event is appropriate. + * + * IMPORTANT : Leave p->ravg.mark_start unchanged, as update_cpu_busy_time() + * depends on it! + */ +static void update_task_demand(struct task_struct *p, struct rq *rq, + int event, u64 wallclock) +{ + u64 mark_start = p->ravg.mark_start; + u64 delta, window_start = rq->window_start; + int new_window, nr_full_windows; + u32 window_size = walt_ravg_window; + + new_window = mark_start < window_start; + if (!account_busy_for_task_demand(p, event)) { + if (new_window) + /* If the time accounted isn't being accounted as + * busy time, and a new window started, only the + * previous window need be closed out with the + * pre-existing demand. Multiple windows may have + * elapsed, but since empty windows are dropped, + * it is not necessary to account those. */ + update_history(rq, p, p->ravg.sum, 1, event); + return; + } + + if (!new_window) { + /* The simple case - busy time contained within the existing + * window. */ + add_to_task_demand(rq, p, wallclock - mark_start); + return; + } + + /* Busy time spans at least two windows. Temporarily rewind + * window_start to first window boundary after mark_start. */ + delta = window_start - mark_start; + nr_full_windows = div64_u64(delta, window_size); + window_start -= (u64)nr_full_windows * (u64)window_size; + + /* Process (window_start - mark_start) first */ + add_to_task_demand(rq, p, window_start - mark_start); + + /* Push new sample(s) into task's demand history */ + update_history(rq, p, p->ravg.sum, 1, event); + if (nr_full_windows) + update_history(rq, p, scale_exec_time(window_size, rq), + nr_full_windows, event); + + /* Roll window_start back to current to process any remainder + * in current window. */ + window_start += (u64)nr_full_windows * (u64)window_size; + + /* Process (wallclock - window_start) next */ + mark_start = window_start; + add_to_task_demand(rq, p, wallclock - mark_start); +} + +/* Reflect task activity on its demand and cpu's busy time statistics */ +void walt_update_task_ravg(struct task_struct *p, struct rq *rq, + int event, u64 wallclock, u64 irqtime) +{ + if (walt_disabled || !rq->window_start) + return; + + lockdep_assert_held(&rq->lock); + + update_window_start(rq, wallclock); + + if (!p->ravg.mark_start) + goto done; + + update_task_demand(p, rq, event, wallclock); + update_cpu_busy_time(p, rq, event, wallclock, irqtime); + +done: + trace_walt_update_task_ravg(p, rq, event, wallclock, irqtime); + + p->ravg.mark_start = wallclock; +} + +unsigned long __weak arch_get_cpu_efficiency(int cpu) +{ + return SCHED_CAPACITY_SCALE; +} + +void walt_init_cpu_efficiency(void) +{ + int i, efficiency; + unsigned int max = 0, min = UINT_MAX; + + for_each_possible_cpu(i) { + efficiency = arch_get_cpu_efficiency(i); + cpu_rq(i)->efficiency = efficiency; + + if (efficiency > max) + max = efficiency; + if (efficiency < min) + min = efficiency; + } + + if (max) + max_possible_efficiency = max; + + if (min) + min_possible_efficiency = min; +} + +static void reset_task_stats(struct task_struct *p) +{ + u32 sum = 0; + + if (exiting_task(p)) + sum = EXITING_TASK_MARKER; + + memset(&p->ravg, 0, sizeof(struct ravg)); + /* Retain EXITING_TASK marker */ + p->ravg.sum_history[0] = sum; +} + +void walt_mark_task_starting(struct task_struct *p) +{ + u64 wallclock; + struct rq *rq = task_rq(p); + + if (!rq->window_start) { + reset_task_stats(p); + return; + } + + wallclock = walt_ktime_clock(); + p->ravg.mark_start = wallclock; +} + +void walt_set_window_start(struct rq *rq) +{ + int cpu = cpu_of(rq); + struct rq *sync_rq = cpu_rq(sync_cpu); + + if (rq->window_start) + return; + + if (cpu == sync_cpu) { + rq->window_start = walt_ktime_clock(); + } else { + raw_spin_unlock(&rq->lock); + double_rq_lock(rq, sync_rq); + rq->window_start = cpu_rq(sync_cpu)->window_start; + rq->curr_runnable_sum = rq->prev_runnable_sum = 0; + raw_spin_unlock(&sync_rq->lock); + } + + rq->curr->ravg.mark_start = rq->window_start; +} + +void walt_migrate_sync_cpu(int cpu) +{ + if (cpu == sync_cpu) + sync_cpu = smp_processor_id(); +} + +void walt_fixup_busy_time(struct task_struct *p, int new_cpu) +{ + struct rq *src_rq = task_rq(p); + struct rq *dest_rq = cpu_rq(new_cpu); + u64 wallclock; + + if (!p->on_rq && p->state != TASK_WAKING) + return; + + if (exiting_task(p)) { + return; + } + + if (p->state == TASK_WAKING) + double_rq_lock(src_rq, dest_rq); + + wallclock = walt_ktime_clock(); + + walt_update_task_ravg(task_rq(p)->curr, task_rq(p), + TASK_UPDATE, wallclock, 0); + walt_update_task_ravg(dest_rq->curr, dest_rq, + TASK_UPDATE, wallclock, 0); + + walt_update_task_ravg(p, task_rq(p), TASK_MIGRATE, wallclock, 0); + + if (p->ravg.curr_window) { + src_rq->curr_runnable_sum -= p->ravg.curr_window; + dest_rq->curr_runnable_sum += p->ravg.curr_window; + } + + if (p->ravg.prev_window) { + src_rq->prev_runnable_sum -= p->ravg.prev_window; + dest_rq->prev_runnable_sum += p->ravg.prev_window; + } + + if ((s64)src_rq->prev_runnable_sum < 0) { + src_rq->prev_runnable_sum = 0; + WARN_ON(1); + } + if ((s64)src_rq->curr_runnable_sum < 0) { + src_rq->curr_runnable_sum = 0; + WARN_ON(1); + } + + trace_walt_migration_update_sum(src_rq, p); + trace_walt_migration_update_sum(dest_rq, p); + + if (p->state == TASK_WAKING) + double_rq_unlock(src_rq, dest_rq); +} + +/* Keep track of max/min capacity possible across CPUs "currently" */ +static void __update_min_max_capacity(void) +{ + int i; + int max = 0, min = INT_MAX; + + for_each_online_cpu(i) { + if (cpu_rq(i)->capacity > max) + max = cpu_rq(i)->capacity; + if (cpu_rq(i)->capacity < min) + min = cpu_rq(i)->capacity; + } + + max_capacity = max; + min_capacity = min; +} + +static void update_min_max_capacity(void) +{ + unsigned long flags; + int i; + + local_irq_save(flags); + for_each_possible_cpu(i) + raw_spin_lock(&cpu_rq(i)->lock); + + __update_min_max_capacity(); + + for_each_possible_cpu(i) + raw_spin_unlock(&cpu_rq(i)->lock); + local_irq_restore(flags); +} + +/* + * Return 'capacity' of a cpu in reference to "least" efficient cpu, such that + * least efficient cpu gets capacity of 1024 + */ +static unsigned long capacity_scale_cpu_efficiency(int cpu) +{ + return (1024 * cpu_rq(cpu)->efficiency) / min_possible_efficiency; +} + +/* + * Return 'capacity' of a cpu in reference to cpu with lowest max_freq + * (min_max_freq), such that one with lowest max_freq gets capacity of 1024. + */ +static unsigned long capacity_scale_cpu_freq(int cpu) +{ + return (1024 * cpu_rq(cpu)->max_freq) / min_max_freq; +} + +/* + * Return load_scale_factor of a cpu in reference to "most" efficient cpu, so + * that "most" efficient cpu gets a load_scale_factor of 1 + */ +static unsigned long load_scale_cpu_efficiency(int cpu) +{ + return DIV_ROUND_UP(1024 * max_possible_efficiency, + cpu_rq(cpu)->efficiency); +} + +/* + * Return load_scale_factor of a cpu in reference to cpu with best max_freq + * (max_possible_freq), so that one with best max_freq gets a load_scale_factor + * of 1. + */ +static unsigned long load_scale_cpu_freq(int cpu) +{ + return DIV_ROUND_UP(1024 * max_possible_freq, cpu_rq(cpu)->max_freq); +} + +static int compute_capacity(int cpu) +{ + int capacity = 1024; + + capacity *= capacity_scale_cpu_efficiency(cpu); + capacity >>= 10; + + capacity *= capacity_scale_cpu_freq(cpu); + capacity >>= 10; + + return capacity; +} + +static int compute_load_scale_factor(int cpu) +{ + int load_scale = 1024; + + /* + * load_scale_factor accounts for the fact that task load + * is in reference to "best" performing cpu. Task's load will need to be + * scaled (up) by a factor to determine suitability to be placed on a + * (little) cpu. + */ + load_scale *= load_scale_cpu_efficiency(cpu); + load_scale >>= 10; + + load_scale *= load_scale_cpu_freq(cpu); + load_scale >>= 10; + + return load_scale; +} + +static int cpufreq_notifier_policy(struct notifier_block *nb, + unsigned long val, void *data) +{ + struct cpufreq_policy *policy = (struct cpufreq_policy *)data; + int i, update_max = 0; + u64 highest_mpc = 0, highest_mplsf = 0; + const struct cpumask *cpus = policy->related_cpus; + unsigned int orig_min_max_freq = min_max_freq; + unsigned int orig_max_possible_freq = max_possible_freq; + /* Initialized to policy->max in case policy->related_cpus is empty! */ + unsigned int orig_max_freq = policy->max; + + if (val != CPUFREQ_NOTIFY && val != CPUFREQ_REMOVE_POLICY && + val != CPUFREQ_CREATE_POLICY) + return 0; + + if (val == CPUFREQ_REMOVE_POLICY || val == CPUFREQ_CREATE_POLICY) { + update_min_max_capacity(); + return 0; + } + + for_each_cpu(i, policy->related_cpus) { + cpumask_copy(&cpu_rq(i)->freq_domain_cpumask, + policy->related_cpus); + orig_max_freq = cpu_rq(i)->max_freq; + cpu_rq(i)->min_freq = policy->min; + cpu_rq(i)->max_freq = policy->max; + cpu_rq(i)->cur_freq = policy->cur; + cpu_rq(i)->max_possible_freq = policy->cpuinfo.max_freq; + } + + max_possible_freq = max(max_possible_freq, policy->cpuinfo.max_freq); + if (min_max_freq == 1) + min_max_freq = UINT_MAX; + min_max_freq = min(min_max_freq, policy->cpuinfo.max_freq); + BUG_ON(!min_max_freq); + BUG_ON(!policy->max); + + /* Changes to policy other than max_freq don't require any updates */ + if (orig_max_freq == policy->max) + return 0; + + /* + * A changed min_max_freq or max_possible_freq (possible during bootup) + * needs to trigger re-computation of load_scale_factor and capacity for + * all possible cpus (even those offline). It also needs to trigger + * re-computation of nr_big_task count on all online cpus. + * + * A changed rq->max_freq otoh needs to trigger re-computation of + * load_scale_factor and capacity for just the cluster of cpus involved. + * Since small task definition depends on max_load_scale_factor, a + * changed load_scale_factor of one cluster could influence + * classification of tasks in another cluster. Hence a changed + * rq->max_freq will need to trigger re-computation of nr_big_task + * count on all online cpus. + * + * While it should be sufficient for nr_big_tasks to be + * re-computed for only online cpus, we have inadequate context + * information here (in policy notifier) with regard to hotplug-safety + * context in which notification is issued. As a result, we can't use + * get_online_cpus() here, as it can lead to deadlock. Until cpufreq is + * fixed up to issue notification always in hotplug-safe context, + * re-compute nr_big_task for all possible cpus. + */ + + if (orig_min_max_freq != min_max_freq || + orig_max_possible_freq != max_possible_freq) { + cpus = cpu_possible_mask; + update_max = 1; + } + + /* + * Changed load_scale_factor can trigger reclassification of tasks as + * big or small. Make this change "atomic" so that tasks are accounted + * properly due to changed load_scale_factor + */ + for_each_cpu(i, cpus) { + struct rq *rq = cpu_rq(i); + + rq->capacity = compute_capacity(i); + rq->load_scale_factor = compute_load_scale_factor(i); + + if (update_max) { + u64 mpc, mplsf; + + mpc = div_u64(((u64) rq->capacity) * + rq->max_possible_freq, rq->max_freq); + rq->max_possible_capacity = (int) mpc; + + mplsf = div_u64(((u64) rq->load_scale_factor) * + rq->max_possible_freq, rq->max_freq); + + if (mpc > highest_mpc) { + highest_mpc = mpc; + cpumask_clear(&mpc_mask); + cpumask_set_cpu(i, &mpc_mask); + } else if (mpc == highest_mpc) { + cpumask_set_cpu(i, &mpc_mask); + } + + if (mplsf > highest_mplsf) + highest_mplsf = mplsf; + } + } + + if (update_max) { + max_possible_capacity = highest_mpc; + max_load_scale_factor = highest_mplsf; + } + + __update_min_max_capacity(); + + return 0; +} + +static int cpufreq_notifier_trans(struct notifier_block *nb, + unsigned long val, void *data) +{ + struct cpufreq_freqs *freq = (struct cpufreq_freqs *)data; + unsigned int cpu = freq->cpu, new_freq = freq->new; + unsigned long flags; + int i; + + if (val != CPUFREQ_POSTCHANGE) + return 0; + + BUG_ON(!new_freq); + + if (cpu_rq(cpu)->cur_freq == new_freq) + return 0; + + for_each_cpu(i, &cpu_rq(cpu)->freq_domain_cpumask) { + struct rq *rq = cpu_rq(i); + + raw_spin_lock_irqsave(&rq->lock, flags); + walt_update_task_ravg(rq->curr, rq, TASK_UPDATE, + walt_ktime_clock(), 0); + rq->cur_freq = new_freq; + raw_spin_unlock_irqrestore(&rq->lock, flags); + } + + return 0; +} + +static struct notifier_block notifier_policy_block = { + .notifier_call = cpufreq_notifier_policy +}; + +static struct notifier_block notifier_trans_block = { + .notifier_call = cpufreq_notifier_trans +}; + +static int register_sched_callback(void) +{ + int ret; + + ret = cpufreq_register_notifier(¬ifier_policy_block, + CPUFREQ_POLICY_NOTIFIER); + + if (!ret) + ret = cpufreq_register_notifier(¬ifier_trans_block, + CPUFREQ_TRANSITION_NOTIFIER); + + return 0; +} + +/* + * cpufreq callbacks can be registered at core_initcall or later time. + * Any registration done prior to that is "forgotten" by cpufreq. See + * initialization of variable init_cpufreq_transition_notifier_list_called + * for further information. + */ +core_initcall(register_sched_callback); + +void walt_init_new_task_load(struct task_struct *p) +{ + int i; + u32 init_load_windows = + div64_u64((u64)sysctl_sched_walt_init_task_load_pct * + (u64)walt_ravg_window, 100); + u32 init_load_pct = current->init_load_pct; + + p->init_load_pct = 0; + memset(&p->ravg, 0, sizeof(struct ravg)); + + if (init_load_pct) { + init_load_windows = div64_u64((u64)init_load_pct * + (u64)walt_ravg_window, 100); + } + + p->ravg.demand = init_load_windows; + for (i = 0; i < RAVG_HIST_SIZE_MAX; ++i) + p->ravg.sum_history[i] = init_load_windows; +} diff --git a/kernel/sched/walt.h b/kernel/sched/walt.h new file mode 100644 index 00000000000000..e181c87a928d89 --- /dev/null +++ b/kernel/sched/walt.h @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2016, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef __WALT_H +#define __WALT_H + +#ifdef CONFIG_SCHED_WALT + +void walt_update_task_ravg(struct task_struct *p, struct rq *rq, int event, + u64 wallclock, u64 irqtime); +void walt_inc_cumulative_runnable_avg(struct rq *rq, struct task_struct *p); +void walt_dec_cumulative_runnable_avg(struct rq *rq, struct task_struct *p); +void walt_inc_cfs_cumulative_runnable_avg(struct cfs_rq *rq, + struct task_struct *p); +void walt_dec_cfs_cumulative_runnable_avg(struct cfs_rq *rq, + struct task_struct *p); +void walt_fixup_busy_time(struct task_struct *p, int new_cpu); +void walt_init_new_task_load(struct task_struct *p); +void walt_mark_task_starting(struct task_struct *p); +void walt_set_window_start(struct rq *rq); +void walt_migrate_sync_cpu(int cpu); +void walt_init_cpu_efficiency(void); +u64 walt_ktime_clock(void); +void walt_account_irqtime(int cpu, struct task_struct *curr, u64 delta, + u64 wallclock); + +u64 walt_irqload(int cpu); +int walt_cpu_high_irqload(int cpu); + +#else /* CONFIG_SCHED_WALT */ + +static inline void walt_update_task_ravg(struct task_struct *p, struct rq *rq, + int event, u64 wallclock, u64 irqtime) { } +static inline void walt_inc_cumulative_runnable_avg(struct rq *rq, struct task_struct *p) { } +static inline void walt_dec_cumulative_runnable_avg(struct rq *rq, struct task_struct *p) { } +static inline void walt_inc_cfs_cumulative_runnable_avg(struct cfs_rq *rq, + struct task_struct *p) { } +static inline void walt_dec_cfs_cumulative_runnable_avg(struct cfs_rq *rq, + struct task_struct *p) { } +static inline void walt_fixup_busy_time(struct task_struct *p, int new_cpu) { } +static inline void walt_init_new_task_load(struct task_struct *p) { } +static inline void walt_mark_task_starting(struct task_struct *p) { } +static inline void walt_set_window_start(struct rq *rq) { } +static inline void walt_migrate_sync_cpu(int cpu) { } +static inline void walt_init_cpu_efficiency(void) { } +static inline u64 walt_ktime_clock(void) { return 0; } + +#endif /* CONFIG_SCHED_WALT */ + +extern unsigned int walt_disabled; + +#endif diff --git a/kernel/stacktrace.c b/kernel/stacktrace.c index b6e4c16377c708..9c15a9124e83b5 100644 --- a/kernel/stacktrace.c +++ b/kernel/stacktrace.c @@ -18,10 +18,8 @@ void print_stack_trace(struct stack_trace *trace, int spaces) if (WARN_ON(!trace->entries)) return; - for (i = 0; i < trace->nr_entries; i++) { - printk("%*c", 1 + spaces, ' '); - print_ip_sym(trace->entries[i]); - } + for (i = 0; i < trace->nr_entries; i++) + printk("%*c%pS\n", 1 + spaces, ' ', (void *)trace->entries[i]); } EXPORT_SYMBOL_GPL(print_stack_trace); @@ -29,7 +27,6 @@ int snprint_stack_trace(char *buf, size_t size, struct stack_trace *trace, int spaces) { int i; - unsigned long ip; int generated; int total = 0; @@ -37,9 +34,8 @@ int snprint_stack_trace(char *buf, size_t size, return 0; for (i = 0; i < trace->nr_entries; i++) { - ip = trace->entries[i]; - generated = snprintf(buf, size, "%*c[<%p>] %pS\n", - 1 + spaces, ' ', (void *) ip, (void *) ip); + generated = snprintf(buf, size, "%*c%pS\n", 1 + spaces, ' ', + (void *)trace->entries[i]); total += generated; diff --git a/kernel/sys.c b/kernel/sys.c index 89d5be418157fa..1157bf7a448f78 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -41,6 +41,8 @@ #include #include #include +#include +#include #include #include @@ -2072,6 +2074,153 @@ static int prctl_get_tid_address(struct task_struct *me, int __user **tid_addr) } #endif +#ifdef CONFIG_MMU +static int prctl_update_vma_anon_name(struct vm_area_struct *vma, + struct vm_area_struct **prev, + unsigned long start, unsigned long end, + const char __user *name_addr) +{ + struct mm_struct *mm = vma->vm_mm; + int error = 0; + pgoff_t pgoff; + + if (name_addr == vma_get_anon_name(vma)) { + *prev = vma; + goto out; + } + + pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT); + *prev = vma_merge(mm, *prev, start, end, vma->vm_flags, vma->anon_vma, + vma->vm_file, pgoff, vma_policy(vma), + vma->vm_userfaultfd_ctx, name_addr); + if (*prev) { + vma = *prev; + goto success; + } + + *prev = vma; + + if (start != vma->vm_start) { + error = split_vma(mm, vma, start, 1); + if (error) + goto out; + } + + if (end != vma->vm_end) { + error = split_vma(mm, vma, end, 0); + if (error) + goto out; + } + +success: + if (!vma->vm_file) + vma->anon_name = name_addr; + +out: + if (error == -ENOMEM) + error = -EAGAIN; + return error; +} + +static int prctl_set_vma_anon_name(unsigned long start, unsigned long end, + unsigned long arg) +{ + unsigned long tmp; + struct vm_area_struct *vma, *prev; + int unmapped_error = 0; + int error = -EINVAL; + + /* + * If the interval [start,end) covers some unmapped address + * ranges, just ignore them, but return -ENOMEM at the end. + * - this matches the handling in madvise. + */ + vma = find_vma_prev(current->mm, start, &prev); + if (vma && start > vma->vm_start) + prev = vma; + + for (;;) { + /* Still start < end. */ + error = -ENOMEM; + if (!vma) + return error; + + /* Here start < (end|vma->vm_end). */ + if (start < vma->vm_start) { + unmapped_error = -ENOMEM; + start = vma->vm_start; + if (start >= end) + return error; + } + + /* Here vma->vm_start <= start < (end|vma->vm_end) */ + tmp = vma->vm_end; + if (end < tmp) + tmp = end; + + /* Here vma->vm_start <= start < tmp <= (end|vma->vm_end). */ + error = prctl_update_vma_anon_name(vma, &prev, start, tmp, + (const char __user *)arg); + if (error) + return error; + start = tmp; + if (prev && start < prev->vm_end) + start = prev->vm_end; + error = unmapped_error; + if (start >= end) + return error; + if (prev) + vma = prev->vm_next; + else /* madvise_remove dropped mmap_sem */ + vma = find_vma(current->mm, start); + } +} + +static int prctl_set_vma(unsigned long opt, unsigned long start, + unsigned long len_in, unsigned long arg) +{ + struct mm_struct *mm = current->mm; + int error; + unsigned long len; + unsigned long end; + + if (start & ~PAGE_MASK) + return -EINVAL; + len = (len_in + ~PAGE_MASK) & PAGE_MASK; + + /* Check to see whether len was rounded up from small -ve to zero */ + if (len_in && !len) + return -EINVAL; + + end = start + len; + if (end < start) + return -EINVAL; + + if (end == start) + return 0; + + down_write(&mm->mmap_sem); + + switch (opt) { + case PR_SET_VMA_ANON_NAME: + error = prctl_set_vma_anon_name(start, end, arg); + break; + default: + error = -EINVAL; + } + + up_write(&mm->mmap_sem); + + return error; +} +#else /* CONFIG_MMU */ +static int prctl_set_vma(unsigned long opt, unsigned long start, + unsigned long len_in, unsigned long arg) +{ + return -EINVAL; +} +#endif + SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, unsigned long, arg4, unsigned long, arg5) { @@ -2270,6 +2419,9 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, case PR_GET_FP_MODE: error = GET_FP_MODE(me); break; + case PR_SET_VMA: + error = prctl_set_vma(arg2, arg3, arg4, arg5); + break; default: error = -EINVAL; break; diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 706309f9ed8422..f73fd318e84a94 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -307,6 +307,64 @@ static struct ctl_table kern_table[] = { .extra1 = &min_sched_granularity_ns, .extra2 = &max_sched_granularity_ns, }, + { + .procname = "sched_is_big_little", + .data = &sysctl_sched_is_big_little, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, +#ifdef CONFIG_SCHED_WALT + { + .procname = "sched_use_walt_cpu_util", + .data = &sysctl_sched_use_walt_cpu_util, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, + { + .procname = "sched_use_walt_task_util", + .data = &sysctl_sched_use_walt_task_util, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, + { + .procname = "sched_walt_init_task_load_pct", + .data = &sysctl_sched_walt_init_task_load_pct, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, + { + .procname = "sched_walt_cpu_high_irqload", + .data = &sysctl_sched_walt_cpu_high_irqload, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, +#endif + { + .procname = "sched_sync_hint_enable", + .data = &sysctl_sched_sync_hint_enable, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, + { + .procname = "sched_initial_task_util", + .data = &sysctl_sched_initial_task_util, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, + { + .procname = "sched_cstate_aware", + .data = &sysctl_sched_cstate_aware, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, { .procname = "sched_wakeup_granularity_ns", .data = &sysctl_sched_wakeup_granularity, @@ -449,6 +507,21 @@ static struct ctl_table kern_table[] = { .extra1 = &one, }, #endif +#ifdef CONFIG_SCHED_TUNE + { + .procname = "sched_cfs_boost", + .data = &sysctl_sched_cfs_boost, + .maxlen = sizeof(sysctl_sched_cfs_boost), +#ifdef CONFIG_CGROUP_SCHEDTUNE + .mode = 0444, +#else + .mode = 0644, +#endif + .proc_handler = &sysctl_sched_cfs_boost_handler, + .extra1 = &zero, + .extra2 = &one_hundred, + }, +#endif #ifdef CONFIG_PROVE_LOCKING { .procname = "prove_locking", @@ -2487,6 +2560,7 @@ static int __do_proc_doulongvec_minmax(void *data, struct ctl_table *table, int break; if (neg) continue; + val = convmul * val / convdiv; if ((min && val < *min) || (max && val > *max)) continue; *i = val; diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c index f6aae7977824ab..d2a20e83ebaed3 100644 --- a/kernel/time/tick-broadcast.c +++ b/kernel/time/tick-broadcast.c @@ -871,6 +871,9 @@ void tick_broadcast_setup_oneshot(struct clock_event_device *bc) { int cpu = smp_processor_id(); + if (!bc) + return; + /* Set it up only once ! */ if (bc->event_handler != tick_handle_oneshot_broadcast) { int was_periodic = clockevent_state_periodic(bc); diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 37dec7e3db43e3..bfe589e929e86e 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -299,10 +299,10 @@ u32 (*arch_gettimeoffset)(void) = default_arch_gettimeoffset; static inline u32 arch_gettimeoffset(void) { return 0; } #endif -static inline s64 timekeeping_delta_to_ns(struct tk_read_base *tkr, +static inline u64 timekeeping_delta_to_ns(struct tk_read_base *tkr, cycle_t delta) { - s64 nsec; + u64 nsec; nsec = delta * tkr->mult + tkr->xtime_nsec; nsec >>= tkr->shift; @@ -425,6 +425,35 @@ u64 ktime_get_raw_fast_ns(void) } EXPORT_SYMBOL_GPL(ktime_get_raw_fast_ns); +/** + * ktime_get_boot_fast_ns - NMI safe and fast access to boot clock. + * + * To keep it NMI safe since we're accessing from tracing, we're not using a + * separate timekeeper with updates to monotonic clock and boot offset + * protected with seqlocks. This has the following minor side effects: + * + * (1) Its possible that a timestamp be taken after the boot offset is updated + * but before the timekeeper is updated. If this happens, the new boot offset + * is added to the old timekeeping making the clock appear to update slightly + * earlier: + * CPU 0 CPU 1 + * timekeeping_inject_sleeptime64() + * __timekeeping_inject_sleeptime(tk, delta); + * timestamp(); + * timekeeping_update(tk, TK_CLEAR_NTP...); + * + * (2) On 32-bit systems, the 64-bit boot offset (tk->offs_boot) may be + * partially updated. Since the tk->offs_boot update is a rare event, this + * should be a rare occurrence which postprocessing should be able to handle. + */ +u64 notrace ktime_get_boot_fast_ns(void) +{ + struct timekeeper *tk = &tk_core.timekeeper; + + return (ktime_get_mono_fast_ns() + ktime_to_ns(tk->offs_boot)); +} +EXPORT_SYMBOL_GPL(ktime_get_boot_fast_ns); + /* Suspend-time cycles value for halted fast timekeeper. */ static cycle_t cycles_at_suspend; diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig index 2a96b063d6590f..da5768901a0d79 100644 --- a/kernel/trace/Kconfig +++ b/kernel/trace/Kconfig @@ -72,6 +72,9 @@ config EVENT_TRACING select CONTEXT_SWITCH_TRACER bool +config GPU_TRACEPOINTS + bool + config CONTEXT_SWITCH_TRACER bool diff --git a/kernel/trace/Makefile b/kernel/trace/Makefile index e5798084554911..b4eaf9c9c61060 100644 --- a/kernel/trace/Makefile +++ b/kernel/trace/Makefile @@ -67,6 +67,7 @@ obj-$(CONFIG_KGDB_KDB) += trace_kdb.o endif obj-$(CONFIG_PROBE_EVENTS) += trace_probe.o obj-$(CONFIG_UPROBE_EVENT) += trace_uprobe.o +obj-$(CONFIG_GPU_TRACEPOINTS) += gpu-traces.o obj-$(CONFIG_TRACEPOINT_BENCHMARK) += trace_benchmark.o diff --git a/kernel/trace/gpu-traces.c b/kernel/trace/gpu-traces.c new file mode 100644 index 00000000000000..a4b3f00faee36b --- /dev/null +++ b/kernel/trace/gpu-traces.c @@ -0,0 +1,23 @@ +/* + * GPU tracepoints + * + * Copyright (C) 2013 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include + +#define CREATE_TRACE_POINTS +#include + +EXPORT_TRACEPOINT_SYMBOL(gpu_sched_switch); +EXPORT_TRACEPOINT_SYMBOL(gpu_job_enqueue); diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 8696ce6bf2f688..2c99e77209ee48 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -1125,6 +1125,7 @@ static struct { { trace_clock, "perf", 1 }, { ktime_get_mono_fast_ns, "mono", 1 }, { ktime_get_raw_fast_ns, "mono_raw", 1 }, + { ktime_get_boot_fast_ns, "boot", 1 }, ARCH_TRACE_CLOCKS }; @@ -1597,6 +1598,7 @@ void tracing_reset_all_online_cpus(void) #define SAVED_CMDLINES_DEFAULT 128 #define NO_CMDLINE_MAP UINT_MAX +static unsigned saved_tgids[SAVED_CMDLINES_DEFAULT]; static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED; struct saved_cmdlines_buffer { unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1]; @@ -1835,7 +1837,7 @@ static int trace_save_cmdline(struct task_struct *tsk) } set_cmdline(idx, tsk->comm); - + saved_tgids[idx] = tsk->tgid; arch_spin_unlock(&trace_cmdline_lock); return 1; @@ -1878,6 +1880,25 @@ void trace_find_cmdline(int pid, char comm[]) preempt_enable(); } +int trace_find_tgid(int pid) +{ + unsigned map; + int tgid; + + preempt_disable(); + arch_spin_lock(&trace_cmdline_lock); + map = savedcmd->map_pid_to_cmdline[pid]; + if (map != NO_CMDLINE_MAP) + tgid = saved_tgids[map]; + else + tgid = -1; + + arch_spin_unlock(&trace_cmdline_lock); + preempt_enable(); + + return tgid; +} + void tracing_record_cmdline(struct task_struct *tsk) { if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on()) @@ -2920,6 +2941,13 @@ static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m) "# | | | | |\n"); } +static void print_func_help_header_tgid(struct trace_buffer *buf, struct seq_file *m) +{ + print_event_info(buf, m); + seq_puts(m, "# TASK-PID TGID CPU# TIMESTAMP FUNCTION\n"); + seq_puts(m, "# | | | | | |\n"); +} + static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m) { print_event_info(buf, m); @@ -2932,6 +2960,18 @@ static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file "# | | | |||| | |\n"); } +static void print_func_help_header_irq_tgid(struct trace_buffer *buf, struct seq_file *m) +{ + print_event_info(buf, m); + seq_puts(m, "# _-----=> irqs-off\n"); + seq_puts(m, "# / _----=> need-resched\n"); + seq_puts(m, "# | / _---=> hardirq/softirq\n"); + seq_puts(m, "# || / _--=> preempt-depth\n"); + seq_puts(m, "# ||| / delay\n"); + seq_puts(m, "# TASK-PID TGID CPU# |||| TIMESTAMP FUNCTION\n"); + seq_puts(m, "# | | | | |||| | |\n"); +} + void print_trace_header(struct seq_file *m, struct trace_iterator *iter) { @@ -3244,9 +3284,15 @@ void trace_default_header(struct seq_file *m) } else { if (!(trace_flags & TRACE_ITER_VERBOSE)) { if (trace_flags & TRACE_ITER_IRQ_INFO) - print_func_help_header_irq(iter->trace_buffer, m); + if (trace_flags & TRACE_ITER_TGID) + print_func_help_header_irq_tgid(iter->trace_buffer, m); + else + print_func_help_header_irq(iter->trace_buffer, m); else - print_func_help_header(iter->trace_buffer, m); + if (trace_flags & TRACE_ITER_TGID) + print_func_help_header_tgid(iter->trace_buffer, m); + else + print_func_help_header(iter->trace_buffer, m); } } } @@ -4581,6 +4627,50 @@ static void trace_insert_enum_map(struct module *mod, trace_insert_enum_map_file(mod, start, len); } +static ssize_t +tracing_saved_tgids_read(struct file *file, char __user *ubuf, + size_t cnt, loff_t *ppos) +{ + char *file_buf; + char *buf; + int len = 0; + int pid; + int i; + + file_buf = kmalloc(SAVED_CMDLINES_DEFAULT*(16+1+16), GFP_KERNEL); + if (!file_buf) + return -ENOMEM; + + buf = file_buf; + + for (i = 0; i < SAVED_CMDLINES_DEFAULT; i++) { + int tgid; + int r; + + pid = savedcmd->map_cmdline_to_pid[i]; + if (pid == -1 || pid == NO_CMDLINE_MAP) + continue; + + tgid = trace_find_tgid(pid); + r = sprintf(buf, "%d %d\n", pid, tgid); + buf += r; + len += r; + } + + len = simple_read_from_buffer(ubuf, cnt, ppos, + file_buf, len); + + kfree(file_buf); + + return len; +} + +static const struct file_operations tracing_saved_tgids_fops = { + .open = tracing_open_generic, + .read = tracing_saved_tgids_read, + .llseek = generic_file_llseek, +}; + static ssize_t tracing_set_trace_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) @@ -7214,6 +7304,9 @@ init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer) trace_create_file("trace_marker", 0220, d_tracer, tr, &tracing_mark_fops); + trace_create_file("saved_tgids", 0444, d_tracer, + tr, &tracing_saved_tgids_fops); + trace_create_file("trace_clock", 0644, d_tracer, tr, &trace_clock_fops); diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index fd24b1f9ac4397..250ebf4658299b 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -690,6 +690,7 @@ extern cycle_t ftrace_now(int cpu); extern void trace_find_cmdline(int pid, char comm[]); extern void trace_event_follow_fork(struct trace_array *tr, bool enable); +extern int trace_find_tgid(int pid); #ifdef CONFIG_DYNAMIC_FTRACE extern unsigned long ftrace_update_tot_cnt; @@ -1007,7 +1008,8 @@ extern int trace_get_user(struct trace_parser *parser, const char __user *ubuf, FUNCTION_FLAGS \ FGRAPH_FLAGS \ STACK_FLAGS \ - BRANCH_FLAGS + BRANCH_FLAGS \ + C(TGID, "print-tgid"), /* * By defining C, we can make TRACE_FLAGS a list of bit names diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c index 4e480e8704746e..3cb38f1bcd2811 100644 --- a/kernel/trace/trace_functions_graph.c +++ b/kernel/trace/trace_functions_graph.c @@ -65,6 +65,9 @@ struct fgraph_data { #define TRACE_GRAPH_INDENT 2 +/* Flag options */ +#define TRACE_GRAPH_PRINT_FLAT 0x80 + static unsigned int max_depth; static struct tracer_opt trace_opts[] = { @@ -88,6 +91,8 @@ static struct tracer_opt trace_opts[] = { { TRACER_OPT(sleep-time, TRACE_GRAPH_SLEEP_TIME) }, /* Include time within nested functions */ { TRACER_OPT(graph-time, TRACE_GRAPH_GRAPH_TIME) }, + /* Use standard trace formatting rather than hierarchical */ + { TRACER_OPT(funcgraph-flat, TRACE_GRAPH_PRINT_FLAT) }, { } /* Empty entry */ }; @@ -842,6 +847,10 @@ print_graph_entry_leaf(struct trace_iterator *iter, cpu_data = per_cpu_ptr(data->cpu_data, cpu); + /* If a graph tracer ignored set_graph_notrace */ + if (call->depth < -1) + call->depth += FTRACE_NOTRACE_DEPTH; + /* * Comments display at + 1 to depth. Since * this is a leaf function, keep the comments @@ -850,7 +859,8 @@ print_graph_entry_leaf(struct trace_iterator *iter, cpu_data->depth = call->depth - 1; /* No need to keep this function around for this depth */ - if (call->depth < FTRACE_RETFUNC_DEPTH) + if (call->depth < FTRACE_RETFUNC_DEPTH && + !WARN_ON_ONCE(call->depth < 0)) cpu_data->enter_funcs[call->depth] = 0; } @@ -880,11 +890,16 @@ print_graph_entry_nested(struct trace_iterator *iter, struct fgraph_cpu_data *cpu_data; int cpu = iter->cpu; + /* If a graph tracer ignored set_graph_notrace */ + if (call->depth < -1) + call->depth += FTRACE_NOTRACE_DEPTH; + cpu_data = per_cpu_ptr(data->cpu_data, cpu); cpu_data->depth = call->depth; /* Save this function pointer to see if the exit matches */ - if (call->depth < FTRACE_RETFUNC_DEPTH) + if (call->depth < FTRACE_RETFUNC_DEPTH && + !WARN_ON_ONCE(call->depth < 0)) cpu_data->enter_funcs[call->depth] = call->func; } @@ -1114,7 +1129,8 @@ print_graph_return(struct ftrace_graph_ret *trace, struct trace_seq *s, */ cpu_data->depth = trace->depth - 1; - if (trace->depth < FTRACE_RETFUNC_DEPTH) { + if (trace->depth < FTRACE_RETFUNC_DEPTH && + !WARN_ON_ONCE(trace->depth < 0)) { if (cpu_data->enter_funcs[trace->depth] != trace->func) func_match = 0; cpu_data->enter_funcs[trace->depth] = 0; @@ -1232,6 +1248,9 @@ print_graph_function_flags(struct trace_iterator *iter, u32 flags) int cpu = iter->cpu; int ret; + if (flags & TRACE_GRAPH_PRINT_FLAT) + return TRACE_TYPE_UNHANDLED; + if (data && per_cpu_ptr(data->cpu_data, cpu)->ignore) { per_cpu_ptr(data->cpu_data, cpu)->ignore = 0; return TRACE_TYPE_HANDLED; @@ -1289,13 +1308,6 @@ print_graph_function(struct trace_iterator *iter) return print_graph_function_flags(iter, tracer_flags.val); } -static enum print_line_t -print_graph_function_event(struct trace_iterator *iter, int flags, - struct trace_event *event) -{ - return print_graph_function(iter); -} - static void print_lat_header(struct seq_file *s, u32 flags) { static const char spaces[] = " " /* 16 spaces */ @@ -1364,6 +1376,11 @@ void print_graph_headers_flags(struct seq_file *s, u32 flags) struct trace_iterator *iter = s->private; struct trace_array *tr = iter->tr; + if (flags & TRACE_GRAPH_PRINT_FLAT) { + trace_default_header(s); + return; + } + if (!(tr->trace_flags & TRACE_ITER_CONTEXT_INFO)) return; @@ -1445,19 +1462,6 @@ func_graph_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set) return 0; } -static struct trace_event_functions graph_functions = { - .trace = print_graph_function_event, -}; - -static struct trace_event graph_trace_entry_event = { - .type = TRACE_GRAPH_ENT, - .funcs = &graph_functions, -}; - -static struct trace_event graph_trace_ret_event = { - .type = TRACE_GRAPH_RET, - .funcs = &graph_functions -}; static struct tracer graph_trace __tracer_data = { .name = "function_graph", @@ -1534,16 +1538,6 @@ static __init int init_graph_trace(void) { max_bytes_for_cpu = snprintf(NULL, 0, "%d", nr_cpu_ids - 1); - if (!register_trace_event(&graph_trace_entry_event)) { - pr_warn("Warning: could not register graph trace events\n"); - return 1; - } - - if (!register_trace_event(&graph_trace_ret_event)) { - pr_warn("Warning: could not register graph trace events\n"); - return 1; - } - return register_tracer(&graph_trace); } diff --git a/kernel/trace/trace_hwlat.c b/kernel/trace/trace_hwlat.c index b97286c48735b8..f00b0131c8f957 100644 --- a/kernel/trace/trace_hwlat.c +++ b/kernel/trace/trace_hwlat.c @@ -266,7 +266,7 @@ static int get_sample(void) static struct cpumask save_cpumask; static bool disable_migrate; -static void move_to_next_cpu(void) +static void move_to_next_cpu(bool initmask) { static struct cpumask *current_mask; int next_cpu; @@ -275,7 +275,7 @@ static void move_to_next_cpu(void) return; /* Just pick the first CPU on first iteration */ - if (!current_mask) { + if (initmask) { current_mask = &save_cpumask; get_online_cpus(); cpumask_and(current_mask, cpu_online_mask, tracing_buffer_mask); @@ -330,10 +330,12 @@ static void move_to_next_cpu(void) static int kthread_fn(void *data) { u64 interval; + bool initmask = true; while (!kthread_should_stop()) { - move_to_next_cpu(); + move_to_next_cpu(initmask); + initmask = false; local_irq_disable(); get_sample(); diff --git a/kernel/trace/trace_output.c b/kernel/trace/trace_output.c index 3fc20422c16696..034675950649a5 100644 --- a/kernel/trace/trace_output.c +++ b/kernel/trace/trace_output.c @@ -530,11 +530,21 @@ int trace_print_context(struct trace_iterator *iter) unsigned long long t; unsigned long secs, usec_rem; char comm[TASK_COMM_LEN]; + int tgid; trace_find_cmdline(entry->pid, comm); - trace_seq_printf(s, "%16s-%-5d [%03d] ", - comm, entry->pid, iter->cpu); + trace_seq_printf(s, "%16s-%-5d ", comm, entry->pid); + + if (tr->trace_flags & TRACE_ITER_TGID) { + tgid = trace_find_tgid(entry->pid); + if (tgid < 0) + trace_seq_puts(s, "(-----) "); + else + trace_seq_printf(s, "(%5d) ", tgid); + } + + trace_seq_printf(s, "[%03d] ", iter->cpu); if (tr->trace_flags & TRACE_ITER_IRQ_INFO) trace_print_lat_fmt(s, entry); @@ -849,6 +859,174 @@ static struct trace_event trace_fn_event = { .funcs = &trace_fn_funcs, }; +/* TRACE_GRAPH_ENT */ +static enum print_line_t trace_graph_ent_trace(struct trace_iterator *iter, int flags, + struct trace_event *event) +{ + struct trace_seq *s = &iter->seq; + struct ftrace_graph_ent_entry *field; + + trace_assign_type(field, iter->ent); + + trace_seq_puts(s, "graph_ent: func="); + if (trace_seq_has_overflowed(s)) + return TRACE_TYPE_PARTIAL_LINE; + + if (!seq_print_ip_sym(s, field->graph_ent.func, flags)) + return TRACE_TYPE_PARTIAL_LINE; + + trace_seq_puts(s, "\n"); + if (trace_seq_has_overflowed(s)) + return TRACE_TYPE_PARTIAL_LINE; + + return TRACE_TYPE_HANDLED; +} + +static enum print_line_t trace_graph_ent_raw(struct trace_iterator *iter, int flags, + struct trace_event *event) +{ + struct ftrace_graph_ent_entry *field; + + trace_assign_type(field, iter->ent); + + trace_seq_printf(&iter->seq, "%lx %d\n", + field->graph_ent.func, + field->graph_ent.depth); + if (trace_seq_has_overflowed(&iter->seq)) + return TRACE_TYPE_PARTIAL_LINE; + + return TRACE_TYPE_HANDLED; +} + +static enum print_line_t trace_graph_ent_hex(struct trace_iterator *iter, int flags, + struct trace_event *event) +{ + struct ftrace_graph_ent_entry *field; + struct trace_seq *s = &iter->seq; + + trace_assign_type(field, iter->ent); + + SEQ_PUT_HEX_FIELD(s, field->graph_ent.func); + SEQ_PUT_HEX_FIELD(s, field->graph_ent.depth); + + return TRACE_TYPE_HANDLED; +} + +static enum print_line_t trace_graph_ent_bin(struct trace_iterator *iter, int flags, + struct trace_event *event) +{ + struct ftrace_graph_ent_entry *field; + struct trace_seq *s = &iter->seq; + + trace_assign_type(field, iter->ent); + + SEQ_PUT_FIELD(s, field->graph_ent.func); + SEQ_PUT_FIELD(s, field->graph_ent.depth); + + return TRACE_TYPE_HANDLED; +} + +static struct trace_event_functions trace_graph_ent_funcs = { + .trace = trace_graph_ent_trace, + .raw = trace_graph_ent_raw, + .hex = trace_graph_ent_hex, + .binary = trace_graph_ent_bin, +}; + +static struct trace_event trace_graph_ent_event = { + .type = TRACE_GRAPH_ENT, + .funcs = &trace_graph_ent_funcs, +}; + +/* TRACE_GRAPH_RET */ +static enum print_line_t trace_graph_ret_trace(struct trace_iterator *iter, int flags, + struct trace_event *event) +{ + struct trace_seq *s = &iter->seq; + struct trace_entry *entry = iter->ent; + struct ftrace_graph_ret_entry *field; + + trace_assign_type(field, entry); + + trace_seq_puts(s, "graph_ret: func="); + if (trace_seq_has_overflowed(s)) + return TRACE_TYPE_PARTIAL_LINE; + + if (!seq_print_ip_sym(s, field->ret.func, flags)) + return TRACE_TYPE_PARTIAL_LINE; + + trace_seq_puts(s, "\n"); + if (trace_seq_has_overflowed(s)) + return TRACE_TYPE_PARTIAL_LINE; + + return TRACE_TYPE_HANDLED; +} + +static enum print_line_t trace_graph_ret_raw(struct trace_iterator *iter, int flags, + struct trace_event *event) +{ + struct ftrace_graph_ret_entry *field; + + trace_assign_type(field, iter->ent); + + trace_seq_printf(&iter->seq, "%lx %lld %lld %ld %d\n", + field->ret.func, + field->ret.calltime, + field->ret.rettime, + field->ret.overrun, + field->ret.depth); + if (trace_seq_has_overflowed(&iter->seq)) + return TRACE_TYPE_PARTIAL_LINE; + + return TRACE_TYPE_HANDLED; +} + +static enum print_line_t trace_graph_ret_hex(struct trace_iterator *iter, int flags, + struct trace_event *event) +{ + struct ftrace_graph_ret_entry *field; + struct trace_seq *s = &iter->seq; + + trace_assign_type(field, iter->ent); + + SEQ_PUT_HEX_FIELD(s, field->ret.func); + SEQ_PUT_HEX_FIELD(s, field->ret.calltime); + SEQ_PUT_HEX_FIELD(s, field->ret.rettime); + SEQ_PUT_HEX_FIELD(s, field->ret.overrun); + SEQ_PUT_HEX_FIELD(s, field->ret.depth); + + return TRACE_TYPE_HANDLED; +} + +static enum print_line_t trace_graph_ret_bin(struct trace_iterator *iter, int flags, + struct trace_event *event) +{ + struct ftrace_graph_ret_entry *field; + struct trace_seq *s = &iter->seq; + + trace_assign_type(field, iter->ent); + + SEQ_PUT_FIELD(s, field->ret.func); + SEQ_PUT_FIELD(s, field->ret.calltime); + SEQ_PUT_FIELD(s, field->ret.rettime); + SEQ_PUT_FIELD(s, field->ret.overrun); + SEQ_PUT_FIELD(s, field->ret.depth); + + return TRACE_TYPE_HANDLED; +} + +static struct trace_event_functions trace_graph_ret_funcs = { + .trace = trace_graph_ret_trace, + .raw = trace_graph_ret_raw, + .hex = trace_graph_ret_hex, + .binary = trace_graph_ret_bin, +}; + +static struct trace_event trace_graph_ret_event = { + .type = TRACE_GRAPH_RET, + .funcs = &trace_graph_ret_funcs, +}; + /* TRACE_CTX an TRACE_WAKE */ static enum print_line_t trace_ctxwake_print(struct trace_iterator *iter, char *delim) @@ -1291,6 +1469,8 @@ static struct trace_event trace_print_event = { static struct trace_event *events[] __initdata = { &trace_fn_event, + &trace_graph_ent_event, + &trace_graph_ret_event, &trace_ctx_event, &trace_wake_event, &trace_stack_event, diff --git a/kernel/ucount.c b/kernel/ucount.c index 9d20d5dd298af2..4bbd38ec37886d 100644 --- a/kernel/ucount.c +++ b/kernel/ucount.c @@ -128,10 +128,10 @@ static struct ucounts *get_ucounts(struct user_namespace *ns, kuid_t uid) struct hlist_head *hashent = ucounts_hashentry(ns, uid); struct ucounts *ucounts, *new; - spin_lock(&ucounts_lock); + spin_lock_irq(&ucounts_lock); ucounts = find_ucounts(ns, uid, hashent); if (!ucounts) { - spin_unlock(&ucounts_lock); + spin_unlock_irq(&ucounts_lock); new = kzalloc(sizeof(*new), GFP_KERNEL); if (!new) @@ -141,7 +141,7 @@ static struct ucounts *get_ucounts(struct user_namespace *ns, kuid_t uid) new->uid = uid; atomic_set(&new->count, 0); - spin_lock(&ucounts_lock); + spin_lock_irq(&ucounts_lock); ucounts = find_ucounts(ns, uid, hashent); if (ucounts) { kfree(new); @@ -152,16 +152,18 @@ static struct ucounts *get_ucounts(struct user_namespace *ns, kuid_t uid) } if (!atomic_add_unless(&ucounts->count, 1, INT_MAX)) ucounts = NULL; - spin_unlock(&ucounts_lock); + spin_unlock_irq(&ucounts_lock); return ucounts; } static void put_ucounts(struct ucounts *ucounts) { + unsigned long flags; + if (atomic_dec_and_test(&ucounts->count)) { - spin_lock(&ucounts_lock); + spin_lock_irqsave(&ucounts_lock, flags); hlist_del_init(&ucounts->node); - spin_unlock(&ucounts_lock); + spin_unlock_irqrestore(&ucounts_lock, flags); kfree(ucounts); } diff --git a/kernel/watchdog.c b/kernel/watchdog.c index 9acb29f280ec95..ea6271bed7840a 100644 --- a/kernel/watchdog.c +++ b/kernel/watchdog.c @@ -104,6 +104,11 @@ static DEFINE_PER_CPU(struct task_struct *, softlockup_task_ptr_saved); static DEFINE_PER_CPU(bool, hard_watchdog_warn); static DEFINE_PER_CPU(bool, watchdog_nmi_touch); static DEFINE_PER_CPU(unsigned long, hrtimer_interrupts_saved); +#endif +#ifdef CONFIG_HARDLOCKUP_DETECTOR_OTHER_CPU +static cpumask_t __read_mostly watchdog_cpus; +#endif +#ifdef CONFIG_HARDLOCKUP_DETECTOR_NMI static DEFINE_PER_CPU(struct perf_event *, watchdog_ev); #endif static unsigned long soft_lockup_nmi_warn; @@ -115,7 +120,7 @@ static unsigned long soft_lockup_nmi_warn; #ifdef CONFIG_HARDLOCKUP_DETECTOR unsigned int __read_mostly hardlockup_panic = CONFIG_BOOTPARAM_HARDLOCKUP_PANIC_VALUE; -static unsigned long hardlockup_allcpu_dumped; +static unsigned long __maybe_unused hardlockup_allcpu_dumped; /* * We may not want to enable hard lockup detection by default in all cases, * for example when running the kernel as a guest on a hypervisor. In these @@ -287,7 +292,7 @@ void touch_softlockup_watchdog_sync(void) __this_cpu_write(watchdog_touch_ts, 0); } -#ifdef CONFIG_HARDLOCKUP_DETECTOR +#ifdef CONFIG_HARDLOCKUP_DETECTOR_NMI /* watchdog detector functions */ static bool is_hardlockup(void) { @@ -301,6 +306,76 @@ static bool is_hardlockup(void) } #endif +#ifdef CONFIG_HARDLOCKUP_DETECTOR_OTHER_CPU +static unsigned int watchdog_next_cpu(unsigned int cpu) +{ + cpumask_t cpus = watchdog_cpus; + unsigned int next_cpu; + + next_cpu = cpumask_next(cpu, &cpus); + if (next_cpu >= nr_cpu_ids) + next_cpu = cpumask_first(&cpus); + + if (next_cpu == cpu) + return nr_cpu_ids; + + return next_cpu; +} + +static int is_hardlockup_other_cpu(unsigned int cpu) +{ + unsigned long hrint = per_cpu(hrtimer_interrupts, cpu); + + if (per_cpu(hrtimer_interrupts_saved, cpu) == hrint) + return 1; + + per_cpu(hrtimer_interrupts_saved, cpu) = hrint; + return 0; +} + +static void watchdog_check_hardlockup_other_cpu(void) +{ + unsigned int next_cpu; + + /* + * Test for hardlockups every 3 samples. The sample period is + * watchdog_thresh * 2 / 5, so 3 samples gets us back to slightly over + * watchdog_thresh (over by 20%). + */ + if (__this_cpu_read(hrtimer_interrupts) % 3 != 0) + return; + + /* check for a hardlockup on the next cpu */ + next_cpu = watchdog_next_cpu(smp_processor_id()); + if (next_cpu >= nr_cpu_ids) + return; + + smp_rmb(); + + if (per_cpu(watchdog_nmi_touch, next_cpu) == true) { + per_cpu(watchdog_nmi_touch, next_cpu) = false; + return; + } + + if (is_hardlockup_other_cpu(next_cpu)) { + /* only warn once */ + if (per_cpu(hard_watchdog_warn, next_cpu) == true) + return; + + if (hardlockup_panic) + panic("Watchdog detected hard LOCKUP on cpu %u", next_cpu); + else + WARN(1, "Watchdog detected hard LOCKUP on cpu %u", next_cpu); + + per_cpu(hard_watchdog_warn, next_cpu) = true; + } else { + per_cpu(hard_watchdog_warn, next_cpu) = false; + } +} +#else +static inline void watchdog_check_hardlockup_other_cpu(void) { return; } +#endif + static int is_softlockup(unsigned long touch_ts) { unsigned long now = get_timestamp(); @@ -313,7 +388,7 @@ static int is_softlockup(unsigned long touch_ts) return 0; } -#ifdef CONFIG_HARDLOCKUP_DETECTOR +#ifdef CONFIG_HARDLOCKUP_DETECTOR_NMI static struct perf_event_attr wd_hw_attr = { .type = PERF_TYPE_HARDWARE, @@ -344,7 +419,6 @@ static void watchdog_overflow_callback(struct perf_event *event, */ if (is_hardlockup()) { int this_cpu = smp_processor_id(); - struct pt_regs *regs = get_irq_regs(); /* only print hardlockups once */ if (__this_cpu_read(hard_watchdog_warn) == true) @@ -376,7 +450,7 @@ static void watchdog_overflow_callback(struct perf_event *event, __this_cpu_write(hard_watchdog_warn, false); return; } -#endif /* CONFIG_HARDLOCKUP_DETECTOR */ +#endif /* CONFIG_HARDLOCKUP_DETECTOR_NMI */ static void watchdog_interrupt_count(void) { @@ -400,6 +474,9 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer) /* kick the hardlockup detector */ watchdog_interrupt_count(); + /* test for hardlockups on the next cpu */ + watchdog_check_hardlockup_other_cpu(); + /* kick the softlockup detector */ wake_up_process(__this_cpu_read(softlockup_watchdog)); @@ -577,7 +654,7 @@ static void watchdog(unsigned int cpu) watchdog_nmi_disable(cpu); } -#ifdef CONFIG_HARDLOCKUP_DETECTOR +#ifdef CONFIG_HARDLOCKUP_DETECTOR_NMI /* * People like the simple clean cpu node info on boot. * Reduce the watchdog noise by only printing messages @@ -675,10 +752,45 @@ static void watchdog_nmi_disable(unsigned int cpu) } } +#else +#ifdef CONFIG_HARDLOCKUP_DETECTOR_OTHER_CPU +static int watchdog_nmi_enable(unsigned int cpu) +{ + /* + * The new cpu will be marked online before the first hrtimer interrupt + * runs on it. If another cpu tests for a hardlockup on the new cpu + * before it has run its first hrtimer, it will get a false positive. + * Touch the watchdog on the new cpu to delay the first check for at + * least 3 sampling periods to guarantee one hrtimer has run on the new + * cpu. + */ + per_cpu(watchdog_nmi_touch, cpu) = true; + smp_wmb(); + cpumask_set_cpu(cpu, &watchdog_cpus); + return 0; +} + +static void watchdog_nmi_disable(unsigned int cpu) +{ + unsigned int next_cpu = watchdog_next_cpu(cpu); + + /* + * Offlining this cpu will cause the cpu before this one to start + * checking the one after this one. If this cpu just finished checking + * the next cpu and updating hrtimer_interrupts_saved, and then the + * previous cpu checks it within one sample period, it will trigger a + * false positive. Touch the watchdog on the next cpu to prevent it. + */ + if (next_cpu < nr_cpu_ids) + per_cpu(watchdog_nmi_touch, next_cpu) = true; + smp_wmb(); + cpumask_clear_cpu(cpu, &watchdog_cpus); +} #else static int watchdog_nmi_enable(unsigned int cpu) { return 0; } static void watchdog_nmi_disable(unsigned int cpu) { return; } -#endif /* CONFIG_HARDLOCKUP_DETECTOR */ +#endif /* CONFIG_HARDLOCKUP_DETECTOR_OTHER_CPU */ +#endif /* CONFIG_HARDLOCKUP_DETECTOR_NMI */ static struct smp_hotplug_thread watchdog_threads = { .store = &softlockup_watchdog, diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index a6c8db1d62f65f..8afce47c31a1ee 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -764,15 +764,27 @@ config LOCKUP_DETECTOR The overhead should be minimal. A periodic hrtimer runs to generate interrupts and kick the watchdog task every 4 seconds. An NMI is generated every 10 seconds or so to check for hardlockups. + If NMIs are not available on the platform, every 12 seconds the + hrtimer interrupt on one cpu will be used to check for hardlockups + on the next cpu. The frequency of hrtimer and NMI events and the soft and hard lockup thresholds can be controlled through the sysctl watchdog_thresh. -config HARDLOCKUP_DETECTOR +config HARDLOCKUP_DETECTOR_NMI def_bool y depends on LOCKUP_DETECTOR && !HAVE_NMI_WATCHDOG depends on PERF_EVENTS && HAVE_PERF_EVENTS_NMI +config HARDLOCKUP_DETECTOR_OTHER_CPU + def_bool y + depends on LOCKUP_DETECTOR && SMP + depends on !HARDLOCKUP_DETECTOR_NMI && !HAVE_NMI_WATCHDOG + +config HARDLOCKUP_DETECTOR + def_bool y + depends on HARDLOCKUP_DETECTOR_NMI || HARDLOCKUP_DETECTOR_OTHER_CPU + config BOOTPARAM_HARDLOCKUP_PANIC bool "Panic (Reboot) On Hard Lockups" depends on HARDLOCKUP_DETECTOR @@ -923,6 +935,15 @@ config SCHED_INFO bool default n +config PANIC_ON_RT_THROTTLING + bool "Panic on RT throttling" + help + Say Y here to enable the kernel to panic when a realtime + runqueue is throttled. This may be useful for detecting + and debugging RT throttling issues. + + Say N if unsure. + config SCHEDSTATS bool "Collect scheduler statistics" depends on DEBUG_KERNEL && PROC_FS diff --git a/lib/iov_iter.c b/lib/iov_iter.c index f2bd21b93dfca4..efb0b4d267a15c 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -678,43 +678,50 @@ size_t iov_iter_copy_from_user_atomic(struct page *page, } EXPORT_SYMBOL(iov_iter_copy_from_user_atomic); +static inline void pipe_truncate(struct iov_iter *i) +{ + struct pipe_inode_info *pipe = i->pipe; + if (pipe->nrbufs) { + size_t off = i->iov_offset; + int idx = i->idx; + int nrbufs = (idx - pipe->curbuf) & (pipe->buffers - 1); + if (off) { + pipe->bufs[idx].len = off - pipe->bufs[idx].offset; + idx = next_idx(idx, pipe); + nrbufs++; + } + while (pipe->nrbufs > nrbufs) { + pipe_buf_release(pipe, &pipe->bufs[idx]); + idx = next_idx(idx, pipe); + pipe->nrbufs--; + } + } +} + static void pipe_advance(struct iov_iter *i, size_t size) { struct pipe_inode_info *pipe = i->pipe; - struct pipe_buffer *buf; - int idx = i->idx; - size_t off = i->iov_offset, orig_sz; - if (unlikely(i->count < size)) size = i->count; - orig_sz = size; - if (size) { + struct pipe_buffer *buf; + size_t off = i->iov_offset, left = size; + int idx = i->idx; if (off) /* make it relative to the beginning of buffer */ - size += off - pipe->bufs[idx].offset; + left += off - pipe->bufs[idx].offset; while (1) { buf = &pipe->bufs[idx]; - if (size <= buf->len) + if (left <= buf->len) break; - size -= buf->len; + left -= buf->len; idx = next_idx(idx, pipe); } - buf->len = size; i->idx = idx; - off = i->iov_offset = buf->offset + size; - } - if (off) - idx = next_idx(idx, pipe); - if (pipe->nrbufs) { - int unused = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1); - /* [curbuf,unused) is in use. Free [idx,unused) */ - while (idx != unused) { - pipe_buf_release(pipe, &pipe->bufs[idx]); - idx = next_idx(idx, pipe); - pipe->nrbufs--; - } + i->iov_offset = buf->offset + left; } - i->count -= orig_sz; + i->count -= size; + /* ... and discard everything past that point */ + pipe_truncate(i); } void iov_iter_advance(struct iov_iter *i, size_t size) @@ -774,6 +781,7 @@ void iov_iter_pipe(struct iov_iter *i, int direction, size_t count) { BUG_ON(direction != ITER_PIPE); + WARN_ON(pipe->nrbufs == pipe->buffers); i->type = direction; i->pipe = pipe; i->idx = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1); diff --git a/lib/swiotlb.c b/lib/swiotlb.c index 22e13a0e19d76a..ad1d2962d129cb 100644 --- a/lib/swiotlb.c +++ b/lib/swiotlb.c @@ -53,7 +53,7 @@ */ #define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT) -int swiotlb_force; +enum swiotlb_force swiotlb_force; /* * Used to do a quick range check in swiotlb_tbl_unmap_single and @@ -106,8 +106,12 @@ setup_io_tlb_npages(char *str) } if (*str == ',') ++str; - if (!strcmp(str, "force")) - swiotlb_force = 1; + if (!strcmp(str, "force")) { + swiotlb_force = SWIOTLB_FORCE; + } else if (!strcmp(str, "noforce")) { + swiotlb_force = SWIOTLB_NO_FORCE; + io_tlb_nslabs = 1; + } return 0; } @@ -541,8 +545,15 @@ static phys_addr_t map_single(struct device *hwdev, phys_addr_t phys, size_t size, enum dma_data_direction dir) { - dma_addr_t start_dma_addr = phys_to_dma(hwdev, io_tlb_start); + dma_addr_t start_dma_addr; + + if (swiotlb_force == SWIOTLB_NO_FORCE) { + dev_warn_ratelimited(hwdev, "Cannot do DMA to address %pa\n", + &phys); + return SWIOTLB_MAP_ERROR; + } + start_dma_addr = phys_to_dma(hwdev, io_tlb_start); return swiotlb_tbl_map_single(hwdev, start_dma_addr, phys, size, dir); } @@ -707,6 +718,9 @@ static void swiotlb_full(struct device *dev, size_t size, enum dma_data_direction dir, int do_panic) { + if (swiotlb_force == SWIOTLB_NO_FORCE) + return; + /* * Ran out of IOMMU space for this operation. This is very bad. * Unfortunately the drivers cannot handle this operation properly. @@ -749,7 +763,7 @@ dma_addr_t swiotlb_map_page(struct device *dev, struct page *page, * we can safely return the device addr and not worry about bounce * buffering it. */ - if (dma_capable(dev, dev_addr, size) && !swiotlb_force) + if (dma_capable(dev, dev_addr, size) && swiotlb_force != SWIOTLB_FORCE) return dev_addr; trace_swiotlb_bounced(dev, dev_addr, size, swiotlb_force); @@ -888,7 +902,7 @@ swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems, phys_addr_t paddr = sg_phys(sg); dma_addr_t dev_addr = phys_to_dma(hwdev, paddr); - if (swiotlb_force || + if (swiotlb_force == SWIOTLB_FORCE || !dma_capable(hwdev, dev_addr, sg->length)) { phys_addr_t map = map_single(hwdev, sg_phys(sg), sg->length, dir); diff --git a/mm/compaction.c b/mm/compaction.c index 0409a4ad6ea136..70e6bec46dc27f 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -634,22 +634,6 @@ isolate_freepages_range(struct compact_control *cc, return pfn; } -/* Update the number of anon and file isolated pages in the zone */ -static void acct_isolated(struct zone *zone, struct compact_control *cc) -{ - struct page *page; - unsigned int count[2] = { 0, }; - - if (list_empty(&cc->migratepages)) - return; - - list_for_each_entry(page, &cc->migratepages, lru) - count[!!page_is_file_cache(page)]++; - - mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_ANON, count[0]); - mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE, count[1]); -} - /* Similar to reclaim, but different enough that they don't share logic */ static bool too_many_isolated(struct zone *zone) { @@ -866,6 +850,8 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, /* Successfully isolated */ del_page_from_lru_list(page, lruvec, page_lru(page)); + inc_node_page_state(page, + NR_ISOLATED_ANON + page_is_file_cache(page)); isolate_success: list_add(&page->lru, &cc->migratepages); @@ -902,7 +888,6 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, spin_unlock_irqrestore(zone_lru_lock(zone), flags); locked = false; } - acct_isolated(zone, cc); putback_movable_pages(&cc->migratepages); cc->nr_migratepages = 0; cc->last_migrated_pfn = 0; @@ -988,7 +973,6 @@ isolate_migratepages_range(struct compact_control *cc, unsigned long start_pfn, if (cc->nr_migratepages == COMPACT_CLUSTER_MAX) break; } - acct_isolated(cc->zone, cc); return pfn; } @@ -1258,10 +1242,8 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone, low_pfn = isolate_migratepages_block(cc, low_pfn, block_end_pfn, isolate_mode); - if (!low_pfn || cc->contended) { - acct_isolated(zone, cc); + if (!low_pfn || cc->contended) return ISOLATE_ABORT; - } /* * Either we isolated something and proceed with migration. Or @@ -1271,7 +1253,6 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone, break; } - acct_isolated(zone, cc); /* Record where migration scanner will be restarted. */ cc->migrate_pfn = low_pfn; diff --git a/mm/filemap.c b/mm/filemap.c index 50b52fe51937ca..d8d7df82c69a9a 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -144,7 +144,7 @@ static int page_cache_tree_insert(struct address_space *mapping, workingset_node_pages_dec(node); /* Wakeup waiters for exceptional entry lock */ dax_wake_mapping_entry_waiter(mapping, page->index, - false); + true); } } radix_tree_replace_slot(slot, page); @@ -1686,7 +1686,7 @@ static ssize_t do_generic_file_read(struct file *filp, loff_t *ppos, int error = 0; if (unlikely(*ppos >= inode->i_sb->s_maxbytes)) - return -EINVAL; + return 0; iov_iter_truncate(iter, inode->i_sb->s_maxbytes); index = *ppos >> PAGE_SHIFT; @@ -1703,6 +1703,11 @@ static ssize_t do_generic_file_read(struct file *filp, loff_t *ppos, cond_resched(); find_page: + if (fatal_signal_pending(current)) { + error = -EINTR; + goto out; + } + page = find_get_page(mapping, index); if (!page) { page_cache_sync_readahead(mapping, diff --git a/mm/huge_memory.c b/mm/huge_memory.c index d4a6e40015128c..917555cf6be064 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -772,6 +772,12 @@ struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr, assert_spin_locked(pmd_lockptr(mm, pmd)); + /* + * When we COW a devmap PMD entry, we split it into PTEs, so we should + * not be in this function with `flags & FOLL_COW` set. + */ + WARN_ONCE(flags & FOLL_COW, "mm: In follow_devmap_pmd with FOLL_COW set"); + if (flags & FOLL_WRITE && !pmd_write(*pmd)) return NULL; @@ -872,15 +878,17 @@ void huge_pmd_set_accessed(struct fault_env *fe, pmd_t orig_pmd) { pmd_t entry; unsigned long haddr; + bool write = fe->flags & FAULT_FLAG_WRITE; fe->ptl = pmd_lock(fe->vma->vm_mm, fe->pmd); if (unlikely(!pmd_same(*fe->pmd, orig_pmd))) goto unlock; entry = pmd_mkyoung(orig_pmd); + if (write) + entry = pmd_mkdirty(entry); haddr = fe->address & HPAGE_PMD_MASK; - if (pmdp_set_access_flags(fe->vma, haddr, fe->pmd, entry, - fe->flags & FAULT_FLAG_WRITE)) + if (pmdp_set_access_flags(fe->vma, haddr, fe->pmd, entry, write)) update_mmu_cache_pmd(fe->vma, fe->address, fe->pmd); unlock: @@ -1116,6 +1124,16 @@ int do_huge_pmd_wp_page(struct fault_env *fe, pmd_t orig_pmd) return ret; } +/* + * FOLL_FORCE can write to even unwritable pmd's, but only + * after we've gone through a COW cycle and they are dirty. + */ +static inline bool can_follow_write_pmd(pmd_t pmd, unsigned int flags) +{ + return pmd_write(pmd) || + ((flags & FOLL_FORCE) && (flags & FOLL_COW) && pmd_dirty(pmd)); +} + struct page *follow_trans_huge_pmd(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmd, @@ -1126,7 +1144,7 @@ struct page *follow_trans_huge_pmd(struct vm_area_struct *vma, assert_spin_locked(pmd_lockptr(mm, pmd)); - if (flags & FOLL_WRITE && !pmd_write(*pmd)) + if (flags & FOLL_WRITE && !can_follow_write_pmd(*pmd, flags)) goto out; /* Avoid dumping huge zero page */ diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 418bf01a50ed1f..b6adedbafaf59c 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -1773,23 +1773,32 @@ static int gather_surplus_pages(struct hstate *h, int delta) } /* - * When releasing a hugetlb pool reservation, any surplus pages that were - * allocated to satisfy the reservation must be explicitly freed if they were - * never used. - * Called with hugetlb_lock held. + * This routine has two main purposes: + * 1) Decrement the reservation count (resv_huge_pages) by the value passed + * in unused_resv_pages. This corresponds to the prior adjustments made + * to the associated reservation map. + * 2) Free any unused surplus pages that may have been allocated to satisfy + * the reservation. As many as unused_resv_pages may be freed. + * + * Called with hugetlb_lock held. However, the lock could be dropped (and + * reacquired) during calls to cond_resched_lock. Whenever dropping the lock, + * we must make sure nobody else can claim pages we are in the process of + * freeing. Do this by ensuring resv_huge_page always is greater than the + * number of huge pages we plan to free when dropping the lock. */ static void return_unused_surplus_pages(struct hstate *h, unsigned long unused_resv_pages) { unsigned long nr_pages; - /* Uncommit the reservation */ - h->resv_huge_pages -= unused_resv_pages; - /* Cannot return gigantic pages currently */ if (hstate_is_gigantic(h)) - return; + goto out; + /* + * Part (or even all) of the reservation could have been backed + * by pre-allocated pages. Only free surplus pages. + */ nr_pages = min(unused_resv_pages, h->surplus_huge_pages); /* @@ -1799,12 +1808,22 @@ static void return_unused_surplus_pages(struct hstate *h, * when the nodes with surplus pages have no free pages. * free_pool_huge_page() will balance the the freed pages across the * on-line nodes with memory and will handle the hstate accounting. + * + * Note that we decrement resv_huge_pages as we free the pages. If + * we drop the lock, resv_huge_pages will still be sufficiently large + * to cover subsequent pages we may free. */ while (nr_pages--) { + h->resv_huge_pages--; + unused_resv_pages--; if (!free_pool_huge_page(h, &node_states[N_MEMORY], 1)) - break; + goto out; cond_resched_lock(&hugetlb_lock); } + +out: + /* Fully uncommit the reservation */ + h->resv_huge_pages -= unused_resv_pages; } @@ -3450,15 +3469,17 @@ static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma, * Keep the pte_same checks anyway to make transition from the mutex easier. */ static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma, - unsigned long address, pte_t *ptep, pte_t pte, - struct page *pagecache_page, spinlock_t *ptl) + unsigned long address, pte_t *ptep, + struct page *pagecache_page, spinlock_t *ptl) { + pte_t pte; struct hstate *h = hstate_vma(vma); struct page *old_page, *new_page; int ret = 0, outside_reserve = 0; unsigned long mmun_start; /* For mmu_notifiers */ unsigned long mmun_end; /* For mmu_notifiers */ + pte = huge_ptep_get(ptep); old_page = pte_page(pte); retry_avoidcopy: @@ -3733,7 +3754,7 @@ static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma, hugetlb_count_add(pages_per_huge_page(h), mm); if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) { /* Optimization, do the COW without a second fault */ - ret = hugetlb_cow(mm, vma, address, ptep, new_pte, page, ptl); + ret = hugetlb_cow(mm, vma, address, ptep, page, ptl); } spin_unlock(ptl); @@ -3888,8 +3909,8 @@ int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, if (flags & FAULT_FLAG_WRITE) { if (!huge_pte_write(entry)) { - ret = hugetlb_cow(mm, vma, address, ptep, entry, - pagecache_page, ptl); + ret = hugetlb_cow(mm, vma, address, ptep, + pagecache_page, ptl); goto out_put_page; } entry = huge_pte_mkdirty(entry); diff --git a/mm/init-mm.c b/mm/init-mm.c index a56a851908d245..975e49f00f34c1 100644 --- a/mm/init-mm.c +++ b/mm/init-mm.c @@ -6,6 +6,7 @@ #include #include +#include #include #include @@ -21,5 +22,6 @@ struct mm_struct init_mm = { .mmap_sem = __RWSEM_INITIALIZER(init_mm.mmap_sem), .page_table_lock = __SPIN_LOCK_UNLOCKED(init_mm.page_table_lock), .mmlist = LIST_HEAD_INIT(init_mm.mmlist), + .user_ns = &init_user_ns, INIT_MM_CONTEXT(init_mm) }; diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 87e1a7ca38463d..5d7c006373d305 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -1403,6 +1403,9 @@ static void collapse_shmem(struct mm_struct *mm, spin_lock_irq(&mapping->tree_lock); + slot = radix_tree_lookup_slot(&mapping->page_tree, index); + VM_BUG_ON_PAGE(page != radix_tree_deref_slot_protected(slot, + &mapping->tree_lock), page); VM_BUG_ON_PAGE(page_mapped(page), page); /* @@ -1426,6 +1429,7 @@ static void collapse_shmem(struct mm_struct *mm, radix_tree_replace_slot(slot, new_page + (index % HPAGE_PMD_NR)); + slot = radix_tree_iter_next(&iter); index++; continue; out_lru: @@ -1521,9 +1525,11 @@ static void collapse_shmem(struct mm_struct *mm, if (!page || iter.index < page->index) { if (!nr_none) break; - /* Put holes back where they were */ - radix_tree_replace_slot(slot, NULL); nr_none--; + /* Put holes back where they were */ + radix_tree_delete(&mapping->page_tree, + iter.index); + slot = radix_tree_iter_next(&iter); continue; } @@ -1537,6 +1543,7 @@ static void collapse_shmem(struct mm_struct *mm, putback_lru_page(page); unlock_page(page); spin_lock_irq(&mapping->tree_lock); + slot = radix_tree_iter_next(&iter); } VM_BUG_ON(nr_none); spin_unlock_irq(&mapping->tree_lock); diff --git a/mm/madvise.c b/mm/madvise.c index 93fb63e88b5ef5..279627aace24aa 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -108,7 +108,7 @@ static long madvise_behavior(struct vm_area_struct *vma, pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT); *prev = vma_merge(mm, *prev, start, end, new_flags, vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma), - vma->vm_userfaultfd_ctx); + vma->vm_userfaultfd_ctx, vma_get_anon_name(vma)); if (*prev) { vma = *prev; goto success; diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 0f870ba43942e7..4c6ade54d83356 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -625,8 +625,8 @@ static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg, unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg, int nid, unsigned int lru_mask) { + struct lruvec *lruvec = mem_cgroup_lruvec(NODE_DATA(nid), memcg); unsigned long nr = 0; - struct mem_cgroup_per_node *mz; enum lru_list lru; VM_BUG_ON((unsigned)nid >= nr_node_ids); @@ -634,8 +634,7 @@ unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg, for_each_lru(lru) { if (!(BIT(lru) & lru_mask)) continue; - mz = mem_cgroup_nodeinfo(memcg, nid); - nr += mz->lru_size[lru]; + nr += mem_cgroup_get_lru_size(lruvec, lru); } return nr; } @@ -1002,6 +1001,7 @@ struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct pglist_data *pgd * mem_cgroup_update_lru_size - account for adding or removing an lru page * @lruvec: mem_cgroup per zone lru vector * @lru: index of lru list the page is sitting on + * @zid: zone id of the accounted pages * @nr_pages: positive when adding or negative when removing * * This function must be called under lru_lock, just before a page is added @@ -1009,27 +1009,25 @@ struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct pglist_data *pgd * so as to allow it to check that lru_size 0 is consistent with list_empty). */ void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru, - int nr_pages) + int zid, int nr_pages) { struct mem_cgroup_per_node *mz; unsigned long *lru_size; long size; - bool empty; if (mem_cgroup_disabled()) return; mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec); - lru_size = mz->lru_size + lru; - empty = list_empty(lruvec->lists + lru); + lru_size = &mz->lru_zone_size[zid][lru]; if (nr_pages < 0) *lru_size += nr_pages; size = *lru_size; - if (WARN_ONCE(size < 0 || empty != !size, - "%s(%p, %d, %d): lru_size %ld but %sempty\n", - __func__, lruvec, lru, nr_pages, size, empty ? "" : "not ")) { + if (WARN_ONCE(size < 0, + "%s(%p, %d, %d): lru_size %ld\n", + __func__, lruvec, lru, nr_pages, size)) { VM_BUG_ON(1); *lru_size = 0; } @@ -4362,9 +4360,9 @@ static int mem_cgroup_do_precharge(unsigned long count) return ret; } - /* Try charges one by one with reclaim */ + /* Try charges one by one with reclaim, but do not retry */ while (count--) { - ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_NORETRY, 1); + ret = try_charge(mc.to, GFP_KERNEL | __GFP_NORETRY, 1); if (ret) return ret; mc.precharge++; diff --git a/mm/memory.c b/mm/memory.c index e18c57bdc75c4c..cbb1e5e5f79159 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -3868,7 +3868,7 @@ EXPORT_SYMBOL_GPL(generic_access_phys); * Access another process' address space as given in mm. If non-NULL, use the * given task for page fault accounting. */ -static int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm, +int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm, unsigned long addr, void *buf, int len, unsigned int gup_flags) { struct vm_area_struct *vma; diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index cad4b9125695cf..ede137345a99f4 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -1033,36 +1033,39 @@ static void node_states_set_node(int node, struct memory_notify *arg) node_set_state(node, N_MEMORY); } -int zone_can_shift(unsigned long pfn, unsigned long nr_pages, - enum zone_type target) +bool zone_can_shift(unsigned long pfn, unsigned long nr_pages, + enum zone_type target, int *zone_shift) { struct zone *zone = page_zone(pfn_to_page(pfn)); enum zone_type idx = zone_idx(zone); int i; + *zone_shift = 0; + if (idx < target) { /* pages must be at end of current zone */ if (pfn + nr_pages != zone_end_pfn(zone)) - return 0; + return false; /* no zones in use between current zone and target */ for (i = idx + 1; i < target; i++) if (zone_is_initialized(zone - idx + i)) - return 0; + return false; } if (target < idx) { /* pages must be at beginning of current zone */ if (pfn != zone->zone_start_pfn) - return 0; + return false; /* no zones in use between current zone and target */ for (i = target + 1; i < idx; i++) if (zone_is_initialized(zone - idx + i)) - return 0; + return false; } - return target - idx; + *zone_shift = target - idx; + return true; } /* Must be protected by mem_hotplug_begin() */ @@ -1089,10 +1092,13 @@ int __ref online_pages(unsigned long pfn, unsigned long nr_pages, int online_typ !can_online_high_movable(zone)) return -EINVAL; - if (online_type == MMOP_ONLINE_KERNEL) - zone_shift = zone_can_shift(pfn, nr_pages, ZONE_NORMAL); - else if (online_type == MMOP_ONLINE_MOVABLE) - zone_shift = zone_can_shift(pfn, nr_pages, ZONE_MOVABLE); + if (online_type == MMOP_ONLINE_KERNEL) { + if (!zone_can_shift(pfn, nr_pages, ZONE_NORMAL, &zone_shift)) + return -EINVAL; + } else if (online_type == MMOP_ONLINE_MOVABLE) { + if (!zone_can_shift(pfn, nr_pages, ZONE_MOVABLE, &zone_shift)) + return -EINVAL; + } zone = move_pfn_range(zone_shift, pfn, pfn + nr_pages); if (!zone) @@ -1477,17 +1483,20 @@ bool is_mem_section_removable(unsigned long start_pfn, unsigned long nr_pages) } /* - * Confirm all pages in a range [start, end) is belongs to the same zone. + * Confirm all pages in a range [start, end) belong to the same zone. + * When true, return its valid [start, end). */ -int test_pages_in_a_zone(unsigned long start_pfn, unsigned long end_pfn) +int test_pages_in_a_zone(unsigned long start_pfn, unsigned long end_pfn, + unsigned long *valid_start, unsigned long *valid_end) { unsigned long pfn, sec_end_pfn; + unsigned long start, end; struct zone *zone = NULL; struct page *page; int i; - for (pfn = start_pfn, sec_end_pfn = SECTION_ALIGN_UP(start_pfn); + for (pfn = start_pfn, sec_end_pfn = SECTION_ALIGN_UP(start_pfn + 1); pfn < end_pfn; - pfn = sec_end_pfn + 1, sec_end_pfn += PAGES_PER_SECTION) { + pfn = sec_end_pfn, sec_end_pfn += PAGES_PER_SECTION) { /* Make sure the memory section is present first */ if (!present_section_nr(pfn_to_section_nr(pfn))) continue; @@ -1503,10 +1512,20 @@ int test_pages_in_a_zone(unsigned long start_pfn, unsigned long end_pfn) page = pfn_to_page(pfn + i); if (zone && page_zone(page) != zone) return 0; + if (!zone) + start = pfn + i; zone = page_zone(page); + end = pfn + MAX_ORDER_NR_PAGES; } } - return 1; + + if (zone) { + *valid_start = start; + *valid_end = end; + return 1; + } else { + return 0; + } } /* @@ -1853,6 +1872,7 @@ static int __ref __offline_pages(unsigned long start_pfn, long offlined_pages; int ret, drain, retry_max, node; unsigned long flags; + unsigned long valid_start, valid_end; struct zone *zone; struct memory_notify arg; @@ -1863,10 +1883,10 @@ static int __ref __offline_pages(unsigned long start_pfn, return -EINVAL; /* This makes hotplug much easier...and readable. we assume this for now. .*/ - if (!test_pages_in_a_zone(start_pfn, end_pfn)) + if (!test_pages_in_a_zone(start_pfn, end_pfn, &valid_start, &valid_end)) return -EINVAL; - zone = page_zone(pfn_to_page(start_pfn)); + zone = page_zone(pfn_to_page(valid_start)); node = zone_to_nid(zone); nr_pages = end_pfn - start_pfn; diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 0b859af06b87df..0adc6f905db224 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -752,7 +752,8 @@ static int mbind_range(struct mm_struct *mm, unsigned long start, ((vmstart - vma->vm_start) >> PAGE_SHIFT); prev = vma_merge(mm, prev, vmstart, vmend, vma->vm_flags, vma->anon_vma, vma->vm_file, pgoff, - new_pol, vma->vm_userfaultfd_ctx); + new_pol, vma->vm_userfaultfd_ctx, + vma_get_anon_name(vma)); if (prev) { vma = prev; next = vma->vm_next; @@ -2023,8 +2024,8 @@ alloc_pages_vma(gfp_t gfp, int order, struct vm_area_struct *vma, nmask = policy_nodemask(gfp, pol); zl = policy_zonelist(gfp, pol, node); - mpol_cond_put(pol); page = __alloc_pages_nodemask(gfp, order, zl, nmask); + mpol_cond_put(pol); out: if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie))) goto retry_cpuset; diff --git a/mm/migrate.c b/mm/migrate.c index 99250aee1ac166..66ce6b490b13a9 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -168,8 +168,6 @@ void putback_movable_pages(struct list_head *l) continue; } list_del(&page->lru); - dec_node_page_state(page, NR_ISOLATED_ANON + - page_is_file_cache(page)); /* * We isolated non-lru movable page so here we can use * __PageMovable because LRU page's mapping cannot have @@ -186,6 +184,8 @@ void putback_movable_pages(struct list_head *l) put_page(page); } else { putback_lru_page(page); + dec_node_page_state(page, NR_ISOLATED_ANON + + page_is_file_cache(page)); } } } @@ -1121,8 +1121,15 @@ static ICE_noinline int unmap_and_move(new_page_t get_new_page, * restored. */ list_del(&page->lru); - dec_node_page_state(page, NR_ISOLATED_ANON + - page_is_file_cache(page)); + + /* + * Compaction can migrate also non-LRU pages which are + * not accounted to NR_ISOLATED_*. They can be recognized + * as __PageMovable + */ + if (likely(!__PageMovable(page))) + dec_node_page_state(page, NR_ISOLATED_ANON + + page_is_file_cache(page)); } /* diff --git a/mm/mlock.c b/mm/mlock.c index cdbed8aaa4268c..facf6e70e93f0d 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -529,7 +529,7 @@ static int mlock_fixup(struct vm_area_struct *vma, struct vm_area_struct **prev, pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT); *prev = vma_merge(mm, *prev, start, end, newflags, vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma), - vma->vm_userfaultfd_ctx); + vma->vm_userfaultfd_ctx, vma_get_anon_name(vma)); if (*prev) { vma = *prev; goto success; diff --git a/mm/mmap.c b/mm/mmap.c index 1af87c14183d08..143d62f5fc5c7c 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -956,7 +956,8 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, */ static inline int is_mergeable_vma(struct vm_area_struct *vma, struct file *file, unsigned long vm_flags, - struct vm_userfaultfd_ctx vm_userfaultfd_ctx) + struct vm_userfaultfd_ctx vm_userfaultfd_ctx, + const char __user *anon_name) { /* * VM_SOFTDIRTY should not prevent from VMA merging, if we @@ -974,6 +975,8 @@ static inline int is_mergeable_vma(struct vm_area_struct *vma, return 0; if (!is_mergeable_vm_userfaultfd_ctx(vma, vm_userfaultfd_ctx)) return 0; + if (vma_get_anon_name(vma) != anon_name) + return 0; return 1; } @@ -1006,9 +1009,10 @@ static int can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags, struct anon_vma *anon_vma, struct file *file, pgoff_t vm_pgoff, - struct vm_userfaultfd_ctx vm_userfaultfd_ctx) + struct vm_userfaultfd_ctx vm_userfaultfd_ctx, + const char __user *anon_name) { - if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx) && + if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name) && is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) { if (vma->vm_pgoff == vm_pgoff) return 1; @@ -1027,9 +1031,10 @@ static int can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags, struct anon_vma *anon_vma, struct file *file, pgoff_t vm_pgoff, - struct vm_userfaultfd_ctx vm_userfaultfd_ctx) + struct vm_userfaultfd_ctx vm_userfaultfd_ctx, + const char __user *anon_name) { - if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx) && + if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name) && is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) { pgoff_t vm_pglen; vm_pglen = vma_pages(vma); @@ -1040,9 +1045,9 @@ can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags, } /* - * Given a mapping request (addr,end,vm_flags,file,pgoff), figure out - * whether that can be merged with its predecessor or its successor. - * Or both (it neatly fills a hole). + * Given a mapping request (addr,end,vm_flags,file,pgoff,anon_name), + * figure out whether that can be merged with its predecessor or its + * successor. Or both (it neatly fills a hole). * * In most cases - when called for mmap, brk or mremap - [addr,end) is * certain not to be mapped by the time vma_merge is called; but when @@ -1084,7 +1089,8 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm, unsigned long end, unsigned long vm_flags, struct anon_vma *anon_vma, struct file *file, pgoff_t pgoff, struct mempolicy *policy, - struct vm_userfaultfd_ctx vm_userfaultfd_ctx) + struct vm_userfaultfd_ctx vm_userfaultfd_ctx, + const char __user *anon_name) { pgoff_t pglen = (end - addr) >> PAGE_SHIFT; struct vm_area_struct *area, *next; @@ -1117,7 +1123,8 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm, mpol_equal(vma_policy(prev), policy) && can_vma_merge_after(prev, vm_flags, anon_vma, file, pgoff, - vm_userfaultfd_ctx)) { + vm_userfaultfd_ctx, + anon_name)) { /* * OK, it can. Can we now merge in the successor as well? */ @@ -1126,7 +1133,8 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm, can_vma_merge_before(next, vm_flags, anon_vma, file, pgoff+pglen, - vm_userfaultfd_ctx) && + vm_userfaultfd_ctx, + anon_name) && is_mergeable_anon_vma(prev->anon_vma, next->anon_vma, NULL)) { /* cases 1, 6 */ @@ -1149,7 +1157,8 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm, mpol_equal(policy, vma_policy(next)) && can_vma_merge_before(next, vm_flags, anon_vma, file, pgoff+pglen, - vm_userfaultfd_ctx)) { + vm_userfaultfd_ctx, + anon_name)) { if (prev && addr < prev->vm_end) /* case 4 */ err = __vma_adjust(prev, prev->vm_start, addr, prev->vm_pgoff, NULL, next); @@ -1627,7 +1636,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr, * Can we just expand an old mapping? */ vma = vma_merge(mm, prev, addr, addr + len, vm_flags, - NULL, file, pgoff, NULL, NULL_VM_UFFD_CTX); + NULL, file, pgoff, NULL, NULL_VM_UFFD_CTX, NULL); if (vma) goto out; @@ -2663,6 +2672,7 @@ int do_munmap(struct mm_struct *mm, unsigned long start, size_t len) return 0; } +EXPORT_SYMBOL(do_munmap); int vm_munmap(unsigned long start, size_t len) { @@ -2858,7 +2868,7 @@ static int do_brk(unsigned long addr, unsigned long request) /* Can we just expand an old private anonymous mapping? */ vma = vma_merge(mm, prev, addr, addr + len, flags, - NULL, NULL, pgoff, NULL, NULL_VM_UFFD_CTX); + NULL, NULL, pgoff, NULL, NULL_VM_UFFD_CTX, NULL); if (vma) goto out; @@ -3019,7 +3029,7 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, return NULL; /* should never get here */ new_vma = vma_merge(mm, prev, addr, addr + len, vma->vm_flags, vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma), - vma->vm_userfaultfd_ctx); + vma->vm_userfaultfd_ctx, vma_get_anon_name(vma)); if (new_vma) { /* * Source vma may have been merged into new_vma diff --git a/mm/mprotect.c b/mm/mprotect.c index 11936526b08b8c..2c4083682f5cf0 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -302,7 +302,7 @@ mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev, pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT); *pprev = vma_merge(mm, *pprev, start, end, newflags, vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma), - vma->vm_userfaultfd_ctx); + vma->vm_userfaultfd_ctx, vma_get_anon_name(vma)); if (*pprev) { vma = *pprev; VM_WARN_ON((vma->vm_flags ^ newflags) & ~VM_SOFTDIRTY); diff --git a/mm/nommu.c b/mm/nommu.c index 8b8faaf2a9e95c..44265e00b701eb 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -1808,7 +1808,7 @@ void filemap_map_pages(struct fault_env *fe, } EXPORT_SYMBOL(filemap_map_pages); -static int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm, +int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm, unsigned long addr, void *buf, int len, unsigned int gup_flags) { struct vm_area_struct *vma; diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 6de9440e3ae2d9..f4a02e240fb68a 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -2192,7 +2192,7 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order, unsigned long count, struct list_head *list, int migratetype, bool cold) { - int i; + int i, alloced = 0; spin_lock(&zone->lock); for (i = 0; i < count; ++i) { @@ -2217,13 +2217,21 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order, else list_add_tail(&page->lru, list); list = &page->lru; + alloced++; if (is_migrate_cma(get_pcppage_migratetype(page))) __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, -(1 << order)); } + + /* + * i pages were removed from the buddy list even if some leak due + * to check_pcp_refill failing so adjust NR_FREE_PAGES based + * on i. Do not confuse with 'alloced' which is the number of + * pages added to the pcp list. + */ __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order)); spin_unlock(&zone->lock); - return i; + return alloced; } #ifdef CONFIG_NUMA @@ -3494,12 +3502,13 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, struct page *page = NULL; unsigned int alloc_flags; unsigned long did_some_progress; - enum compact_priority compact_priority = DEF_COMPACT_PRIORITY; + enum compact_priority compact_priority; enum compact_result compact_result; - int compaction_retries = 0; - int no_progress_loops = 0; + int compaction_retries; + int no_progress_loops; unsigned long alloc_start = jiffies; unsigned int stall_timeout = 10 * HZ; + unsigned int cpuset_mems_cookie; /* * In the slowpath, we sanity check order to avoid ever trying to @@ -3520,6 +3529,23 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM))) gfp_mask &= ~__GFP_ATOMIC; +retry_cpuset: + compaction_retries = 0; + no_progress_loops = 0; + compact_priority = DEF_COMPACT_PRIORITY; + cpuset_mems_cookie = read_mems_allowed_begin(); + /* + * We need to recalculate the starting point for the zonelist iterator + * because we might have used different nodemask in the fast path, or + * there was a cpuset modification and we are retrying - otherwise we + * could end up iterating over non-eligible zones endlessly. + */ + ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, + ac->high_zoneidx, ac->nodemask); + if (!ac->preferred_zoneref->zone) + goto nopage; + + /* * The fast path uses conservative alloc_flags to succeed only until * kswapd needs to be woken up, and to avoid the cost of setting up @@ -3679,6 +3705,13 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, &compaction_retries)) goto retry; + /* + * It's possible we raced with cpuset update so the OOM would be + * premature (see below the nopage: label for full explanation). + */ + if (read_mems_allowed_retry(cpuset_mems_cookie)) + goto retry_cpuset; + /* Reclaim has failed us, start killing things */ page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress); if (page) @@ -3691,6 +3724,16 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, } nopage: + /* + * When updating a task's mems_allowed or mempolicy nodemask, it is + * possible to race with parallel threads in such a way that our + * allocation can fail while the mask is being updated. If we are about + * to fail, check if the cpuset changed during allocation and if so, + * retry. + */ + if (read_mems_allowed_retry(cpuset_mems_cookie)) + goto retry_cpuset; + warn_alloc(gfp_mask, "page allocation failure: order:%u", order); got_pg: @@ -3705,7 +3748,6 @@ __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, struct zonelist *zonelist, nodemask_t *nodemask) { struct page *page; - unsigned int cpuset_mems_cookie; unsigned int alloc_flags = ALLOC_WMARK_LOW; gfp_t alloc_mask = gfp_mask; /* The gfp_t that was actually used for allocation */ struct alloc_context ac = { @@ -3742,9 +3784,6 @@ __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, if (IS_ENABLED(CONFIG_CMA) && ac.migratetype == MIGRATE_MOVABLE) alloc_flags |= ALLOC_CMA; -retry_cpuset: - cpuset_mems_cookie = read_mems_allowed_begin(); - /* Dirty zone balancing only done in the fast path */ ac.spread_dirty_pages = (gfp_mask & __GFP_WRITE); @@ -3755,8 +3794,13 @@ __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, */ ac.preferred_zoneref = first_zones_zonelist(ac.zonelist, ac.high_zoneidx, ac.nodemask); - if (!ac.preferred_zoneref) { + if (!ac.preferred_zoneref->zone) { page = NULL; + /* + * This might be due to race with cpuset_current_mems_allowed + * update, so make sure we retry with original nodemask in the + * slow path. + */ goto no_zone; } @@ -3765,6 +3809,7 @@ __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, if (likely(page)) goto out; +no_zone: /* * Runtime PM, block IO and its error handling path can deadlock * because I/O on the device might not complete. @@ -3776,21 +3821,10 @@ __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, * Restore the original nodemask if it was potentially replaced with * &cpuset_current_mems_allowed to optimize the fast-path attempt. */ - if (cpusets_enabled()) + if (unlikely(ac.nodemask != nodemask)) ac.nodemask = nodemask; - page = __alloc_pages_slowpath(alloc_mask, order, &ac); -no_zone: - /* - * When updating a task's mems_allowed, it is possible to race with - * parallel threads in such a way that an allocation can fail while - * the mask is being updated. If a page allocation is about to fail, - * check if the cpuset changed during allocation and if so, retry. - */ - if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie))) { - alloc_mask = gfp_mask; - goto retry_cpuset; - } + page = __alloc_pages_slowpath(alloc_mask, order, &ac); out: if (memcg_kmem_enabled() && (gfp_mask & __GFP_ACCOUNT) && page && diff --git a/mm/shmem.c b/mm/shmem.c index 9d32e1cb9f38f7..e9c2b6e0ed930d 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -4052,6 +4052,14 @@ struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags } EXPORT_SYMBOL_GPL(shmem_file_setup); +void shmem_set_file(struct vm_area_struct *vma, struct file *file) +{ + if (vma->vm_file) + fput(vma->vm_file); + vma->vm_file = file; + vma->vm_ops = &shmem_vm_ops; +} + /** * shmem_zero_setup - setup a shared anonymous mapping * @vma: the vma to be mmapped is prepared by do_mmap_pgoff @@ -4071,10 +4079,7 @@ int shmem_zero_setup(struct vm_area_struct *vma) if (IS_ERR(file)) return PTR_ERR(file); - if (vma->vm_file) - fput(vma->vm_file); - vma->vm_file = file; - vma->vm_ops = &shmem_vm_ops; + shmem_set_file(vma, file); if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE) && ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) < diff --git a/mm/slab.c b/mm/slab.c index 0b0550ca85b40c..bd878f051a3b9b 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -2475,7 +2475,6 @@ union freelist_init_state { unsigned int pos; unsigned int *list; unsigned int count; - unsigned int rand; }; struct rnd_state rnd_state; }; @@ -2501,8 +2500,7 @@ static bool freelist_state_initialize(union freelist_init_state *state, } else { state->list = cachep->random_seq; state->count = count; - state->pos = 0; - state->rand = rand; + state->pos = rand % count; ret = true; } return ret; @@ -2511,7 +2509,9 @@ static bool freelist_state_initialize(union freelist_init_state *state, /* Get the next entry on the list and randomize it using a random shift */ static freelist_idx_t next_random_slot(union freelist_init_state *state) { - return (state->list[state->pos++] + state->rand) % state->count; + if (state->pos >= state->count) + state->pos = 0; + return state->list[state->pos++]; } /* Swap two freelist entries */ diff --git a/mm/slub.c b/mm/slub.c index 2b3e740609e92e..7aa0e97af92837 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -1419,6 +1419,10 @@ static int init_cache_random_seq(struct kmem_cache *s) int err; unsigned long i, count = oo_objects(s->oo); + /* Bailout if already initialised */ + if (s->random_seq) + return 0; + err = cache_random_seq_create(s, count, GFP_KERNEL); if (err) { pr_err("SLUB: Unable to initialize free list for %s\n", diff --git a/mm/swapfile.c b/mm/swapfile.c index f30438970cd176..d76b2a18f044a0 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -943,11 +943,25 @@ bool reuse_swap_page(struct page *page, int *total_mapcount) count = page_trans_huge_mapcount(page, total_mapcount); if (count <= 1 && PageSwapCache(page)) { count += page_swapcount(page); - if (count == 1 && !PageWriteback(page)) { + if (count != 1) + goto out; + if (!PageWriteback(page)) { delete_from_swap_cache(page); SetPageDirty(page); + } else { + swp_entry_t entry; + struct swap_info_struct *p; + + entry.val = page_private(page); + p = swap_info_get(entry); + if (p->flags & SWP_STABLE_WRITES) { + spin_unlock(&p->lock); + return false; + } + spin_unlock(&p->lock); } } +out: return count <= 1; } @@ -2449,6 +2463,10 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags) error = -ENOMEM; goto bad_swap; } + + if (bdi_cap_stable_pages_required(inode_to_bdi(inode))) + p->flags |= SWP_STABLE_WRITES; + if (p->bdev && blk_queue_nonrot(bdev_get_queue(p->bdev))) { int cpu; diff --git a/mm/vmscan.c b/mm/vmscan.c index d75cdf36073018..fa30010a52773f 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -242,6 +242,16 @@ unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru) return node_page_state(lruvec_pgdat(lruvec), NR_LRU_BASE + lru); } +unsigned long lruvec_zone_lru_size(struct lruvec *lruvec, enum lru_list lru, + int zone_idx) +{ + if (!mem_cgroup_disabled()) + return mem_cgroup_get_zone_lru_size(lruvec, lru, zone_idx); + + return zone_page_state(&lruvec_pgdat(lruvec)->node_zones[zone_idx], + NR_ZONE_LRU_BASE + lru); +} + /* * Add a shrinker callback to be called from the vm. */ @@ -291,6 +301,7 @@ static unsigned long do_shrink_slab(struct shrink_control *shrinkctl, int nid = shrinkctl->nid; long batch_size = shrinker->batch ? shrinker->batch : SHRINK_BATCH; + long scanned = 0, next_deferred; freeable = shrinker->count_objects(shrinker, shrinkctl); if (freeable == 0) @@ -312,7 +323,9 @@ static unsigned long do_shrink_slab(struct shrink_control *shrinkctl, pr_err("shrink_slab: %pF negative objects to delete nr=%ld\n", shrinker->scan_objects, total_scan); total_scan = freeable; - } + next_deferred = nr; + } else + next_deferred = total_scan; /* * We need to avoid excessive windup on filesystem shrinkers @@ -369,17 +382,22 @@ static unsigned long do_shrink_slab(struct shrink_control *shrinkctl, count_vm_events(SLABS_SCANNED, nr_to_scan); total_scan -= nr_to_scan; + scanned += nr_to_scan; cond_resched(); } + if (next_deferred >= scanned) + next_deferred -= scanned; + else + next_deferred = 0; /* * move the unused scan count back into the shrinker in a * manner that handles concurrent updates. If we exhausted the * scan, there is no need to do an update. */ - if (total_scan > 0) - new_nr = atomic_long_add_return(total_scan, + if (next_deferred > 0) + new_nr = atomic_long_add_return(next_deferred, &shrinker->nr_deferred[nid]); else new_nr = atomic_long_read(&shrinker->nr_deferred[nid]); @@ -1374,8 +1392,7 @@ int __isolate_lru_page(struct page *page, isolate_mode_t mode) * be complete before mem_cgroup_update_lru_size due to a santity check. */ static __always_inline void update_lru_sizes(struct lruvec *lruvec, - enum lru_list lru, unsigned long *nr_zone_taken, - unsigned long nr_taken) + enum lru_list lru, unsigned long *nr_zone_taken) { int zid; @@ -1384,11 +1401,11 @@ static __always_inline void update_lru_sizes(struct lruvec *lruvec, continue; __update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]); - } - #ifdef CONFIG_MEMCG - mem_cgroup_update_lru_size(lruvec, lru, -nr_taken); + mem_cgroup_update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]); #endif + } + } /* @@ -1493,7 +1510,7 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan, *nr_scanned = scan; trace_mm_vmscan_lru_isolate(sc->reclaim_idx, sc->order, nr_to_scan, scan, nr_taken, mode, is_file_lru(lru)); - update_lru_sizes(lruvec, lru, nr_zone_taken, nr_taken); + update_lru_sizes(lruvec, lru, nr_zone_taken); return nr_taken; } @@ -2039,10 +2056,8 @@ static bool inactive_list_is_low(struct lruvec *lruvec, bool file, if (!managed_zone(zone)) continue; - inactive_zone = zone_page_state(zone, - NR_ZONE_LRU_BASE + (file * LRU_FILE)); - active_zone = zone_page_state(zone, - NR_ZONE_LRU_BASE + (file * LRU_FILE) + LRU_ACTIVE); + inactive_zone = lruvec_zone_lru_size(lruvec, file * LRU_FILE, zid); + active_zone = lruvec_zone_lru_size(lruvec, (file * LRU_FILE) + LRU_ACTIVE, zid); inactive -= min(inactive, inactive_zone); active -= min(active, active_zone); diff --git a/mm/zswap.c b/mm/zswap.c index 275b22cc8df435..dbef27822a987f 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -78,7 +78,13 @@ static u64 zswap_duplicate_entry; /* Enable/disable zswap (disabled by default) */ static bool zswap_enabled; -module_param_named(enabled, zswap_enabled, bool, 0644); +static int zswap_enabled_param_set(const char *, + const struct kernel_param *); +static struct kernel_param_ops zswap_enabled_param_ops = { + .set = zswap_enabled_param_set, + .get = param_get_bool, +}; +module_param_cb(enabled, &zswap_enabled_param_ops, &zswap_enabled, 0644); /* Crypto compressor to use */ #define ZSWAP_COMPRESSOR_DEFAULT "lzo" @@ -176,6 +182,9 @@ static atomic_t zswap_pools_count = ATOMIC_INIT(0); /* used by param callback function */ static bool zswap_init_started; +/* fatal error during init */ +static bool zswap_init_failed; + /********************************* * helpers and fwd declarations **********************************/ @@ -706,6 +715,11 @@ static int __zswap_param_set(const char *val, const struct kernel_param *kp, char *s = strstrip((char *)val); int ret; + if (zswap_init_failed) { + pr_err("can't set param, initialization failed\n"); + return -ENODEV; + } + /* no change required */ if (!strcmp(s, *(char **)kp->arg)) return 0; @@ -785,6 +799,17 @@ static int zswap_zpool_param_set(const char *val, return __zswap_param_set(val, kp, NULL, zswap_compressor); } +static int zswap_enabled_param_set(const char *val, + const struct kernel_param *kp) +{ + if (zswap_init_failed) { + pr_err("can't enable, initialization failed\n"); + return -ENODEV; + } + + return param_set_bool(val, kp); +} + /********************************* * writeback code **********************************/ @@ -1271,6 +1296,9 @@ static int __init init_zswap(void) dstmem_fail: zswap_entry_cache_destroy(); cache_fail: + /* if built-in, we aren't unloaded on failure; don't allow use */ + zswap_init_failed = true; + zswap_enabled = false; return -ENOMEM; } /* must be late so crypto has time to come up */ diff --git a/net/Kconfig b/net/Kconfig index 7b6cd340b72bc5..1870b35a7aba23 100644 --- a/net/Kconfig +++ b/net/Kconfig @@ -89,6 +89,12 @@ source "net/netlabel/Kconfig" endif # if INET +config ANDROID_PARANOID_NETWORK + bool "Only allow certain groups to create sockets" + default y + help + none + config NETWORK_SECMARK bool "Security Marking" help diff --git a/net/ax25/ax25_subr.c b/net/ax25/ax25_subr.c index 655a7d4c96e1e2..983f0b5e14f1fa 100644 --- a/net/ax25/ax25_subr.c +++ b/net/ax25/ax25_subr.c @@ -264,7 +264,7 @@ void ax25_disconnect(ax25_cb *ax25, int reason) { ax25_clear_queues(ax25); - if (!sock_flag(ax25->sk, SOCK_DESTROY)) + if (!ax25->sk || !sock_flag(ax25->sk, SOCK_DESTROY)) ax25_stop_heartbeat(ax25); ax25_stop_t1timer(ax25); ax25_stop_t2timer(ax25); diff --git a/net/bluetooth/af_bluetooth.c b/net/bluetooth/af_bluetooth.c index 1aff2da9bc7478..4b325250236fd8 100644 --- a/net/bluetooth/af_bluetooth.c +++ b/net/bluetooth/af_bluetooth.c @@ -106,11 +106,40 @@ void bt_sock_unregister(int proto) } EXPORT_SYMBOL(bt_sock_unregister); +#ifdef CONFIG_PARANOID_NETWORK +static inline int current_has_bt_admin(void) +{ + return !current_euid(); +} + +static inline int current_has_bt(void) +{ + return current_has_bt_admin(); +} +# else +static inline int current_has_bt_admin(void) +{ + return 1; +} + +static inline int current_has_bt(void) +{ + return 1; +} +#endif + static int bt_sock_create(struct net *net, struct socket *sock, int proto, int kern) { int err; + if (proto == BTPROTO_RFCOMM || proto == BTPROTO_SCO || + proto == BTPROTO_L2CAP) { + if (!current_has_bt()) + return -EPERM; + } else if (!current_has_bt_admin()) + return -EPERM; + if (net != &init_net) return -EAFNOSUPPORT; diff --git a/net/bridge/br_device.c b/net/bridge/br_device.c index 89a687f3c0a33a..fcaa48402aa30c 100644 --- a/net/bridge/br_device.c +++ b/net/bridge/br_device.c @@ -48,16 +48,17 @@ netdev_tx_t br_dev_xmit(struct sk_buff *skb, struct net_device *dev) return NETDEV_TX_OK; } - u64_stats_update_begin(&brstats->syncp); - brstats->tx_packets++; - brstats->tx_bytes += skb->len; - u64_stats_update_end(&brstats->syncp); - BR_INPUT_SKB_CB(skb)->brdev = dev; skb_reset_mac_header(skb); skb_pull(skb, ETH_HLEN); + u64_stats_update_begin(&brstats->syncp); + brstats->tx_packets++; + /* Exclude ETH_HLEN from byte stats for consistency with Rx chain */ + brstats->tx_bytes += skb->len; + u64_stats_update_end(&brstats->syncp); + if (!br_allowed_ingress(br, br_vlan_group_rcu(br), skb, &vid)) goto out; diff --git a/net/bridge/br_netfilter_hooks.c b/net/bridge/br_netfilter_hooks.c index 2fe9345c140710..7fbdbae58e65e0 100644 --- a/net/bridge/br_netfilter_hooks.c +++ b/net/bridge/br_netfilter_hooks.c @@ -399,7 +399,7 @@ static int br_nf_pre_routing_finish(struct net *net, struct sock *sk, struct sk_ br_nf_hook_thresh(NF_BR_PRE_ROUTING, net, sk, skb, skb->dev, NULL, - br_nf_pre_routing_finish); + br_nf_pre_routing_finish_bridge); return 0; } ether_addr_copy(eth_hdr(skb)->h_dest, dev->dev_addr); diff --git a/net/bridge/br_netlink.c b/net/bridge/br_netlink.c index e99037c6f7b7b7..04741064a173f4 100644 --- a/net/bridge/br_netlink.c +++ b/net/bridge/br_netlink.c @@ -781,20 +781,6 @@ static int br_validate(struct nlattr *tb[], struct nlattr *data[]) return 0; } -static int br_dev_newlink(struct net *src_net, struct net_device *dev, - struct nlattr *tb[], struct nlattr *data[]) -{ - struct net_bridge *br = netdev_priv(dev); - - if (tb[IFLA_ADDRESS]) { - spin_lock_bh(&br->lock); - br_stp_change_bridge_id(br, nla_data(tb[IFLA_ADDRESS])); - spin_unlock_bh(&br->lock); - } - - return register_netdevice(dev); -} - static int br_port_slave_changelink(struct net_device *brdev, struct net_device *dev, struct nlattr *tb[], @@ -1093,6 +1079,25 @@ static int br_changelink(struct net_device *brdev, struct nlattr *tb[], return 0; } +static int br_dev_newlink(struct net *src_net, struct net_device *dev, + struct nlattr *tb[], struct nlattr *data[]) +{ + struct net_bridge *br = netdev_priv(dev); + int err; + + if (tb[IFLA_ADDRESS]) { + spin_lock_bh(&br->lock); + br_stp_change_bridge_id(br, nla_data(tb[IFLA_ADDRESS])); + spin_unlock_bh(&br->lock); + } + + err = br_changelink(dev, tb, data); + if (err) + return err; + + return register_netdevice(dev); +} + static size_t br_get_size(const struct net_device *brdev) { return nla_total_size(sizeof(u32)) + /* IFLA_BR_FORWARD_DELAY */ diff --git a/net/can/bcm.c b/net/can/bcm.c index 436a7537e6a9d3..5e9ed5ec2860bd 100644 --- a/net/can/bcm.c +++ b/net/can/bcm.c @@ -734,14 +734,23 @@ static struct bcm_op *bcm_find_op(struct list_head *ops, static void bcm_remove_op(struct bcm_op *op) { - hrtimer_cancel(&op->timer); - hrtimer_cancel(&op->thrtimer); - - if (op->tsklet.func) - tasklet_kill(&op->tsklet); + if (op->tsklet.func) { + while (test_bit(TASKLET_STATE_SCHED, &op->tsklet.state) || + test_bit(TASKLET_STATE_RUN, &op->tsklet.state) || + hrtimer_active(&op->timer)) { + hrtimer_cancel(&op->timer); + tasklet_kill(&op->tsklet); + } + } - if (op->thrtsklet.func) - tasklet_kill(&op->thrtsklet); + if (op->thrtsklet.func) { + while (test_bit(TASKLET_STATE_SCHED, &op->thrtsklet.state) || + test_bit(TASKLET_STATE_RUN, &op->thrtsklet.state) || + hrtimer_active(&op->thrtimer)) { + hrtimer_cancel(&op->thrtimer); + tasklet_kill(&op->thrtsklet); + } + } if ((op->frames) && (op->frames != &op->sframe)) kfree(op->frames); diff --git a/net/ceph/auth_x.c b/net/ceph/auth_x.c index a0905f04bd13f3..b216131915e74a 100644 --- a/net/ceph/auth_x.c +++ b/net/ceph/auth_x.c @@ -39,56 +39,58 @@ static int ceph_x_should_authenticate(struct ceph_auth_client *ac) return need != 0; } +static int ceph_x_encrypt_offset(void) +{ + return sizeof(u32) + sizeof(struct ceph_x_encrypt_header); +} + static int ceph_x_encrypt_buflen(int ilen) { - return sizeof(struct ceph_x_encrypt_header) + ilen + 16 + - sizeof(u32); + return ceph_x_encrypt_offset() + ilen + 16; } -static int ceph_x_encrypt(struct ceph_crypto_key *secret, - void *ibuf, int ilen, void *obuf, size_t olen) +static int ceph_x_encrypt(struct ceph_crypto_key *secret, void *buf, + int buf_len, int plaintext_len) { - struct ceph_x_encrypt_header head = { - .struct_v = 1, - .magic = cpu_to_le64(CEPHX_ENC_MAGIC) - }; - size_t len = olen - sizeof(u32); + struct ceph_x_encrypt_header *hdr = buf + sizeof(u32); + int ciphertext_len; int ret; - ret = ceph_encrypt2(secret, obuf + sizeof(u32), &len, - &head, sizeof(head), ibuf, ilen); + hdr->struct_v = 1; + hdr->magic = cpu_to_le64(CEPHX_ENC_MAGIC); + + ret = ceph_crypt(secret, true, buf + sizeof(u32), buf_len - sizeof(u32), + plaintext_len + sizeof(struct ceph_x_encrypt_header), + &ciphertext_len); if (ret) return ret; - ceph_encode_32(&obuf, len); - return len + sizeof(u32); + + ceph_encode_32(&buf, ciphertext_len); + return sizeof(u32) + ciphertext_len; } -static int ceph_x_decrypt(struct ceph_crypto_key *secret, - void **p, void *end, void **obuf, size_t olen) +static int ceph_x_decrypt(struct ceph_crypto_key *secret, void **p, void *end) { - struct ceph_x_encrypt_header head; - size_t head_len = sizeof(head); - int len, ret; - - len = ceph_decode_32(p); - if (*p + len > end) - return -EINVAL; + struct ceph_x_encrypt_header *hdr = *p + sizeof(u32); + int ciphertext_len, plaintext_len; + int ret; - dout("ceph_x_decrypt len %d\n", len); - if (*obuf == NULL) { - *obuf = kmalloc(len, GFP_NOFS); - if (!*obuf) - return -ENOMEM; - olen = len; - } + ceph_decode_32_safe(p, end, ciphertext_len, e_inval); + ceph_decode_need(p, end, ciphertext_len, e_inval); - ret = ceph_decrypt2(secret, &head, &head_len, *obuf, &olen, *p, len); + ret = ceph_crypt(secret, false, *p, end - *p, ciphertext_len, + &plaintext_len); if (ret) return ret; - if (head.struct_v != 1 || le64_to_cpu(head.magic) != CEPHX_ENC_MAGIC) + + if (hdr->struct_v != 1 || le64_to_cpu(hdr->magic) != CEPHX_ENC_MAGIC) return -EPERM; - *p += len; - return olen; + + *p += ciphertext_len; + return plaintext_len - sizeof(struct ceph_x_encrypt_header); + +e_inval: + return -EINVAL; } /* @@ -143,13 +145,10 @@ static int process_one_ticket(struct ceph_auth_client *ac, int type; u8 tkt_struct_v, blob_struct_v; struct ceph_x_ticket_handler *th; - void *dbuf = NULL; void *dp, *dend; int dlen; char is_enc; struct timespec validity; - struct ceph_crypto_key old_key; - void *ticket_buf = NULL; void *tp, *tpend; void **ptp; struct ceph_crypto_key new_session_key; @@ -174,20 +173,17 @@ static int process_one_ticket(struct ceph_auth_client *ac, } /* blob for me */ - dlen = ceph_x_decrypt(secret, p, end, &dbuf, 0); - if (dlen <= 0) { - ret = dlen; + dp = *p + ceph_x_encrypt_offset(); + ret = ceph_x_decrypt(secret, p, end); + if (ret < 0) goto out; - } - dout(" decrypted %d bytes\n", dlen); - dp = dbuf; - dend = dp + dlen; + dout(" decrypted %d bytes\n", ret); + dend = dp + ret; tkt_struct_v = ceph_decode_8(&dp); if (tkt_struct_v != 1) goto bad; - memcpy(&old_key, &th->session_key, sizeof(old_key)); ret = ceph_crypto_key_decode(&new_session_key, &dp, dend); if (ret) goto out; @@ -203,15 +199,13 @@ static int process_one_ticket(struct ceph_auth_client *ac, ceph_decode_8_safe(p, end, is_enc, bad); if (is_enc) { /* encrypted */ - dout(" encrypted ticket\n"); - dlen = ceph_x_decrypt(&old_key, p, end, &ticket_buf, 0); - if (dlen < 0) { - ret = dlen; + tp = *p + ceph_x_encrypt_offset(); + ret = ceph_x_decrypt(&th->session_key, p, end); + if (ret < 0) goto out; - } - tp = ticket_buf; + dout(" encrypted ticket, decrypted %d bytes\n", ret); ptp = &tp; - tpend = *ptp + dlen; + tpend = tp + ret; } else { /* unencrypted */ ptp = p; @@ -242,8 +236,6 @@ static int process_one_ticket(struct ceph_auth_client *ac, xi->have_keys |= th->service; out: - kfree(ticket_buf); - kfree(dbuf); return ret; bad: @@ -294,7 +286,7 @@ static int ceph_x_build_authorizer(struct ceph_auth_client *ac, { int maxlen; struct ceph_x_authorize_a *msg_a; - struct ceph_x_authorize_b msg_b; + struct ceph_x_authorize_b *msg_b; void *p, *end; int ret; int ticket_blob_len = @@ -308,8 +300,8 @@ static int ceph_x_build_authorizer(struct ceph_auth_client *ac, if (ret) goto out_au; - maxlen = sizeof(*msg_a) + sizeof(msg_b) + - ceph_x_encrypt_buflen(ticket_blob_len); + maxlen = sizeof(*msg_a) + ticket_blob_len + + ceph_x_encrypt_buflen(sizeof(*msg_b)); dout(" need len %d\n", maxlen); if (au->buf && au->buf->alloc_len < maxlen) { ceph_buffer_put(au->buf); @@ -343,18 +335,19 @@ static int ceph_x_build_authorizer(struct ceph_auth_client *ac, p += ticket_blob_len; end = au->buf->vec.iov_base + au->buf->vec.iov_len; + msg_b = p + ceph_x_encrypt_offset(); + msg_b->struct_v = 1; get_random_bytes(&au->nonce, sizeof(au->nonce)); - msg_b.struct_v = 1; - msg_b.nonce = cpu_to_le64(au->nonce); - ret = ceph_x_encrypt(&au->session_key, &msg_b, sizeof(msg_b), - p, end - p); + msg_b->nonce = cpu_to_le64(au->nonce); + ret = ceph_x_encrypt(&au->session_key, p, end - p, sizeof(*msg_b)); if (ret < 0) goto out_au; + p += ret; + WARN_ON(p > end); au->buf->vec.iov_len = p - au->buf->vec.iov_base; dout(" built authorizer nonce %llx len %d\n", au->nonce, (int)au->buf->vec.iov_len); - BUG_ON(au->buf->vec.iov_len > maxlen); return 0; out_au: @@ -452,8 +445,9 @@ static int ceph_x_build_request(struct ceph_auth_client *ac, if (need & CEPH_ENTITY_TYPE_AUTH) { struct ceph_x_authenticate *auth = (void *)(head + 1); void *p = auth + 1; - struct ceph_x_challenge_blob tmp; - char tmp_enc[40]; + void *enc_buf = xi->auth_authorizer.enc_buf; + struct ceph_x_challenge_blob *blob = enc_buf + + ceph_x_encrypt_offset(); u64 *u; if (p > end) @@ -464,16 +458,16 @@ static int ceph_x_build_request(struct ceph_auth_client *ac, /* encrypt and hash */ get_random_bytes(&auth->client_challenge, sizeof(u64)); - tmp.client_challenge = auth->client_challenge; - tmp.server_challenge = cpu_to_le64(xi->server_challenge); - ret = ceph_x_encrypt(&xi->secret, &tmp, sizeof(tmp), - tmp_enc, sizeof(tmp_enc)); + blob->client_challenge = auth->client_challenge; + blob->server_challenge = cpu_to_le64(xi->server_challenge); + ret = ceph_x_encrypt(&xi->secret, enc_buf, CEPHX_AU_ENC_BUF_LEN, + sizeof(*blob)); if (ret < 0) return ret; auth->struct_v = 1; auth->key = 0; - for (u = (u64 *)tmp_enc; u + 1 <= (u64 *)(tmp_enc + ret); u++) + for (u = (u64 *)enc_buf; u + 1 <= (u64 *)(enc_buf + ret); u++) auth->key ^= *(__le64 *)u; dout(" server_challenge %llx client_challenge %llx key %llx\n", xi->server_challenge, le64_to_cpu(auth->client_challenge), @@ -600,8 +594,8 @@ static int ceph_x_create_authorizer( auth->authorizer = (struct ceph_authorizer *) au; auth->authorizer_buf = au->buf->vec.iov_base; auth->authorizer_buf_len = au->buf->vec.iov_len; - auth->authorizer_reply_buf = au->reply_buf; - auth->authorizer_reply_buf_len = sizeof (au->reply_buf); + auth->authorizer_reply_buf = au->enc_buf; + auth->authorizer_reply_buf_len = CEPHX_AU_ENC_BUF_LEN; auth->sign_message = ac->ops->sign_message; auth->check_message_signature = ac->ops->check_message_signature; @@ -632,24 +626,22 @@ static int ceph_x_verify_authorizer_reply(struct ceph_auth_client *ac, struct ceph_authorizer *a, size_t len) { struct ceph_x_authorizer *au = (void *)a; - int ret = 0; - struct ceph_x_authorize_reply reply; - void *preply = &reply; - void *p = au->reply_buf; - void *end = p + sizeof(au->reply_buf); + void *p = au->enc_buf; + struct ceph_x_authorize_reply *reply = p + ceph_x_encrypt_offset(); + int ret; - ret = ceph_x_decrypt(&au->session_key, &p, end, &preply, sizeof(reply)); + ret = ceph_x_decrypt(&au->session_key, &p, p + CEPHX_AU_ENC_BUF_LEN); if (ret < 0) return ret; - if (ret != sizeof(reply)) + if (ret != sizeof(*reply)) return -EPERM; - if (au->nonce + 1 != le64_to_cpu(reply.nonce_plus_one)) + if (au->nonce + 1 != le64_to_cpu(reply->nonce_plus_one)) ret = -EPERM; else ret = 0; dout("verify_authorizer_reply nonce %llx got %llx ret %d\n", - au->nonce, le64_to_cpu(reply.nonce_plus_one), ret); + au->nonce, le64_to_cpu(reply->nonce_plus_one), ret); return ret; } @@ -704,35 +696,48 @@ static void ceph_x_invalidate_authorizer(struct ceph_auth_client *ac, invalidate_ticket(ac, CEPH_ENTITY_TYPE_AUTH); } -static int calcu_signature(struct ceph_x_authorizer *au, - struct ceph_msg *msg, __le64 *sig) +static int calc_signature(struct ceph_x_authorizer *au, struct ceph_msg *msg, + __le64 *psig) { + void *enc_buf = au->enc_buf; + struct { + __le32 len; + __le32 header_crc; + __le32 front_crc; + __le32 middle_crc; + __le32 data_crc; + } __packed *sigblock = enc_buf + ceph_x_encrypt_offset(); int ret; - char tmp_enc[40]; - __le32 tmp[5] = { - cpu_to_le32(16), msg->hdr.crc, msg->footer.front_crc, - msg->footer.middle_crc, msg->footer.data_crc, - }; - ret = ceph_x_encrypt(&au->session_key, &tmp, sizeof(tmp), - tmp_enc, sizeof(tmp_enc)); + + sigblock->len = cpu_to_le32(4*sizeof(u32)); + sigblock->header_crc = msg->hdr.crc; + sigblock->front_crc = msg->footer.front_crc; + sigblock->middle_crc = msg->footer.middle_crc; + sigblock->data_crc = msg->footer.data_crc; + ret = ceph_x_encrypt(&au->session_key, enc_buf, CEPHX_AU_ENC_BUF_LEN, + sizeof(*sigblock)); if (ret < 0) return ret; - *sig = *(__le64*)(tmp_enc + 4); + + *psig = *(__le64 *)(enc_buf + sizeof(u32)); return 0; } static int ceph_x_sign_message(struct ceph_auth_handshake *auth, struct ceph_msg *msg) { + __le64 sig; int ret; if (ceph_test_opt(from_msgr(msg->con->msgr), NOMSGSIGN)) return 0; - ret = calcu_signature((struct ceph_x_authorizer *)auth->authorizer, - msg, &msg->footer.sig); - if (ret < 0) + ret = calc_signature((struct ceph_x_authorizer *)auth->authorizer, + msg, &sig); + if (ret) return ret; + + msg->footer.sig = sig; msg->footer.flags |= CEPH_MSG_FOOTER_SIGNED; return 0; } @@ -746,9 +751,9 @@ static int ceph_x_check_message_signature(struct ceph_auth_handshake *auth, if (ceph_test_opt(from_msgr(msg->con->msgr), NOMSGSIGN)) return 0; - ret = calcu_signature((struct ceph_x_authorizer *)auth->authorizer, - msg, &sig_check); - if (ret < 0) + ret = calc_signature((struct ceph_x_authorizer *)auth->authorizer, + msg, &sig_check); + if (ret) return ret; if (sig_check == msg->footer.sig) return 0; diff --git a/net/ceph/auth_x.h b/net/ceph/auth_x.h index 21a5af904bae75..48e9ad41bd2aa7 100644 --- a/net/ceph/auth_x.h +++ b/net/ceph/auth_x.h @@ -24,6 +24,7 @@ struct ceph_x_ticket_handler { unsigned long renew_after, expires; }; +#define CEPHX_AU_ENC_BUF_LEN 128 /* big enough for encrypted blob */ struct ceph_x_authorizer { struct ceph_authorizer base; @@ -32,7 +33,7 @@ struct ceph_x_authorizer { unsigned int service; u64 nonce; u64 secret_id; - char reply_buf[128]; /* big enough for encrypted blob */ + char enc_buf[CEPHX_AU_ENC_BUF_LEN] __aligned(8); }; struct ceph_x_info { diff --git a/net/ceph/crypto.c b/net/ceph/crypto.c index db2847ac5f1229..292e33bd916e65 100644 --- a/net/ceph/crypto.c +++ b/net/ceph/crypto.c @@ -13,14 +13,60 @@ #include #include "crypto.h" +/* + * Set ->key and ->tfm. The rest of the key should be filled in before + * this function is called. + */ +static int set_secret(struct ceph_crypto_key *key, void *buf) +{ + unsigned int noio_flag; + int ret; + + key->key = NULL; + key->tfm = NULL; + + switch (key->type) { + case CEPH_CRYPTO_NONE: + return 0; /* nothing to do */ + case CEPH_CRYPTO_AES: + break; + default: + return -ENOTSUPP; + } + + WARN_ON(!key->len); + key->key = kmemdup(buf, key->len, GFP_NOIO); + if (!key->key) { + ret = -ENOMEM; + goto fail; + } + + /* crypto_alloc_skcipher() allocates with GFP_KERNEL */ + noio_flag = memalloc_noio_save(); + key->tfm = crypto_alloc_skcipher("cbc(aes)", 0, CRYPTO_ALG_ASYNC); + memalloc_noio_restore(noio_flag); + if (IS_ERR(key->tfm)) { + ret = PTR_ERR(key->tfm); + key->tfm = NULL; + goto fail; + } + + ret = crypto_skcipher_setkey(key->tfm, key->key, key->len); + if (ret) + goto fail; + + return 0; + +fail: + ceph_crypto_key_destroy(key); + return ret; +} + int ceph_crypto_key_clone(struct ceph_crypto_key *dst, const struct ceph_crypto_key *src) { memcpy(dst, src, sizeof(struct ceph_crypto_key)); - dst->key = kmemdup(src->key, src->len, GFP_NOFS); - if (!dst->key) - return -ENOMEM; - return 0; + return set_secret(dst, src->key); } int ceph_crypto_key_encode(struct ceph_crypto_key *key, void **p, void *end) @@ -37,16 +83,16 @@ int ceph_crypto_key_encode(struct ceph_crypto_key *key, void **p, void *end) int ceph_crypto_key_decode(struct ceph_crypto_key *key, void **p, void *end) { + int ret; + ceph_decode_need(p, end, 2*sizeof(u16) + sizeof(key->created), bad); key->type = ceph_decode_16(p); ceph_decode_copy(p, &key->created, sizeof(key->created)); key->len = ceph_decode_16(p); ceph_decode_need(p, end, key->len, bad); - key->key = kmalloc(key->len, GFP_NOFS); - if (!key->key) - return -ENOMEM; - ceph_decode_copy(p, key->key, key->len); - return 0; + ret = set_secret(key, *p); + *p += key->len; + return ret; bad: dout("failed to decode crypto key\n"); @@ -80,9 +126,14 @@ int ceph_crypto_key_unarmor(struct ceph_crypto_key *key, const char *inkey) return 0; } -static struct crypto_skcipher *ceph_crypto_alloc_cipher(void) +void ceph_crypto_key_destroy(struct ceph_crypto_key *key) { - return crypto_alloc_skcipher("cbc(aes)", 0, CRYPTO_ALG_ASYNC); + if (key) { + kfree(key->key); + key->key = NULL; + crypto_free_skcipher(key->tfm); + key->tfm = NULL; + } } static const u8 *aes_iv = (u8 *)CEPH_AES_IV; @@ -157,372 +208,82 @@ static void teardown_sgtable(struct sg_table *sgt) sg_free_table(sgt); } -static int ceph_aes_encrypt(const void *key, int key_len, - void *dst, size_t *dst_len, - const void *src, size_t src_len) -{ - struct scatterlist sg_in[2], prealloc_sg; - struct sg_table sg_out; - struct crypto_skcipher *tfm = ceph_crypto_alloc_cipher(); - SKCIPHER_REQUEST_ON_STACK(req, tfm); - int ret; - char iv[AES_BLOCK_SIZE]; - size_t zero_padding = (0x10 - (src_len & 0x0f)); - char pad[16]; - - if (IS_ERR(tfm)) - return PTR_ERR(tfm); - - memset(pad, zero_padding, zero_padding); - - *dst_len = src_len + zero_padding; - - sg_init_table(sg_in, 2); - sg_set_buf(&sg_in[0], src, src_len); - sg_set_buf(&sg_in[1], pad, zero_padding); - ret = setup_sgtable(&sg_out, &prealloc_sg, dst, *dst_len); - if (ret) - goto out_tfm; - - crypto_skcipher_setkey((void *)tfm, key, key_len); - memcpy(iv, aes_iv, AES_BLOCK_SIZE); - - skcipher_request_set_tfm(req, tfm); - skcipher_request_set_callback(req, 0, NULL, NULL); - skcipher_request_set_crypt(req, sg_in, sg_out.sgl, - src_len + zero_padding, iv); - - /* - print_hex_dump(KERN_ERR, "enc key: ", DUMP_PREFIX_NONE, 16, 1, - key, key_len, 1); - print_hex_dump(KERN_ERR, "enc src: ", DUMP_PREFIX_NONE, 16, 1, - src, src_len, 1); - print_hex_dump(KERN_ERR, "enc pad: ", DUMP_PREFIX_NONE, 16, 1, - pad, zero_padding, 1); - */ - ret = crypto_skcipher_encrypt(req); - skcipher_request_zero(req); - if (ret < 0) { - pr_err("ceph_aes_crypt failed %d\n", ret); - goto out_sg; - } - /* - print_hex_dump(KERN_ERR, "enc out: ", DUMP_PREFIX_NONE, 16, 1, - dst, *dst_len, 1); - */ - -out_sg: - teardown_sgtable(&sg_out); -out_tfm: - crypto_free_skcipher(tfm); - return ret; -} - -static int ceph_aes_encrypt2(const void *key, int key_len, void *dst, - size_t *dst_len, - const void *src1, size_t src1_len, - const void *src2, size_t src2_len) -{ - struct scatterlist sg_in[3], prealloc_sg; - struct sg_table sg_out; - struct crypto_skcipher *tfm = ceph_crypto_alloc_cipher(); - SKCIPHER_REQUEST_ON_STACK(req, tfm); - int ret; - char iv[AES_BLOCK_SIZE]; - size_t zero_padding = (0x10 - ((src1_len + src2_len) & 0x0f)); - char pad[16]; - - if (IS_ERR(tfm)) - return PTR_ERR(tfm); - - memset(pad, zero_padding, zero_padding); - - *dst_len = src1_len + src2_len + zero_padding; - - sg_init_table(sg_in, 3); - sg_set_buf(&sg_in[0], src1, src1_len); - sg_set_buf(&sg_in[1], src2, src2_len); - sg_set_buf(&sg_in[2], pad, zero_padding); - ret = setup_sgtable(&sg_out, &prealloc_sg, dst, *dst_len); - if (ret) - goto out_tfm; - - crypto_skcipher_setkey((void *)tfm, key, key_len); - memcpy(iv, aes_iv, AES_BLOCK_SIZE); - - skcipher_request_set_tfm(req, tfm); - skcipher_request_set_callback(req, 0, NULL, NULL); - skcipher_request_set_crypt(req, sg_in, sg_out.sgl, - src1_len + src2_len + zero_padding, iv); - - /* - print_hex_dump(KERN_ERR, "enc key: ", DUMP_PREFIX_NONE, 16, 1, - key, key_len, 1); - print_hex_dump(KERN_ERR, "enc src1: ", DUMP_PREFIX_NONE, 16, 1, - src1, src1_len, 1); - print_hex_dump(KERN_ERR, "enc src2: ", DUMP_PREFIX_NONE, 16, 1, - src2, src2_len, 1); - print_hex_dump(KERN_ERR, "enc pad: ", DUMP_PREFIX_NONE, 16, 1, - pad, zero_padding, 1); - */ - ret = crypto_skcipher_encrypt(req); - skcipher_request_zero(req); - if (ret < 0) { - pr_err("ceph_aes_crypt2 failed %d\n", ret); - goto out_sg; - } - /* - print_hex_dump(KERN_ERR, "enc out: ", DUMP_PREFIX_NONE, 16, 1, - dst, *dst_len, 1); - */ - -out_sg: - teardown_sgtable(&sg_out); -out_tfm: - crypto_free_skcipher(tfm); - return ret; -} - -static int ceph_aes_decrypt(const void *key, int key_len, - void *dst, size_t *dst_len, - const void *src, size_t src_len) +static int ceph_aes_crypt(const struct ceph_crypto_key *key, bool encrypt, + void *buf, int buf_len, int in_len, int *pout_len) { - struct sg_table sg_in; - struct scatterlist sg_out[2], prealloc_sg; - struct crypto_skcipher *tfm = ceph_crypto_alloc_cipher(); - SKCIPHER_REQUEST_ON_STACK(req, tfm); - char pad[16]; - char iv[AES_BLOCK_SIZE]; + SKCIPHER_REQUEST_ON_STACK(req, key->tfm); + struct sg_table sgt; + struct scatterlist prealloc_sg; + char iv[AES_BLOCK_SIZE] __aligned(8); + int pad_byte = AES_BLOCK_SIZE - (in_len & (AES_BLOCK_SIZE - 1)); + int crypt_len = encrypt ? in_len + pad_byte : in_len; int ret; - int last_byte; - - if (IS_ERR(tfm)) - return PTR_ERR(tfm); - sg_init_table(sg_out, 2); - sg_set_buf(&sg_out[0], dst, *dst_len); - sg_set_buf(&sg_out[1], pad, sizeof(pad)); - ret = setup_sgtable(&sg_in, &prealloc_sg, src, src_len); + WARN_ON(crypt_len > buf_len); + if (encrypt) + memset(buf + in_len, pad_byte, pad_byte); + ret = setup_sgtable(&sgt, &prealloc_sg, buf, crypt_len); if (ret) - goto out_tfm; + return ret; - crypto_skcipher_setkey((void *)tfm, key, key_len); memcpy(iv, aes_iv, AES_BLOCK_SIZE); - - skcipher_request_set_tfm(req, tfm); + skcipher_request_set_tfm(req, key->tfm); skcipher_request_set_callback(req, 0, NULL, NULL); - skcipher_request_set_crypt(req, sg_in.sgl, sg_out, - src_len, iv); + skcipher_request_set_crypt(req, sgt.sgl, sgt.sgl, crypt_len, iv); /* - print_hex_dump(KERN_ERR, "dec key: ", DUMP_PREFIX_NONE, 16, 1, - key, key_len, 1); - print_hex_dump(KERN_ERR, "dec in: ", DUMP_PREFIX_NONE, 16, 1, - src, src_len, 1); + print_hex_dump(KERN_ERR, "key: ", DUMP_PREFIX_NONE, 16, 1, + key->key, key->len, 1); + print_hex_dump(KERN_ERR, " in: ", DUMP_PREFIX_NONE, 16, 1, + buf, crypt_len, 1); */ - ret = crypto_skcipher_decrypt(req); - skcipher_request_zero(req); - if (ret < 0) { - pr_err("ceph_aes_decrypt failed %d\n", ret); - goto out_sg; - } - - if (src_len <= *dst_len) - last_byte = ((char *)dst)[src_len - 1]; + if (encrypt) + ret = crypto_skcipher_encrypt(req); else - last_byte = pad[src_len - *dst_len - 1]; - if (last_byte <= 16 && src_len >= last_byte) { - *dst_len = src_len - last_byte; - } else { - pr_err("ceph_aes_decrypt got bad padding %d on src len %d\n", - last_byte, (int)src_len); - return -EPERM; /* bad padding */ - } - /* - print_hex_dump(KERN_ERR, "dec out: ", DUMP_PREFIX_NONE, 16, 1, - dst, *dst_len, 1); - */ - -out_sg: - teardown_sgtable(&sg_in); -out_tfm: - crypto_free_skcipher(tfm); - return ret; -} - -static int ceph_aes_decrypt2(const void *key, int key_len, - void *dst1, size_t *dst1_len, - void *dst2, size_t *dst2_len, - const void *src, size_t src_len) -{ - struct sg_table sg_in; - struct scatterlist sg_out[3], prealloc_sg; - struct crypto_skcipher *tfm = ceph_crypto_alloc_cipher(); - SKCIPHER_REQUEST_ON_STACK(req, tfm); - char pad[16]; - char iv[AES_BLOCK_SIZE]; - int ret; - int last_byte; - - if (IS_ERR(tfm)) - return PTR_ERR(tfm); - - sg_init_table(sg_out, 3); - sg_set_buf(&sg_out[0], dst1, *dst1_len); - sg_set_buf(&sg_out[1], dst2, *dst2_len); - sg_set_buf(&sg_out[2], pad, sizeof(pad)); - ret = setup_sgtable(&sg_in, &prealloc_sg, src, src_len); - if (ret) - goto out_tfm; - - crypto_skcipher_setkey((void *)tfm, key, key_len); - memcpy(iv, aes_iv, AES_BLOCK_SIZE); - - skcipher_request_set_tfm(req, tfm); - skcipher_request_set_callback(req, 0, NULL, NULL); - skcipher_request_set_crypt(req, sg_in.sgl, sg_out, - src_len, iv); - - /* - print_hex_dump(KERN_ERR, "dec key: ", DUMP_PREFIX_NONE, 16, 1, - key, key_len, 1); - print_hex_dump(KERN_ERR, "dec in: ", DUMP_PREFIX_NONE, 16, 1, - src, src_len, 1); - */ - ret = crypto_skcipher_decrypt(req); + ret = crypto_skcipher_decrypt(req); skcipher_request_zero(req); - if (ret < 0) { - pr_err("ceph_aes_decrypt failed %d\n", ret); - goto out_sg; - } - - if (src_len <= *dst1_len) - last_byte = ((char *)dst1)[src_len - 1]; - else if (src_len <= *dst1_len + *dst2_len) - last_byte = ((char *)dst2)[src_len - *dst1_len - 1]; - else - last_byte = pad[src_len - *dst1_len - *dst2_len - 1]; - if (last_byte <= 16 && src_len >= last_byte) { - src_len -= last_byte; - } else { - pr_err("ceph_aes_decrypt got bad padding %d on src len %d\n", - last_byte, (int)src_len); - return -EPERM; /* bad padding */ - } - - if (src_len < *dst1_len) { - *dst1_len = src_len; - *dst2_len = 0; - } else { - *dst2_len = src_len - *dst1_len; + if (ret) { + pr_err("%s %scrypt failed: %d\n", __func__, + encrypt ? "en" : "de", ret); + goto out_sgt; } /* - print_hex_dump(KERN_ERR, "dec out1: ", DUMP_PREFIX_NONE, 16, 1, - dst1, *dst1_len, 1); - print_hex_dump(KERN_ERR, "dec out2: ", DUMP_PREFIX_NONE, 16, 1, - dst2, *dst2_len, 1); + print_hex_dump(KERN_ERR, "out: ", DUMP_PREFIX_NONE, 16, 1, + buf, crypt_len, 1); */ -out_sg: - teardown_sgtable(&sg_in); -out_tfm: - crypto_free_skcipher(tfm); - return ret; -} - - -int ceph_decrypt(struct ceph_crypto_key *secret, void *dst, size_t *dst_len, - const void *src, size_t src_len) -{ - switch (secret->type) { - case CEPH_CRYPTO_NONE: - if (*dst_len < src_len) - return -ERANGE; - memcpy(dst, src, src_len); - *dst_len = src_len; - return 0; - - case CEPH_CRYPTO_AES: - return ceph_aes_decrypt(secret->key, secret->len, dst, - dst_len, src, src_len); - - default: - return -EINVAL; - } -} - -int ceph_decrypt2(struct ceph_crypto_key *secret, - void *dst1, size_t *dst1_len, - void *dst2, size_t *dst2_len, - const void *src, size_t src_len) -{ - size_t t; - - switch (secret->type) { - case CEPH_CRYPTO_NONE: - if (*dst1_len + *dst2_len < src_len) - return -ERANGE; - t = min(*dst1_len, src_len); - memcpy(dst1, src, t); - *dst1_len = t; - src += t; - src_len -= t; - if (src_len) { - t = min(*dst2_len, src_len); - memcpy(dst2, src, t); - *dst2_len = t; + if (encrypt) { + *pout_len = crypt_len; + } else { + pad_byte = *(char *)(buf + in_len - 1); + if (pad_byte > 0 && pad_byte <= AES_BLOCK_SIZE && + in_len >= pad_byte) { + *pout_len = in_len - pad_byte; + } else { + pr_err("%s got bad padding %d on in_len %d\n", + __func__, pad_byte, in_len); + ret = -EPERM; + goto out_sgt; } - return 0; - - case CEPH_CRYPTO_AES: - return ceph_aes_decrypt2(secret->key, secret->len, - dst1, dst1_len, dst2, dst2_len, - src, src_len); - - default: - return -EINVAL; } -} - -int ceph_encrypt(struct ceph_crypto_key *secret, void *dst, size_t *dst_len, - const void *src, size_t src_len) -{ - switch (secret->type) { - case CEPH_CRYPTO_NONE: - if (*dst_len < src_len) - return -ERANGE; - memcpy(dst, src, src_len); - *dst_len = src_len; - return 0; - case CEPH_CRYPTO_AES: - return ceph_aes_encrypt(secret->key, secret->len, dst, - dst_len, src, src_len); - - default: - return -EINVAL; - } +out_sgt: + teardown_sgtable(&sgt); + return ret; } -int ceph_encrypt2(struct ceph_crypto_key *secret, void *dst, size_t *dst_len, - const void *src1, size_t src1_len, - const void *src2, size_t src2_len) +int ceph_crypt(const struct ceph_crypto_key *key, bool encrypt, + void *buf, int buf_len, int in_len, int *pout_len) { - switch (secret->type) { + switch (key->type) { case CEPH_CRYPTO_NONE: - if (*dst_len < src1_len + src2_len) - return -ERANGE; - memcpy(dst, src1, src1_len); - memcpy(dst + src1_len, src2, src2_len); - *dst_len = src1_len + src2_len; + *pout_len = in_len; return 0; - case CEPH_CRYPTO_AES: - return ceph_aes_encrypt2(secret->key, secret->len, dst, dst_len, - src1, src1_len, src2, src2_len); - + return ceph_aes_crypt(key, encrypt, buf, buf_len, in_len, + pout_len); default: - return -EINVAL; + return -ENOTSUPP; } } diff --git a/net/ceph/crypto.h b/net/ceph/crypto.h index 2e9cab09f37ba6..58d83aa7740f64 100644 --- a/net/ceph/crypto.h +++ b/net/ceph/crypto.h @@ -12,37 +12,19 @@ struct ceph_crypto_key { struct ceph_timespec created; int len; void *key; + struct crypto_skcipher *tfm; }; -static inline void ceph_crypto_key_destroy(struct ceph_crypto_key *key) -{ - if (key) { - kfree(key->key); - key->key = NULL; - } -} - int ceph_crypto_key_clone(struct ceph_crypto_key *dst, const struct ceph_crypto_key *src); int ceph_crypto_key_encode(struct ceph_crypto_key *key, void **p, void *end); int ceph_crypto_key_decode(struct ceph_crypto_key *key, void **p, void *end); int ceph_crypto_key_unarmor(struct ceph_crypto_key *key, const char *in); +void ceph_crypto_key_destroy(struct ceph_crypto_key *key); /* crypto.c */ -int ceph_decrypt(struct ceph_crypto_key *secret, - void *dst, size_t *dst_len, - const void *src, size_t src_len); -int ceph_encrypt(struct ceph_crypto_key *secret, - void *dst, size_t *dst_len, - const void *src, size_t src_len); -int ceph_decrypt2(struct ceph_crypto_key *secret, - void *dst1, size_t *dst1_len, - void *dst2, size_t *dst2_len, - const void *src, size_t src_len); -int ceph_encrypt2(struct ceph_crypto_key *secret, - void *dst, size_t *dst_len, - const void *src1, size_t src1_len, - const void *src2, size_t src2_len); +int ceph_crypt(const struct ceph_crypto_key *key, bool encrypt, + void *buf, int buf_len, int in_len, int *pout_len); int ceph_crypto_init(void); void ceph_crypto_shutdown(void); diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c index a5502898ea33b0..2efb335deada60 100644 --- a/net/ceph/messenger.c +++ b/net/ceph/messenger.c @@ -2027,6 +2027,19 @@ static int process_connect(struct ceph_connection *con) dout("process_connect on %p tag %d\n", con, (int)con->in_tag); + if (con->auth_reply_buf) { + /* + * Any connection that defines ->get_authorizer() + * should also define ->verify_authorizer_reply(). + * See get_connect_authorizer(). + */ + ret = con->ops->verify_authorizer_reply(con, 0); + if (ret < 0) { + con->error_msg = "bad authorize reply"; + return ret; + } + } + switch (con->in_reply.tag) { case CEPH_MSGR_TAG_FEATURES: pr_err("%s%lld %s feature set mismatch," diff --git a/net/core/dev.c b/net/core/dev.c index 6666b28b6815e1..df51c50927ab6d 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -2815,9 +2815,9 @@ static netdev_features_t harmonize_features(struct sk_buff *skb, if (skb->ip_summed != CHECKSUM_NONE && !can_checksum_protocol(features, type)) { features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); - } else if (illegal_highdma(skb->dev, skb)) { - features &= ~NETIF_F_SG; } + if (illegal_highdma(skb->dev, skb)) + features &= ~NETIF_F_SG; return features; } @@ -4453,7 +4453,9 @@ static void skb_gro_reset_offset(struct sk_buff *skb) pinfo->nr_frags && !PageHighMem(skb_frag_page(frag0))) { NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0); - NAPI_GRO_CB(skb)->frag0_len = skb_frag_size(frag0); + NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int, + skb_frag_size(frag0), + skb->end - skb->tail); } } diff --git a/net/core/drop_monitor.c b/net/core/drop_monitor.c index 72cfb0c611254c..ca2c9c8b9a3e98 100644 --- a/net/core/drop_monitor.c +++ b/net/core/drop_monitor.c @@ -80,6 +80,7 @@ static struct sk_buff *reset_per_cpu_data(struct per_cpu_dm_data *data) struct nlattr *nla; struct sk_buff *skb; unsigned long flags; + void *msg_header; al = sizeof(struct net_dm_alert_msg); al += dm_hit_limit * sizeof(struct net_dm_drop_point); @@ -87,21 +88,41 @@ static struct sk_buff *reset_per_cpu_data(struct per_cpu_dm_data *data) skb = genlmsg_new(al, GFP_KERNEL); - if (skb) { - genlmsg_put(skb, 0, 0, &net_drop_monitor_family, - 0, NET_DM_CMD_ALERT); - nla = nla_reserve(skb, NLA_UNSPEC, - sizeof(struct net_dm_alert_msg)); - msg = nla_data(nla); - memset(msg, 0, al); - } else { - mod_timer(&data->send_timer, jiffies + HZ / 10); + if (!skb) + goto err; + + msg_header = genlmsg_put(skb, 0, 0, &net_drop_monitor_family, + 0, NET_DM_CMD_ALERT); + if (!msg_header) { + nlmsg_free(skb); + skb = NULL; + goto err; + } + nla = nla_reserve(skb, NLA_UNSPEC, + sizeof(struct net_dm_alert_msg)); + if (!nla) { + nlmsg_free(skb); + skb = NULL; + goto err; } + msg = nla_data(nla); + memset(msg, 0, al); + goto out; +err: + mod_timer(&data->send_timer, jiffies + HZ / 10); +out: spin_lock_irqsave(&data->lock, flags); swap(data->skb, skb); spin_unlock_irqrestore(&data->lock, flags); + if (skb) { + struct nlmsghdr *nlh = (struct nlmsghdr *)skb->data; + struct genlmsghdr *gnlh = (struct genlmsghdr *)nlmsg_data(nlh); + + genlmsg_end(skb, genlmsg_data(gnlh)); + } + return skb; } diff --git a/net/core/fib_rules.c b/net/core/fib_rules.c index be4629c344a6de..5de436a73be254 100644 --- a/net/core/fib_rules.c +++ b/net/core/fib_rules.c @@ -18,6 +18,11 @@ #include #include +static const struct fib_kuid_range fib_kuid_range_unset = { + KUIDT_INIT(0), + KUIDT_INIT(~0), +}; + int fib_default_rule_add(struct fib_rules_ops *ops, u32 pref, u32 table, u32 flags) { @@ -33,6 +38,7 @@ int fib_default_rule_add(struct fib_rules_ops *ops, r->table = table; r->flags = flags; r->fr_net = ops->fro_net; + r->uid_range = fib_kuid_range_unset; r->suppress_prefixlen = -1; r->suppress_ifgroup = -1; @@ -172,6 +178,34 @@ void fib_rules_unregister(struct fib_rules_ops *ops) } EXPORT_SYMBOL_GPL(fib_rules_unregister); +static int uid_range_set(struct fib_kuid_range *range) +{ + return uid_valid(range->start) && uid_valid(range->end); +} + +static struct fib_kuid_range nla_get_kuid_range(struct nlattr **tb) +{ + struct fib_rule_uid_range *in; + struct fib_kuid_range out; + + in = (struct fib_rule_uid_range *)nla_data(tb[FRA_UID_RANGE]); + + out.start = make_kuid(current_user_ns(), in->start); + out.end = make_kuid(current_user_ns(), in->end); + + return out; +} + +static int nla_put_uid_range(struct sk_buff *skb, struct fib_kuid_range *range) +{ + struct fib_rule_uid_range out = { + from_kuid_munged(current_user_ns(), range->start), + from_kuid_munged(current_user_ns(), range->end) + }; + + return nla_put(skb, FRA_UID_RANGE, sizeof(out), &out); +} + static int fib_rule_match(struct fib_rule *rule, struct fib_rules_ops *ops, struct flowi *fl, int flags, struct fib_lookup_arg *arg) @@ -193,6 +227,10 @@ static int fib_rule_match(struct fib_rule *rule, struct fib_rules_ops *ops, if (rule->l3mdev && !l3mdev_fib_rule_match(rule->fr_net, fl, arg)) goto out; + if (uid_lt(fl->flowi_uid, rule->uid_range.start) || + uid_gt(fl->flowi_uid, rule->uid_range.end)) + goto out; + ret = ops->match(rule, fl, flags); out: return (rule->flags & FIB_RULE_INVERT) ? !ret : ret; @@ -429,6 +467,21 @@ int fib_nl_newrule(struct sk_buff *skb, struct nlmsghdr *nlh) if (rule->l3mdev && rule->table) goto errout_free; + if (tb[FRA_UID_RANGE]) { + if (current_user_ns() != net->user_ns) { + err = -EPERM; + goto errout_free; + } + + rule->uid_range = nla_get_kuid_range(tb); + + if (!uid_range_set(&rule->uid_range) || + !uid_lte(rule->uid_range.start, rule->uid_range.end)) + goto errout_free; + } else { + rule->uid_range = fib_kuid_range_unset; + } + if ((nlh->nlmsg_flags & NLM_F_EXCL) && rule_exists(ops, frh, tb, rule)) { err = -EEXIST; @@ -497,6 +550,7 @@ int fib_nl_delrule(struct sk_buff *skb, struct nlmsghdr *nlh) struct fib_rules_ops *ops = NULL; struct fib_rule *rule, *tmp; struct nlattr *tb[FRA_MAX+1]; + struct fib_kuid_range range; int err = -EINVAL; if (nlh->nlmsg_len < nlmsg_msg_size(sizeof(*frh))) @@ -516,6 +570,14 @@ int fib_nl_delrule(struct sk_buff *skb, struct nlmsghdr *nlh) if (err < 0) goto errout; + if (tb[FRA_UID_RANGE]) { + range = nla_get_kuid_range(tb); + if (!uid_range_set(&range)) + goto errout; + } else { + range = fib_kuid_range_unset; + } + list_for_each_entry(rule, &ops->rules_list, list) { if (frh->action && (frh->action != rule->action)) continue; @@ -552,6 +614,11 @@ int fib_nl_delrule(struct sk_buff *skb, struct nlmsghdr *nlh) (rule->l3mdev != nla_get_u8(tb[FRA_L3MDEV]))) continue; + if (uid_range_set(&range) && + (!uid_eq(rule->uid_range.start, range.start) || + !uid_eq(rule->uid_range.end, range.end))) + continue; + if (!ops->compare(rule, frh, tb)) continue; @@ -619,7 +686,8 @@ static inline size_t fib_rule_nlmsg_size(struct fib_rules_ops *ops, + nla_total_size(4) /* FRA_SUPPRESS_IFGROUP */ + nla_total_size(4) /* FRA_FWMARK */ + nla_total_size(4) /* FRA_FWMASK */ - + nla_total_size_64bit(8); /* FRA_TUN_ID */ + + nla_total_size_64bit(8) /* FRA_TUN_ID */ + + nla_total_size(sizeof(struct fib_kuid_range)); if (ops->nlmsg_payload) payload += ops->nlmsg_payload(rule); @@ -679,7 +747,9 @@ static int fib_nl_fill_rule(struct sk_buff *skb, struct fib_rule *rule, (rule->tun_id && nla_put_be64(skb, FRA_TUN_ID, rule->tun_id, FRA_PAD)) || (rule->l3mdev && - nla_put_u8(skb, FRA_L3MDEV, rule->l3mdev))) + nla_put_u8(skb, FRA_L3MDEV, rule->l3mdev)) || + (uid_range_set(&rule->uid_range) && + nla_put_uid_range(skb, &rule->uid_range))) goto nla_put_failure; if (rule->suppress_ifgroup != -1) { diff --git a/net/core/flow_dissector.c b/net/core/flow_dissector.c index c6d8207ffa7e9e..32e4e0158846b1 100644 --- a/net/core/flow_dissector.c +++ b/net/core/flow_dissector.c @@ -445,8 +445,9 @@ bool __skb_flow_dissect(const struct sk_buff *skb, if (hdr->flags & GRE_ACK) offset += sizeof(((struct pptp_gre_header *)0)->ack); - ppp_hdr = skb_header_pointer(skb, nhoff + offset, - sizeof(_ppp_hdr), _ppp_hdr); + ppp_hdr = __skb_header_pointer(skb, nhoff + offset, + sizeof(_ppp_hdr), + data, hlen, _ppp_hdr); if (!ppp_hdr) goto out_bad; diff --git a/net/core/lwtunnel.c b/net/core/lwtunnel.c index e5f84c26ba1a6f..afa64f086d8720 100644 --- a/net/core/lwtunnel.c +++ b/net/core/lwtunnel.c @@ -26,6 +26,7 @@ #include #include #include +#include #ifdef CONFIG_MODULES @@ -65,6 +66,15 @@ EXPORT_SYMBOL(lwtunnel_state_alloc); static const struct lwtunnel_encap_ops __rcu * lwtun_encaps[LWTUNNEL_ENCAP_MAX + 1] __read_mostly; +void lwtstate_free(struct lwtunnel_state *lws) +{ + const struct lwtunnel_encap_ops *ops = lwtun_encaps[lws->type]; + + kfree(lws); + module_put(ops->owner); +} +EXPORT_SYMBOL(lwtstate_free); + int lwtunnel_encap_add_ops(const struct lwtunnel_encap_ops *ops, unsigned int num) { @@ -110,25 +120,77 @@ int lwtunnel_build_state(struct net_device *dev, u16 encap_type, ret = -EOPNOTSUPP; rcu_read_lock(); ops = rcu_dereference(lwtun_encaps[encap_type]); + if (likely(ops && ops->build_state && try_module_get(ops->owner))) { + ret = ops->build_state(dev, encap, family, cfg, lws); + if (ret) + module_put(ops->owner); + } + rcu_read_unlock(); + + return ret; +} +EXPORT_SYMBOL(lwtunnel_build_state); + +int lwtunnel_valid_encap_type(u16 encap_type) +{ + const struct lwtunnel_encap_ops *ops; + int ret = -EINVAL; + + if (encap_type == LWTUNNEL_ENCAP_NONE || + encap_type > LWTUNNEL_ENCAP_MAX) + return ret; + + rcu_read_lock(); + ops = rcu_dereference(lwtun_encaps[encap_type]); + rcu_read_unlock(); #ifdef CONFIG_MODULES if (!ops) { const char *encap_type_str = lwtunnel_encap_str(encap_type); if (encap_type_str) { - rcu_read_unlock(); + __rtnl_unlock(); request_module("rtnl-lwt-%s", encap_type_str); + rtnl_lock(); + rcu_read_lock(); ops = rcu_dereference(lwtun_encaps[encap_type]); + rcu_read_unlock(); } } #endif - if (likely(ops && ops->build_state)) - ret = ops->build_state(dev, encap, family, cfg, lws); - rcu_read_unlock(); + return ops ? 0 : -EOPNOTSUPP; +} +EXPORT_SYMBOL(lwtunnel_valid_encap_type); - return ret; +int lwtunnel_valid_encap_type_attr(struct nlattr *attr, int remaining) +{ + struct rtnexthop *rtnh = (struct rtnexthop *)attr; + struct nlattr *nla_entype; + struct nlattr *attrs; + struct nlattr *nla; + u16 encap_type; + int attrlen; + + while (rtnh_ok(rtnh, remaining)) { + attrlen = rtnh_attrlen(rtnh); + if (attrlen > 0) { + attrs = rtnh_attrs(rtnh); + nla = nla_find(attrs, attrlen, RTA_ENCAP); + nla_entype = nla_find(attrs, attrlen, RTA_ENCAP_TYPE); + + if (nla_entype) { + encap_type = nla_get_u16(nla_entype); + + if (lwtunnel_valid_encap_type(encap_type) != 0) + return -EOPNOTSUPP; + } + } + rtnh = rtnh_next(rtnh, &remaining); + } + + return 0; } -EXPORT_SYMBOL(lwtunnel_build_state); +EXPORT_SYMBOL(lwtunnel_valid_encap_type_attr); int lwtunnel_fill_encap(struct sk_buff *skb, struct lwtunnel_state *lwtstate) { diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c index a6196cf844f664..b7f9ae7b1c5f97 100644 --- a/net/core/rtnetlink.c +++ b/net/core/rtnetlink.c @@ -3886,6 +3886,9 @@ static int rtnl_stats_get(struct sk_buff *skb, struct nlmsghdr *nlh) u32 filter_mask; int err; + if (nlmsg_len(nlh) < sizeof(*ifsm)) + return -EINVAL; + ifsm = nlmsg_data(nlh); if (ifsm->ifindex > 0) dev = __dev_get_by_index(net, ifsm->ifindex); @@ -3935,6 +3938,9 @@ static int rtnl_stats_dump(struct sk_buff *skb, struct netlink_callback *cb) cb->seq = net->dev_base_seq; + if (nlmsg_len(cb->nlh) < sizeof(*ifsm)) + return -EINVAL; + ifsm = nlmsg_data(cb->nlh); filter_mask = ifsm->filter_mask; if (!filter_mask) diff --git a/net/core/sock.c b/net/core/sock.c index 00a074dbfe9bf1..87a740b48f49df 100644 --- a/net/core/sock.c +++ b/net/core/sock.c @@ -222,7 +222,7 @@ static const char *const af_family_key_strings[AF_MAX+1] = { "sk_lock-AF_RXRPC" , "sk_lock-AF_ISDN" , "sk_lock-AF_PHONET" , "sk_lock-AF_IEEE802154", "sk_lock-AF_CAIF" , "sk_lock-AF_ALG" , "sk_lock-AF_NFC" , "sk_lock-AF_VSOCK" , "sk_lock-AF_KCM" , - "sk_lock-AF_MAX" + "sk_lock-AF_QIPCRTR", "sk_lock-AF_MAX" }; static const char *const af_family_slock_key_strings[AF_MAX+1] = { "slock-AF_UNSPEC", "slock-AF_UNIX" , "slock-AF_INET" , @@ -239,7 +239,7 @@ static const char *const af_family_slock_key_strings[AF_MAX+1] = { "slock-AF_RXRPC" , "slock-AF_ISDN" , "slock-AF_PHONET" , "slock-AF_IEEE802154", "slock-AF_CAIF" , "slock-AF_ALG" , "slock-AF_NFC" , "slock-AF_VSOCK" ,"slock-AF_KCM" , - "slock-AF_MAX" + "slock-AF_QIPCRTR", "slock-AF_MAX" }; static const char *const af_family_clock_key_strings[AF_MAX+1] = { "clock-AF_UNSPEC", "clock-AF_UNIX" , "clock-AF_INET" , @@ -256,7 +256,7 @@ static const char *const af_family_clock_key_strings[AF_MAX+1] = { "clock-AF_RXRPC" , "clock-AF_ISDN" , "clock-AF_PHONET" , "clock-AF_IEEE802154", "clock-AF_CAIF" , "clock-AF_ALG" , "clock-AF_NFC" , "clock-AF_VSOCK" , "clock-AF_KCM" , - "clock-AF_MAX" + "clock-AF_QIPCRTR", "clock-AF_MAX" }; /* @@ -2436,8 +2436,11 @@ void sock_init_data(struct socket *sock, struct sock *sk) sk->sk_type = sock->type; sk->sk_wq = sock->wq; sock->sk = sk; - } else + sk->sk_uid = SOCK_INODE(sock)->i_uid; + } else { sk->sk_wq = NULL; + sk->sk_uid = make_kuid(sock_net(sk)->user_ns, 0); + } rwlock_init(&sk->sk_callback_lock); lockdep_set_class_and_name(&sk->sk_callback_lock, diff --git a/net/dsa/dsa2.c b/net/dsa/dsa2.c index 5fff951a0a4928..da38621245458b 100644 --- a/net/dsa/dsa2.c +++ b/net/dsa/dsa2.c @@ -394,9 +394,11 @@ static int dsa_dst_apply(struct dsa_switch_tree *dst) return err; } - err = dsa_cpu_port_ethtool_setup(dst->ds[0]); - if (err) - return err; + if (dst->ds[0]) { + err = dsa_cpu_port_ethtool_setup(dst->ds[0]); + if (err) + return err; + } /* If we use a tagging format that doesn't have an ethertype * field, make sure that all packets from this point on get @@ -433,7 +435,8 @@ static void dsa_dst_unapply(struct dsa_switch_tree *dst) dsa_ds_unapply(dst, ds); } - dsa_cpu_port_ethtool_restore(dst->ds[0]); + if (dst->ds[0]) + dsa_cpu_port_ethtool_restore(dst->ds[0]); pr_info("DSA: tree %d unapplied\n", dst->tree); dst->applied = false; diff --git a/net/dsa/slave.c b/net/dsa/slave.c index 30e2e21d76196f..3ff9d97cf56b38 100644 --- a/net/dsa/slave.c +++ b/net/dsa/slave.c @@ -1201,6 +1201,8 @@ int dsa_slave_suspend(struct net_device *slave_dev) { struct dsa_slave_priv *p = netdev_priv(slave_dev); + netif_device_detach(slave_dev); + if (p->phy) { phy_stop(p->phy); p->old_pause = -1; diff --git a/net/ipv4/Makefile b/net/ipv4/Makefile index bc6a6c8b9bcd79..a8b934aa9d848d 100644 --- a/net/ipv4/Makefile +++ b/net/ipv4/Makefile @@ -16,6 +16,7 @@ obj-y := route.o inetpeer.o protocol.o \ obj-$(CONFIG_NET_IP_TUNNEL) += ip_tunnel.o obj-$(CONFIG_SYSCTL) += sysctl_net_ipv4.o +obj-$(CONFIG_SYSFS) += sysfs_net_ipv4.o obj-$(CONFIG_PROC_FS) += proc.o obj-$(CONFIG_IP_MULTIPLE_TABLES) += fib_rules.o obj-$(CONFIG_IP_MROUTE) += ipmr.o diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c index 215143246e4b3e..c836bfeb4f43ed 100644 --- a/net/ipv4/af_inet.c +++ b/net/ipv4/af_inet.c @@ -121,6 +121,19 @@ #endif #include +#ifdef CONFIG_ANDROID_PARANOID_NETWORK +#include + +static inline int current_has_network(void) +{ + return in_egroup_p(AID_INET) || capable(CAP_NET_RAW); +} +#else +static inline int current_has_network(void) +{ + return 1; +} +#endif /* The inetsw table contains everything that inet_create needs to * build a new socket. @@ -255,6 +268,9 @@ static int inet_create(struct net *net, struct socket *sock, int protocol, if (protocol < 0 || protocol >= IPPROTO_MAX) return -EINVAL; + if (!current_has_network()) + return -EACCES; + sock->state = SS_UNCONNECTED; /* Look for the requested type/protocol pair. */ @@ -303,8 +319,7 @@ static int inet_create(struct net *net, struct socket *sock, int protocol, } err = -EPERM; - if (sock->type == SOCK_RAW && !kern && - !ns_capable(net->user_ns, CAP_NET_RAW)) + if (sock->type == SOCK_RAW && !kern && !capable(CAP_NET_RAW)) goto out_rcu_unlock; sock->ops = answer->ops; diff --git a/net/ipv4/fib_frontend.c b/net/ipv4/fib_frontend.c index 161fc0f0d752fb..18412f989b32a6 100644 --- a/net/ipv4/fib_frontend.c +++ b/net/ipv4/fib_frontend.c @@ -46,6 +46,7 @@ #include #include #include +#include #include #ifndef CONFIG_IP_MULTIPLE_TABLES @@ -85,7 +86,7 @@ struct fib_table *fib_new_table(struct net *net, u32 id) if (tb) return tb; - if (id == RT_TABLE_LOCAL) + if (id == RT_TABLE_LOCAL && !net->ipv4.fib_has_custom_rules) alias = fib_new_table(net, RT_TABLE_MAIN); tb = fib_trie_table(id, alias); @@ -620,6 +621,7 @@ const struct nla_policy rtm_ipv4_policy[RTA_MAX + 1] = { [RTA_FLOW] = { .type = NLA_U32 }, [RTA_ENCAP_TYPE] = { .type = NLA_U16 }, [RTA_ENCAP] = { .type = NLA_NESTED }, + [RTA_UID] = { .type = NLA_U32 }, }; static int rtm_to_fib_config(struct net *net, struct sk_buff *skb, @@ -676,6 +678,10 @@ static int rtm_to_fib_config(struct net *net, struct sk_buff *skb, cfg->fc_mx_len = nla_len(attr); break; case RTA_MULTIPATH: + err = lwtunnel_valid_encap_type_attr(nla_data(attr), + nla_len(attr)); + if (err < 0) + goto errout; cfg->fc_mp = nla_data(attr); cfg->fc_mp_len = nla_len(attr); break; @@ -690,6 +696,9 @@ static int rtm_to_fib_config(struct net *net, struct sk_buff *skb, break; case RTA_ENCAP_TYPE: cfg->fc_encap_type = nla_get_u16(attr); + err = lwtunnel_valid_encap_type(cfg->fc_encap_type); + if (err < 0) + goto errout; break; } } diff --git a/net/ipv4/fib_semantics.c b/net/ipv4/fib_semantics.c index 388d3e21629b17..6a4068031aaa41 100644 --- a/net/ipv4/fib_semantics.c +++ b/net/ipv4/fib_semantics.c @@ -1278,8 +1278,9 @@ int fib_dump_info(struct sk_buff *skb, u32 portid, u32 seq, int event, nla_put_u32(skb, RTA_FLOW, fi->fib_nh[0].nh_tclassid)) goto nla_put_failure; #endif - if (fi->fib_nh->nh_lwtstate) - lwtunnel_fill_encap(skb, fi->fib_nh->nh_lwtstate); + if (fi->fib_nh->nh_lwtstate && + lwtunnel_fill_encap(skb, fi->fib_nh->nh_lwtstate) < 0) + goto nla_put_failure; } #ifdef CONFIG_IP_ROUTE_MULTIPATH if (fi->fib_nhs > 1) { @@ -1315,8 +1316,10 @@ int fib_dump_info(struct sk_buff *skb, u32 portid, u32 seq, int event, nla_put_u32(skb, RTA_FLOW, nh->nh_tclassid)) goto nla_put_failure; #endif - if (nh->nh_lwtstate) - lwtunnel_fill_encap(skb, nh->nh_lwtstate); + if (nh->nh_lwtstate && + lwtunnel_fill_encap(skb, nh->nh_lwtstate) < 0) + goto nla_put_failure; + /* length of rtnetlink header + attributes */ rtnh->rtnh_len = nlmsg_get_pos(skb) - (void *) rtnh; } endfor_nexthops(fi); @@ -1617,8 +1620,13 @@ void fib_select_multipath(struct fib_result *res, int hash) void fib_select_path(struct net *net, struct fib_result *res, struct flowi4 *fl4, int mp_hash) { + bool oif_check; + + oif_check = (fl4->flowi4_oif == 0 || + fl4->flowi4_flags & FLOWI_FLAG_SKIP_NH_OIF); + #ifdef CONFIG_IP_ROUTE_MULTIPATH - if (res->fi->fib_nhs > 1 && fl4->flowi4_oif == 0) { + if (res->fi->fib_nhs > 1 && oif_check) { if (mp_hash < 0) mp_hash = get_hash_from_flowi4(fl4) >> 1; @@ -1628,7 +1636,7 @@ void fib_select_path(struct net *net, struct fib_result *res, #endif if (!res->prefixlen && res->table->tb_num_default > 1 && - res->type == RTN_UNICAST && !fl4->flowi4_oif) + res->type == RTN_UNICAST && oif_check) fib_select_default(fl4, res); if (!fl4->saddr) diff --git a/net/ipv4/icmp.c b/net/ipv4/icmp.c index 48734ee6293f33..691146abde2df6 100644 --- a/net/ipv4/icmp.c +++ b/net/ipv4/icmp.c @@ -425,6 +425,7 @@ static void icmp_reply(struct icmp_bxm *icmp_param, struct sk_buff *skb) fl4.daddr = daddr; fl4.saddr = saddr; fl4.flowi4_mark = mark; + fl4.flowi4_uid = sock_net_uid(net, NULL); fl4.flowi4_tos = RT_TOS(ip_hdr(skb)->tos); fl4.flowi4_proto = IPPROTO_ICMP; fl4.flowi4_oif = l3mdev_master_ifindex(skb->dev); @@ -473,6 +474,7 @@ static struct rtable *icmp_route_lookup(struct net *net, param->replyopts.opt.opt.faddr : iph->saddr); fl4->saddr = saddr; fl4->flowi4_mark = mark; + fl4->flowi4_uid = sock_net_uid(net, NULL); fl4->flowi4_tos = RT_TOS(tos); fl4->flowi4_proto = IPPROTO_ICMP; fl4->fl4_icmp_type = type; diff --git a/net/ipv4/igmp.c b/net/ipv4/igmp.c index 15db786d50ed28..32a08bc010bfc0 100644 --- a/net/ipv4/igmp.c +++ b/net/ipv4/igmp.c @@ -219,9 +219,14 @@ static void igmp_start_timer(struct ip_mc_list *im, int max_delay) static void igmp_gq_start_timer(struct in_device *in_dev) { int tv = prandom_u32() % in_dev->mr_maxdelay; + unsigned long exp = jiffies + tv + 2; + + if (in_dev->mr_gq_running && + time_after_eq(exp, (in_dev->mr_gq_timer).expires)) + return; in_dev->mr_gq_running = 1; - if (!mod_timer(&in_dev->mr_gq_timer, jiffies+tv+2)) + if (!mod_timer(&in_dev->mr_gq_timer, exp)) in_dev_hold(in_dev); } diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c index 61a9deec299332..d5d3ead0a6c31e 100644 --- a/net/ipv4/inet_connection_sock.c +++ b/net/ipv4/inet_connection_sock.c @@ -415,7 +415,7 @@ struct dst_entry *inet_csk_route_req(const struct sock *sk, sk->sk_protocol, inet_sk_flowi_flags(sk), (opt && opt->opt.srr) ? opt->opt.faddr : ireq->ir_rmt_addr, ireq->ir_loc_addr, ireq->ir_rmt_port, - htons(ireq->ir_num)); + htons(ireq->ir_num), sk->sk_uid); security_req_classify_flow(req, flowi4_to_flowi(fl4)); rt = ip_route_output_flow(net, fl4, sk); if (IS_ERR(rt)) @@ -452,7 +452,7 @@ struct dst_entry *inet_csk_route_child_sock(const struct sock *sk, sk->sk_protocol, inet_sk_flowi_flags(sk), (opt && opt->opt.srr) ? opt->opt.faddr : ireq->ir_rmt_addr, ireq->ir_loc_addr, ireq->ir_rmt_port, - htons(ireq->ir_num)); + htons(ireq->ir_num), sk->sk_uid); security_req_classify_flow(req, flowi4_to_flowi(fl4)); rt = ip_route_output_flow(net, fl4, sk); if (IS_ERR(rt)) diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c index 877bdb02e887f5..d24fa2075c635e 100644 --- a/net/ipv4/ip_output.c +++ b/net/ipv4/ip_output.c @@ -1594,7 +1594,8 @@ void ip_send_unicast_reply(struct sock *sk, struct sk_buff *skb, RT_SCOPE_UNIVERSE, ip_hdr(skb)->protocol, ip_reply_arg_flowi_flags(arg), daddr, saddr, - tcp_hdr(skb)->source, tcp_hdr(skb)->dest); + tcp_hdr(skb)->source, tcp_hdr(skb)->dest, + arg->uid); security_skb_classify_flow(skb, flowi4_to_flowi(&fl4)); rt = ip_route_output_key(net, &fl4); if (IS_ERR(rt)) diff --git a/net/ipv4/ip_sockglue.c b/net/ipv4/ip_sockglue.c index b8a2d63d1fb82f..f226f4086e05f0 100644 --- a/net/ipv4/ip_sockglue.c +++ b/net/ipv4/ip_sockglue.c @@ -137,7 +137,7 @@ static void ip_cmsg_recv_dstaddr(struct msghdr *msg, struct sk_buff *skb) const struct iphdr *iph = ip_hdr(skb); __be16 *ports = (__be16 *)skb_transport_header(skb); - if (skb_transport_offset(skb) + 4 > skb->len) + if (skb_transport_offset(skb) + 4 > (int)skb->len) return; /* All current transport protocols have the port numbers in the @@ -1202,8 +1202,14 @@ void ipv4_pktinfo_prepare(const struct sock *sk, struct sk_buff *skb) * which has interface index (iif) as the first member of the * underlying inet{6}_skb_parm struct. This code then overlays * PKTINFO_SKB_CB and in_pktinfo also has iif as the first - * element so the iif is picked up from the prior IPCB + * element so the iif is picked up from the prior IPCB. If iif + * is the loopback interface, then return the sending interface + * (e.g., process binds socket to eth0 for Tx which is + * redirected to loopback in the rtable/dst). */ + if (pktinfo->ipi_ifindex == LOOPBACK_IFINDEX) + pktinfo->ipi_ifindex = inet_iif(skb); + pktinfo->ipi_spec_dst.s_addr = fib_compute_spec_dst(skb); } else { pktinfo->ipi_ifindex = 0; diff --git a/net/ipv4/ip_tunnel_core.c b/net/ipv4/ip_tunnel_core.c index fed3d29f9eb3b7..0fd1976ab63bbd 100644 --- a/net/ipv4/ip_tunnel_core.c +++ b/net/ipv4/ip_tunnel_core.c @@ -313,6 +313,7 @@ static const struct lwtunnel_encap_ops ip_tun_lwt_ops = { .fill_encap = ip_tun_fill_encap_info, .get_encap_size = ip_tun_encap_nlsize, .cmp_encap = ip_tun_cmp_encap, + .owner = THIS_MODULE, }; static const struct nla_policy ip6_tun_policy[LWTUNNEL_IP6_MAX + 1] = { @@ -403,6 +404,7 @@ static const struct lwtunnel_encap_ops ip6_tun_lwt_ops = { .fill_encap = ip6_tun_fill_encap_info, .get_encap_size = ip6_tun_encap_nlsize, .cmp_encap = ip_tun_cmp_encap, + .owner = THIS_MODULE, }; void __init ip_tunnel_core_init(void) diff --git a/net/ipv4/ping.c b/net/ipv4/ping.c index 96b8e2b957313d..5b2635e69a9268 100644 --- a/net/ipv4/ping.c +++ b/net/ipv4/ping.c @@ -793,7 +793,8 @@ static int ping_v4_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) flowi4_init_output(&fl4, ipc.oif, sk->sk_mark, tos, RT_SCOPE_UNIVERSE, sk->sk_protocol, - inet_sk_flowi_flags(sk), faddr, saddr, 0, 0); + inet_sk_flowi_flags(sk), faddr, saddr, 0, 0, + sk->sk_uid); security_sk_classify_flow(sk, flowi4_to_flowi(&fl4)); rt = ip_route_output_flow(net, &fl4, sk); diff --git a/net/ipv4/raw.c b/net/ipv4/raw.c index ecbe5a7c2d6d3f..7525f5eab622b9 100644 --- a/net/ipv4/raw.c +++ b/net/ipv4/raw.c @@ -604,7 +604,7 @@ static int raw_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol, inet_sk_flowi_flags(sk) | (inet->hdrincl ? FLOWI_FLAG_KNOWN_NH : 0), - daddr, saddr, 0, 0); + daddr, saddr, 0, 0, sk->sk_uid); if (!inet->hdrincl) { rfv.msg = msg; diff --git a/net/ipv4/route.c b/net/ipv4/route.c index 2a57566e6e9194..10b686b215f15b 100644 --- a/net/ipv4/route.c +++ b/net/ipv4/route.c @@ -507,7 +507,8 @@ void __ip_select_ident(struct net *net, struct iphdr *iph, int segs) } EXPORT_SYMBOL(__ip_select_ident); -static void __build_flow_key(struct flowi4 *fl4, const struct sock *sk, +static void __build_flow_key(const struct net *net, struct flowi4 *fl4, + const struct sock *sk, const struct iphdr *iph, int oif, u8 tos, u8 prot, u32 mark, int flow_flags) @@ -523,19 +524,21 @@ static void __build_flow_key(struct flowi4 *fl4, const struct sock *sk, flowi4_init_output(fl4, oif, mark, tos, RT_SCOPE_UNIVERSE, prot, flow_flags, - iph->daddr, iph->saddr, 0, 0); + iph->daddr, iph->saddr, 0, 0, + sock_net_uid(net, sk)); } static void build_skb_flow_key(struct flowi4 *fl4, const struct sk_buff *skb, const struct sock *sk) { + const struct net *net = dev_net(skb->dev); const struct iphdr *iph = ip_hdr(skb); int oif = skb->dev->ifindex; u8 tos = RT_TOS(iph->tos); u8 prot = iph->protocol; u32 mark = skb->mark; - __build_flow_key(fl4, sk, iph, oif, tos, prot, mark, 0); + __build_flow_key(net, fl4, sk, iph, oif, tos, prot, mark, 0); } static void build_sk_flow_key(struct flowi4 *fl4, const struct sock *sk) @@ -552,7 +555,7 @@ static void build_sk_flow_key(struct flowi4 *fl4, const struct sock *sk) RT_CONN_FLAGS(sk), RT_SCOPE_UNIVERSE, inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol, inet_sk_flowi_flags(sk), - daddr, inet->inet_saddr, 0, 0); + daddr, inet->inet_saddr, 0, 0, sk->sk_uid); rcu_read_unlock(); } @@ -802,7 +805,7 @@ static void ip_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buf rt = (struct rtable *) dst; - __build_flow_key(&fl4, sk, iph, oif, tos, prot, mark, 0); + __build_flow_key(sock_net(sk), &fl4, sk, iph, oif, tos, prot, mark, 0); __ip_do_redirect(rt, skb, &fl4, true); } @@ -1020,7 +1023,7 @@ void ipv4_update_pmtu(struct sk_buff *skb, struct net *net, u32 mtu, if (!mark) mark = IP4_REPLY_MARK(net, skb->mark); - __build_flow_key(&fl4, NULL, iph, oif, + __build_flow_key(net, &fl4, NULL, iph, oif, RT_TOS(iph->tos), protocol, mark, flow_flags); rt = __ip_route_output_key(net, &fl4); if (!IS_ERR(rt)) { @@ -1036,7 +1039,7 @@ static void __ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu) struct flowi4 fl4; struct rtable *rt; - __build_flow_key(&fl4, sk, iph, 0, 0, 0, 0, 0); + __build_flow_key(sock_net(sk), &fl4, sk, iph, 0, 0, 0, 0, 0); if (!fl4.flowi4_mark) fl4.flowi4_mark = IP4_REPLY_MARK(sock_net(sk), skb->mark); @@ -1055,6 +1058,7 @@ void ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu) struct rtable *rt; struct dst_entry *odst = NULL; bool new = false; + struct net *net = sock_net(sk); bh_lock_sock(sk); @@ -1068,7 +1072,7 @@ void ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu) goto out; } - __build_flow_key(&fl4, sk, iph, 0, 0, 0, 0, 0); + __build_flow_key(net, &fl4, sk, iph, 0, 0, 0, 0, 0); rt = (struct rtable *)odst; if (odst->obsolete && !odst->ops->check(odst, 0)) { @@ -1108,7 +1112,7 @@ void ipv4_redirect(struct sk_buff *skb, struct net *net, struct flowi4 fl4; struct rtable *rt; - __build_flow_key(&fl4, NULL, iph, oif, + __build_flow_key(net, &fl4, NULL, iph, oif, RT_TOS(iph->tos), protocol, mark, flow_flags); rt = __ip_route_output_key(net, &fl4); if (!IS_ERR(rt)) { @@ -1123,9 +1127,10 @@ void ipv4_sk_redirect(struct sk_buff *skb, struct sock *sk) const struct iphdr *iph = (const struct iphdr *) skb->data; struct flowi4 fl4; struct rtable *rt; + struct net *net = sock_net(sk); - __build_flow_key(&fl4, sk, iph, 0, 0, 0, 0, 0); - rt = __ip_route_output_key(sock_net(sk), &fl4); + __build_flow_key(net, &fl4, sk, iph, 0, 0, 0, 0, 0); + rt = __ip_route_output_key(net, &fl4); if (!IS_ERR(rt)) { __ip_do_redirect(rt, skb, &fl4, false); ip_rt_put(rt); @@ -1902,7 +1907,8 @@ out: return err; } } - rth = rt_dst_alloc(net->loopback_dev, flags | RTCF_LOCAL, res.type, + rth = rt_dst_alloc(l3mdev_master_dev_rcu(dev) ? : net->loopback_dev, + flags | RTCF_LOCAL, res.type, IN_DEV_CONF_GET(in_dev, NOPOLICY), false, do_cache); if (!rth) goto e_nobufs; @@ -2439,7 +2445,7 @@ static int rt_fill_info(struct net *net, __be32 dst, __be32 src, u32 table_id, r->rtm_dst_len = 32; r->rtm_src_len = 0; r->rtm_tos = fl4->flowi4_tos; - r->rtm_table = table_id; + r->rtm_table = table_id < 256 ? table_id : RT_TABLE_COMPAT; if (nla_put_u32(skb, RTA_TABLE, table_id)) goto nla_put_failure; r->rtm_type = rt->rt_type; @@ -2495,6 +2501,11 @@ static int rt_fill_info(struct net *net, __be32 dst, __be32 src, u32 table_id, nla_put_u32(skb, RTA_MARK, fl4->flowi4_mark)) goto nla_put_failure; + if (!uid_eq(fl4->flowi4_uid, INVALID_UID) && + nla_put_u32(skb, RTA_UID, + from_kuid_munged(current_user_ns(), fl4->flowi4_uid))) + goto nla_put_failure; + error = rt->dst.error; if (rt_is_input_route(rt)) { @@ -2547,6 +2558,7 @@ static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh) int mark; struct sk_buff *skb; u32 table_id = RT_TABLE_MAIN; + kuid_t uid; err = nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX, rtm_ipv4_policy); if (err < 0) @@ -2574,6 +2586,10 @@ static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh) dst = tb[RTA_DST] ? nla_get_in_addr(tb[RTA_DST]) : 0; iif = tb[RTA_IIF] ? nla_get_u32(tb[RTA_IIF]) : 0; mark = tb[RTA_MARK] ? nla_get_u32(tb[RTA_MARK]) : 0; + if (tb[RTA_UID]) + uid = make_kuid(current_user_ns(), nla_get_u32(tb[RTA_UID])); + else + uid = (iif ? INVALID_UID : current_uid()); memset(&fl4, 0, sizeof(fl4)); fl4.daddr = dst; @@ -2581,6 +2597,7 @@ static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh) fl4.flowi4_tos = rtm->rtm_tos; fl4.flowi4_oif = tb[RTA_OIF] ? nla_get_u32(tb[RTA_OIF]) : 0; fl4.flowi4_mark = mark; + fl4.flowi4_uid = uid; if (iif) { struct net_device *dev; diff --git a/net/ipv4/syncookies.c b/net/ipv4/syncookies.c index e3c4043c27de28..0dc6286272aae6 100644 --- a/net/ipv4/syncookies.c +++ b/net/ipv4/syncookies.c @@ -372,7 +372,7 @@ struct sock *cookie_v4_check(struct sock *sk, struct sk_buff *skb) RT_CONN_FLAGS(sk), RT_SCOPE_UNIVERSE, IPPROTO_TCP, inet_sk_flowi_flags(sk), opt->srr ? opt->faddr : ireq->ir_rmt_addr, - ireq->ir_loc_addr, th->source, th->dest); + ireq->ir_loc_addr, th->source, th->dest, sk->sk_uid); security_req_classify_flow(req, flowi4_to_flowi(&fl4)); rt = ip_route_output_key(sock_net(sk), &fl4); if (IS_ERR(rt)) { diff --git a/net/ipv4/sysctl_net_ipv4.c b/net/ipv4/sysctl_net_ipv4.c index 80bc36b25de21d..cf7cfa45c7d294 100644 --- a/net/ipv4/sysctl_net_ipv4.c +++ b/net/ipv4/sysctl_net_ipv4.c @@ -151,6 +151,21 @@ static int ipv4_ping_group_range(struct ctl_table *table, int write, return ret; } +/* Validate changes from /proc interface. */ +static int proc_tcp_default_init_rwnd(struct ctl_table *ctl, int write, + void __user *buffer, + size_t *lenp, loff_t *ppos) +{ + int old_value = *(int *)ctl->data; + int ret = proc_dointvec(ctl, write, buffer, lenp, ppos); + int new_value = *(int *)ctl->data; + + if (write && ret == 0 && (new_value < 3 || new_value > 100)) + *(int *)ctl->data = old_value; + + return ret; +} + static int proc_tcp_congestion_control(struct ctl_table *ctl, int write, void __user *buffer, size_t *lenp, loff_t *ppos) { @@ -623,6 +638,13 @@ static struct ctl_table ipv4_table[] = { .mode = 0644, .proc_handler = proc_dointvec_ms_jiffies, }, + { + .procname = "tcp_default_init_rwnd", + .data = &sysctl_tcp_default_init_rwnd, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_tcp_default_init_rwnd + }, { .procname = "icmp_msgs_per_sec", .data = &sysctl_icmp_msgs_per_sec, diff --git a/net/ipv4/sysfs_net_ipv4.c b/net/ipv4/sysfs_net_ipv4.c new file mode 100644 index 00000000000000..0cbbf10026a67c --- /dev/null +++ b/net/ipv4/sysfs_net_ipv4.c @@ -0,0 +1,88 @@ +/* + * net/ipv4/sysfs_net_ipv4.c + * + * sysfs-based networking knobs (so we can, unlike with sysctl, control perms) + * + * Copyright (C) 2008 Google, Inc. + * + * Robert Love + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include + +#define CREATE_IPV4_FILE(_name, _var) \ +static ssize_t _name##_show(struct kobject *kobj, \ + struct kobj_attribute *attr, char *buf) \ +{ \ + return sprintf(buf, "%d\n", _var); \ +} \ +static ssize_t _name##_store(struct kobject *kobj, \ + struct kobj_attribute *attr, \ + const char *buf, size_t count) \ +{ \ + int val, ret; \ + ret = sscanf(buf, "%d", &val); \ + if (ret != 1) \ + return -EINVAL; \ + if (val < 0) \ + return -EINVAL; \ + _var = val; \ + return count; \ +} \ +static struct kobj_attribute _name##_attr = \ + __ATTR(_name, 0644, _name##_show, _name##_store) + +CREATE_IPV4_FILE(tcp_wmem_min, sysctl_tcp_wmem[0]); +CREATE_IPV4_FILE(tcp_wmem_def, sysctl_tcp_wmem[1]); +CREATE_IPV4_FILE(tcp_wmem_max, sysctl_tcp_wmem[2]); + +CREATE_IPV4_FILE(tcp_rmem_min, sysctl_tcp_rmem[0]); +CREATE_IPV4_FILE(tcp_rmem_def, sysctl_tcp_rmem[1]); +CREATE_IPV4_FILE(tcp_rmem_max, sysctl_tcp_rmem[2]); + +static struct attribute *ipv4_attrs[] = { + &tcp_wmem_min_attr.attr, + &tcp_wmem_def_attr.attr, + &tcp_wmem_max_attr.attr, + &tcp_rmem_min_attr.attr, + &tcp_rmem_def_attr.attr, + &tcp_rmem_max_attr.attr, + NULL +}; + +static struct attribute_group ipv4_attr_group = { + .attrs = ipv4_attrs, +}; + +static __init int sysfs_ipv4_init(void) +{ + struct kobject *ipv4_kobject; + int ret; + + ipv4_kobject = kobject_create_and_add("ipv4", kernel_kobj); + if (!ipv4_kobject) + return -ENOMEM; + + ret = sysfs_create_group(ipv4_kobject, &ipv4_attr_group); + if (ret) { + kobject_put(ipv4_kobject); + return ret; + } + + return 0; +} + +subsys_initcall(sysfs_ipv4_init); diff --git a/net/ipv4/tcp_fastopen.c b/net/ipv4/tcp_fastopen.c index 4e777a3243f944..dd2560c83a8592 100644 --- a/net/ipv4/tcp_fastopen.c +++ b/net/ipv4/tcp_fastopen.c @@ -113,7 +113,7 @@ static bool tcp_fastopen_cookie_gen(struct request_sock *req, struct tcp_fastopen_cookie tmp; if (__tcp_fastopen_cookie_gen(&ip6h->saddr, &tmp)) { - struct in6_addr *buf = (struct in6_addr *) tmp.val; + struct in6_addr *buf = &tmp.addr; int i; for (i = 0; i < 4; i++) @@ -205,6 +205,7 @@ static struct sock *tcp_fastopen_create_child(struct sock *sk, * scaled. So correct it appropriately. */ tp->snd_wnd = ntohs(tcp_hdr(skb)->window); + tp->max_window = tp->snd_wnd; /* Activate the retrans timer so that SYNACK can be retransmitted. * The request socket is not added to the ehash diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index c71d49ce0c9379..7fb67045fcb92b 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -100,6 +100,7 @@ int sysctl_tcp_thin_dupack __read_mostly; int sysctl_tcp_moderate_rcvbuf __read_mostly = 1; int sysctl_tcp_early_retrans __read_mostly = 3; int sysctl_tcp_invalid_ratelimit __read_mostly = HZ/2; +int sysctl_tcp_default_init_rwnd __read_mostly = TCP_INIT_CWND * 2; #define FLAG_DATA 0x01 /* Incoming frame contained data. */ #define FLAG_WIN_UPDATE 0x02 /* Incoming ACK was a window update. */ diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index 2259114c7242c7..eb5a0e1fb18750 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -691,6 +691,7 @@ static void tcp_v4_send_reset(const struct sock *sk, struct sk_buff *skb) offsetof(struct inet_timewait_sock, tw_bound_dev_if)); arg.tos = ip_hdr(skb)->tos; + arg.uid = sock_net_uid(net, sk && sk_fullsock(sk) ? sk : NULL); local_bh_disable(); ip_send_unicast_reply(*this_cpu_ptr(net->ipv4.tcp_sk), skb, &TCP_SKB_CB(skb)->header.h4.opt, @@ -711,7 +712,7 @@ static void tcp_v4_send_reset(const struct sock *sk, struct sk_buff *skb) outside socket context is ugly, certainly. What can I do? */ -static void tcp_v4_send_ack(struct net *net, +static void tcp_v4_send_ack(const struct sock *sk, struct sk_buff *skb, u32 seq, u32 ack, u32 win, u32 tsval, u32 tsecr, int oif, struct tcp_md5sig_key *key, @@ -726,6 +727,7 @@ static void tcp_v4_send_ack(struct net *net, #endif ]; } rep; + struct net *net = sock_net(sk); struct ip_reply_arg arg; memset(&rep.th, 0, sizeof(struct tcphdr)); @@ -775,6 +777,7 @@ static void tcp_v4_send_ack(struct net *net, if (oif) arg.bound_dev_if = oif; arg.tos = tos; + arg.uid = sock_net_uid(net, sk_fullsock(sk) ? sk : NULL); local_bh_disable(); ip_send_unicast_reply(*this_cpu_ptr(net->ipv4.tcp_sk), skb, &TCP_SKB_CB(skb)->header.h4.opt, @@ -790,7 +793,7 @@ static void tcp_v4_timewait_ack(struct sock *sk, struct sk_buff *skb) struct inet_timewait_sock *tw = inet_twsk(sk); struct tcp_timewait_sock *tcptw = tcp_twsk(sk); - tcp_v4_send_ack(sock_net(sk), skb, + tcp_v4_send_ack(sk, skb, tcptw->tw_snd_nxt, tcptw->tw_rcv_nxt, tcptw->tw_rcv_wnd >> tw->tw_rcv_wscale, tcp_time_stamp + tcptw->tw_ts_offset, @@ -818,7 +821,7 @@ static void tcp_v4_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb, * exception of segments, MUST be right-shifted by * Rcv.Wind.Shift bits: */ - tcp_v4_send_ack(sock_net(sk), skb, seq, + tcp_v4_send_ack(sk, skb, seq, tcp_rsk(req)->rcv_nxt, req->rsk_rcv_wnd >> inet_rsk(req)->rcv_wscale, tcp_time_stamp, diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c index 896e9dfbdb5cd9..cd8e1898d4f407 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c @@ -188,7 +188,7 @@ u32 tcp_default_init_rwnd(u32 mss) * (RFC 3517, Section 4, NextSeg() rule (2)). Further place a * limit when mss is larger than 1460. */ - u32 init_rwnd = TCP_INIT_CWND * 2; + u32 init_rwnd = sysctl_tcp_default_init_rwnd; if (mss > 1460) init_rwnd = max((1460 * init_rwnd) / mss, 2U); diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index 5bab6c3f7a2fc1..5093bb84205669 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c @@ -1019,7 +1019,8 @@ int udp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) flowi4_init_output(fl4, ipc.oif, sk->sk_mark, tos, RT_SCOPE_UNIVERSE, sk->sk_protocol, flow_flags, - faddr, saddr, dport, inet->inet_sport); + faddr, saddr, dport, inet->inet_sport, + sk->sk_uid); security_sk_classify_flow(sk, flowi4_to_flowi(fl4)); rt = ip_route_output_flow(net, fl4, sk); diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c index 4bc5ba3ae452eb..e4570b5b180fca 100644 --- a/net/ipv6/addrconf.c +++ b/net/ipv6/addrconf.c @@ -226,6 +226,7 @@ static struct ipv6_devconf ipv6_devconf __read_mostly = { .accept_ra_rt_info_max_plen = 0, #endif #endif + .accept_ra_rt_table = 0, .proxy_ndp = 0, .accept_source_route = 0, /* we do not accept RH0 by default. */ .disable_ipv6 = 0, @@ -272,6 +273,7 @@ static struct ipv6_devconf ipv6_devconf_dflt __read_mostly = { .accept_ra_rt_info_max_plen = 0, #endif #endif + .accept_ra_rt_table = 0, .proxy_ndp = 0, .accept_source_route = 0, /* we do not accept RH0 by default. */ .disable_ipv6 = 0, @@ -2200,6 +2202,31 @@ static void ipv6_try_regen_rndid(struct inet6_dev *idev, struct in6_addr *tmpad ipv6_regen_rndid(idev); } +u32 addrconf_rt_table(const struct net_device *dev, u32 default_table) { + /* Determines into what table to put autoconf PIO/RIO/default routes + * learned on this device. + * + * - If 0, use the same table for every device. This puts routes into + * one of RT_TABLE_{PREFIX,INFO,DFLT} depending on the type of route + * (but note that these three are currently all equal to + * RT6_TABLE_MAIN). + * - If > 0, use the specified table. + * - If < 0, put routes into table dev->ifindex + (-rt_table). + */ + struct inet6_dev *idev = in6_dev_get(dev); + u32 table; + int sysctl = idev->cnf.accept_ra_rt_table; + if (sysctl == 0) { + table = default_table; + } else if (sysctl > 0) { + table = (u32) sysctl; + } else { + table = (unsigned) dev->ifindex + (-sysctl); + } + in6_dev_put(idev); + return table; +} + /* * Add prefix route. */ @@ -2209,7 +2236,7 @@ addrconf_prefix_route(struct in6_addr *pfx, int plen, struct net_device *dev, unsigned long expires, u32 flags) { struct fib6_config cfg = { - .fc_table = l3mdev_fib_table(dev) ? : RT6_TABLE_PREFIX, + .fc_table = l3mdev_fib_table(dev) ? : addrconf_rt_table(dev, RT6_TABLE_PREFIX), .fc_metric = IP6_RT_PRIO_ADDRCONF, .fc_ifindex = dev->ifindex, .fc_expires = expires, @@ -2242,7 +2269,7 @@ static struct rt6_info *addrconf_get_prefix_route(const struct in6_addr *pfx, struct fib6_node *fn; struct rt6_info *rt = NULL; struct fib6_table *table; - u32 tb_id = l3mdev_fib_table(dev) ? : RT6_TABLE_PREFIX; + u32 tb_id = l3mdev_fib_table(dev) ? : addrconf_rt_table(dev, RT6_TABLE_PREFIX); table = fib6_get_table(dev_net(dev), tb_id); if (!table) @@ -4928,6 +4955,7 @@ static inline void ipv6_store_devconf(struct ipv6_devconf *cnf, array[DEVCONF_ACCEPT_RA_RT_INFO_MAX_PLEN] = cnf->accept_ra_rt_info_max_plen; #endif #endif + array[DEVCONF_ACCEPT_RA_RT_TABLE] = cnf->accept_ra_rt_table; array[DEVCONF_PROXY_NDP] = cnf->proxy_ndp; array[DEVCONF_ACCEPT_SOURCE_ROUTE] = cnf->accept_source_route; #ifdef CONFIG_IPV6_OPTIMISTIC_DAD @@ -5515,8 +5543,7 @@ static void addrconf_disable_change(struct net *net, __s32 newf) struct net_device *dev; struct inet6_dev *idev; - rcu_read_lock(); - for_each_netdev_rcu(net, dev) { + for_each_netdev(net, dev) { idev = __in6_dev_get(dev); if (idev) { int changed = (!idev->cnf.disable_ipv6) ^ (!newf); @@ -5525,7 +5552,6 @@ static void addrconf_disable_change(struct net *net, __s32 newf) dev_disable_change(idev); } } - rcu_read_unlock(); } static int addrconf_disable_ipv6(struct ctl_table *table, int *p, int newf) @@ -5910,6 +5936,13 @@ static const struct ctl_table addrconf_sysctl[] = { }, #endif #endif + { + .procname = "accept_ra_rt_table", + .data = &ipv6_devconf.accept_ra_rt_table, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, { .procname = "proxy_ndp", .data = &ipv6_devconf.proxy_ndp, diff --git a/net/ipv6/af_inet6.c b/net/ipv6/af_inet6.c index 46ad699937fdcc..02816456a53ecd 100644 --- a/net/ipv6/af_inet6.c +++ b/net/ipv6/af_inet6.c @@ -65,6 +65,20 @@ #include #include +#ifdef CONFIG_ANDROID_PARANOID_NETWORK +#include + +static inline int current_has_network(void) +{ + return in_egroup_p(AID_INET) || capable(CAP_NET_RAW); +} +#else +static inline int current_has_network(void) +{ + return 1; +} +#endif + #include "ip6_offload.h" MODULE_AUTHOR("Cast of dozens"); @@ -121,6 +135,9 @@ static int inet6_create(struct net *net, struct socket *sock, int protocol, if (protocol < 0 || protocol >= IPPROTO_MAX) return -EINVAL; + if (!current_has_network()) + return -EACCES; + /* Look for the requested type/protocol pair. */ lookup_protocol: err = -ESOCKTNOSUPPORT; @@ -167,8 +184,7 @@ static int inet6_create(struct net *net, struct socket *sock, int protocol, } err = -EPERM; - if (sock->type == SOCK_RAW && !kern && - !ns_capable(net->user_ns, CAP_NET_RAW)) + if (sock->type == SOCK_RAW && !kern && !capable(CAP_NET_RAW)) goto out_rcu_unlock; sock->ops = answer->ops; @@ -678,6 +694,7 @@ int inet6_sk_rebuild_header(struct sock *sk) fl6.flowi6_mark = sk->sk_mark; fl6.fl6_dport = inet->inet_dport; fl6.fl6_sport = inet->inet_sport; + fl6.flowi6_uid = sk->sk_uid; security_sk_classify_flow(sk, flowi6_to_flowi(&fl6)); rcu_read_lock(); diff --git a/net/ipv6/ah6.c b/net/ipv6/ah6.c index 0630a4d5daaaba..189eb10b742d02 100644 --- a/net/ipv6/ah6.c +++ b/net/ipv6/ah6.c @@ -662,9 +662,10 @@ static int ah6_err(struct sk_buff *skb, struct inet6_skb_parm *opt, return 0; if (type == NDISC_REDIRECT) - ip6_redirect(skb, net, skb->dev->ifindex, 0); + ip6_redirect(skb, net, skb->dev->ifindex, 0, + sock_net_uid(net, NULL)); else - ip6_update_pmtu(skb, net, info, 0, 0); + ip6_update_pmtu(skb, net, info, 0, 0, sock_net_uid(net, NULL)); xfrm_state_put(x); return 0; diff --git a/net/ipv6/datagram.c b/net/ipv6/datagram.c index ccf40550c475d3..1529833f085c49 100644 --- a/net/ipv6/datagram.c +++ b/net/ipv6/datagram.c @@ -54,6 +54,7 @@ static void ip6_datagram_flow_key_init(struct flowi6 *fl6, struct sock *sk) fl6->fl6_dport = inet->inet_dport; fl6->fl6_sport = inet->inet_sport; fl6->flowlabel = np->flow_label; + fl6->flowi6_uid = sk->sk_uid; if (!fl6->flowi6_oif) fl6->flowi6_oif = np->sticky_pktinfo.ipi6_ifindex; @@ -700,7 +701,7 @@ void ip6_datagram_recv_specific_ctl(struct sock *sk, struct msghdr *msg, struct sockaddr_in6 sin6; __be16 *ports = (__be16 *) skb_transport_header(skb); - if (skb_transport_offset(skb) + 4 <= skb->len) { + if (skb_transport_offset(skb) + 4 <= (int)skb->len) { /* All current transport protocols have the port numbers in the * first four bytes of the transport header and this function is * written with this assumption in mind. diff --git a/net/ipv6/esp6.c b/net/ipv6/esp6.c index 111ba55fd512fb..cbcdd5db31f473 100644 --- a/net/ipv6/esp6.c +++ b/net/ipv6/esp6.c @@ -474,9 +474,10 @@ static int esp6_err(struct sk_buff *skb, struct inet6_skb_parm *opt, return 0; if (type == NDISC_REDIRECT) - ip6_redirect(skb, net, skb->dev->ifindex, 0); + ip6_redirect(skb, net, skb->dev->ifindex, 0, + sock_net_uid(net, NULL)); else - ip6_update_pmtu(skb, net, info, 0, 0); + ip6_update_pmtu(skb, net, info, 0, 0, sock_net_uid(net, NULL)); xfrm_state_put(x); return 0; diff --git a/net/ipv6/exthdrs_core.c b/net/ipv6/exthdrs_core.c index 305e2ed730bf42..477692f80f0da4 100644 --- a/net/ipv6/exthdrs_core.c +++ b/net/ipv6/exthdrs_core.c @@ -166,15 +166,15 @@ EXPORT_SYMBOL_GPL(ipv6_find_tlv); * to explore inner IPv6 header, eg. ICMPv6 error messages. * * If target header is found, its offset is set in *offset and return protocol - * number. Otherwise, return -1. + * number. Otherwise, return -ENOENT or -EBADMSG. * * If the first fragment doesn't contain the final protocol header or * NEXTHDR_NONE it is considered invalid. * * Note that non-1st fragment is special case that "the protocol number * of last header" is "next header" field in Fragment header. In this case, - * *offset is meaningless and fragment offset is stored in *fragoff if fragoff - * isn't NULL. + * *offset is meaningless. If fragoff is not NULL, the fragment offset is + * stored in *fragoff; if it is NULL, return -EINVAL. * * if flags is not NULL and it's a fragment, then the frag flag * IP6_FH_F_FRAG will be set. If it's an AH header, the @@ -253,9 +253,12 @@ int ipv6_find_hdr(const struct sk_buff *skb, unsigned int *offset, if (target < 0 && ((!ipv6_ext_hdr(hp->nexthdr)) || hp->nexthdr == NEXTHDR_NONE)) { - if (fragoff) + if (fragoff) { *fragoff = _frag_off; - return hp->nexthdr; + return hp->nexthdr; + } else { + return -EINVAL; + } } if (!found) return -ENOENT; diff --git a/net/ipv6/icmp.c b/net/ipv6/icmp.c index 2772004ba5a188..17fa28f7a0ffe8 100644 --- a/net/ipv6/icmp.c +++ b/net/ipv6/icmp.c @@ -92,9 +92,10 @@ static void icmpv6_err(struct sk_buff *skb, struct inet6_skb_parm *opt, struct net *net = dev_net(skb->dev); if (type == ICMPV6_PKT_TOOBIG) - ip6_update_pmtu(skb, net, info, 0, 0); + ip6_update_pmtu(skb, net, info, 0, 0, sock_net_uid(net, NULL)); else if (type == NDISC_REDIRECT) - ip6_redirect(skb, net, skb->dev->ifindex, 0); + ip6_redirect(skb, net, skb->dev->ifindex, 0, + sock_net_uid(net, NULL)); if (!(type & ICMPV6_INFOMSG_MASK)) if (icmp6->icmp6_type == ICMPV6_ECHO_REQUEST) @@ -486,6 +487,7 @@ static void icmp6_send(struct sk_buff *skb, u8 type, u8 code, __u32 info, fl6.flowi6_oif = iif; fl6.fl6_icmp_type = type; fl6.fl6_icmp_code = code; + fl6.flowi6_uid = sock_net_uid(net, NULL); security_skb_classify_flow(skb, flowi6_to_flowi(&fl6)); sk = icmpv6_xmit_lock(net); @@ -660,6 +662,7 @@ static void icmpv6_echo_reply(struct sk_buff *skb) fl6.flowi6_oif = skb->dev->ifindex; fl6.fl6_icmp_type = ICMPV6_ECHO_REPLY; fl6.flowi6_mark = mark; + fl6.flowi6_uid = sock_net_uid(net, NULL); security_skb_classify_flow(skb, flowi6_to_flowi(&fl6)); sk = icmpv6_xmit_lock(net); diff --git a/net/ipv6/ila/ila_lwt.c b/net/ipv6/ila/ila_lwt.c index e50c27a93e17d8..f3db364fc85360 100644 --- a/net/ipv6/ila/ila_lwt.c +++ b/net/ipv6/ila/ila_lwt.c @@ -164,6 +164,7 @@ static const struct lwtunnel_encap_ops ila_encap_ops = { .fill_encap = ila_fill_encap_info, .get_encap_size = ila_encap_nlsize, .cmp_encap = ila_encap_cmp, + .owner = THIS_MODULE, }; int ila_lwt_init(void) diff --git a/net/ipv6/inet6_connection_sock.c b/net/ipv6/inet6_connection_sock.c index 532c3ef282c5f0..1c86c478f578b4 100644 --- a/net/ipv6/inet6_connection_sock.c +++ b/net/ipv6/inet6_connection_sock.c @@ -88,6 +88,7 @@ struct dst_entry *inet6_csk_route_req(const struct sock *sk, fl6->flowi6_mark = ireq->ir_mark; fl6->fl6_dport = ireq->ir_rmt_port; fl6->fl6_sport = htons(ireq->ir_num); + fl6->flowi6_uid = sk->sk_uid; security_req_classify_flow(req, flowi6_to_flowi(fl6)); dst = ip6_dst_lookup_flow(sk, fl6, final_p); @@ -136,6 +137,7 @@ static struct dst_entry *inet6_csk_route_socket(struct sock *sk, fl6->flowi6_mark = sk->sk_mark; fl6->fl6_sport = inet->inet_sport; fl6->fl6_dport = inet->inet_dport; + fl6->flowi6_uid = sk->sk_uid; security_sk_classify_flow(sk, flowi6_to_flowi(fl6)); rcu_read_lock(); diff --git a/net/ipv6/ip6_gre.c b/net/ipv6/ip6_gre.c index d7d6d3ae0b3b62..710bc79f91131b 100644 --- a/net/ipv6/ip6_gre.c +++ b/net/ipv6/ip6_gre.c @@ -548,6 +548,8 @@ static inline int ip6gre_xmit_ipv4(struct sk_buff *skb, struct net_device *dev) if (t->parms.flags & IP6_TNL_F_USE_ORIG_FWMARK) fl6.flowi6_mark = skb->mark; + fl6.flowi6_uid = sock_net_uid(dev_net(dev), NULL); + err = gre_handle_offloads(skb, !!(t->parms.o_flags & TUNNEL_CSUM)); if (err) return -1; @@ -602,6 +604,8 @@ static inline int ip6gre_xmit_ipv6(struct sk_buff *skb, struct net_device *dev) if (t->parms.flags & IP6_TNL_F_USE_ORIG_FWMARK) fl6.flowi6_mark = skb->mark; + fl6.flowi6_uid = sock_net_uid(dev_net(dev), NULL); + if (gre_handle_offloads(skb, !!(t->parms.o_flags & TUNNEL_CSUM))) return -1; diff --git a/net/ipv6/ip6_offload.c b/net/ipv6/ip6_offload.c index 89c59e656f4493..fc7b4017ba241f 100644 --- a/net/ipv6/ip6_offload.c +++ b/net/ipv6/ip6_offload.c @@ -191,6 +191,7 @@ static struct sk_buff **ipv6_gro_receive(struct sk_buff **head, ops = rcu_dereference(inet6_offloads[proto]); if (!ops || !ops->callbacks.gro_receive) { __pskb_pull(skb, skb_gro_offset(skb)); + skb_gro_frag0_invalidate(skb); proto = ipv6_gso_pull_exthdrs(skb, proto); skb_gro_pull(skb, -skb_transport_offset(skb)); skb_reset_transport_header(skb); diff --git a/net/ipv6/ip6_tunnel.c b/net/ipv6/ip6_tunnel.c index d76674efe523cf..c1f497bb398faf 100644 --- a/net/ipv6/ip6_tunnel.c +++ b/net/ipv6/ip6_tunnel.c @@ -1108,7 +1108,7 @@ int ip6_tnl_xmit(struct sk_buff *skb, struct net_device *dev, __u8 dsfield, t->parms.name); goto tx_err_dst_release; } - mtu = dst_mtu(dst) - psh_hlen; + mtu = dst_mtu(dst) - psh_hlen - t->tun_hlen; if (encap_limit >= 0) { max_headroom += 8; mtu -= 8; @@ -1117,7 +1117,7 @@ int ip6_tnl_xmit(struct sk_buff *skb, struct net_device *dev, __u8 dsfield, mtu = IPV6_MIN_MTU; if (skb_dst(skb) && !t->parms.collect_md) skb_dst(skb)->ops->update_pmtu(skb_dst(skb), NULL, skb, mtu); - if (skb->len > mtu && !skb_is_gso(skb)) { + if (skb->len - t->tun_hlen > mtu && !skb_is_gso(skb)) { *pmtu = mtu; err = -EMSGSIZE; goto tx_err_dst_release; @@ -1248,6 +1248,8 @@ ip4ip6_tnl_xmit(struct sk_buff *skb, struct net_device *dev) fl6.flowi6_mark = skb->mark; } + fl6.flowi6_uid = sock_net_uid(dev_net(dev), NULL); + if (iptunnel_handle_offloads(skb, SKB_GSO_IPXIP6)) return -1; @@ -1326,6 +1328,8 @@ ip6ip6_tnl_xmit(struct sk_buff *skb, struct net_device *dev) fl6.flowi6_mark = skb->mark; } + fl6.flowi6_uid = sock_net_uid(dev_net(dev), NULL); + if (iptunnel_handle_offloads(skb, SKB_GSO_IPXIP6)) return -1; diff --git a/net/ipv6/ip6_vti.c b/net/ipv6/ip6_vti.c index c299c1e2bbf06e..3bce120cf5a4b3 100644 --- a/net/ipv6/ip6_vti.c +++ b/net/ipv6/ip6_vti.c @@ -608,9 +608,10 @@ static int vti6_err(struct sk_buff *skb, struct inet6_skb_parm *opt, return 0; if (type == NDISC_REDIRECT) - ip6_redirect(skb, net, skb->dev->ifindex, 0); + ip6_redirect(skb, net, skb->dev->ifindex, 0, + sock_net_uid(net, NULL)); else - ip6_update_pmtu(skb, net, info, 0, 0); + ip6_update_pmtu(skb, net, info, 0, 0, sock_net_uid(net, NULL)); xfrm_state_put(x); return 0; diff --git a/net/ipv6/ipcomp6.c b/net/ipv6/ipcomp6.c index 1b9316e1386a96..54d165b9845a02 100644 --- a/net/ipv6/ipcomp6.c +++ b/net/ipv6/ipcomp6.c @@ -74,9 +74,10 @@ static int ipcomp6_err(struct sk_buff *skb, struct inet6_skb_parm *opt, return 0; if (type == NDISC_REDIRECT) - ip6_redirect(skb, net, skb->dev->ifindex, 0); + ip6_redirect(skb, net, skb->dev->ifindex, 0, + sock_net_uid(net, NULL)); else - ip6_update_pmtu(skb, net, info, 0, 0); + ip6_update_pmtu(skb, net, info, 0, 0, sock_net_uid(net, NULL)); xfrm_state_put(x); return 0; diff --git a/net/ipv6/netfilter.c b/net/ipv6/netfilter.c index d11c46833d615b..39970e212ad574 100644 --- a/net/ipv6/netfilter.c +++ b/net/ipv6/netfilter.c @@ -26,6 +26,7 @@ int ip6_route_me_harder(struct net *net, struct sk_buff *skb) struct flowi6 fl6 = { .flowi6_oif = skb->sk ? skb->sk->sk_bound_dev_if : 0, .flowi6_mark = skb->mark, + .flowi6_uid = sock_net_uid(net, skb->sk), .daddr = iph->daddr, .saddr = iph->saddr, }; diff --git a/net/ipv6/ping.c b/net/ipv6/ping.c index 66e2d9dfc43a87..e1f8b34d7a2ef8 100644 --- a/net/ipv6/ping.c +++ b/net/ipv6/ping.c @@ -113,6 +113,7 @@ static int ping_v6_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) fl6.daddr = *daddr; fl6.flowi6_oif = oif; fl6.flowi6_mark = sk->sk_mark; + fl6.flowi6_uid = sk->sk_uid; fl6.fl6_icmp_type = user_icmph.icmp6_type; fl6.fl6_icmp_code = user_icmph.icmp6_code; security_sk_classify_flow(sk, flowi6_to_flowi(&fl6)); diff --git a/net/ipv6/raw.c b/net/ipv6/raw.c index 054a1d84fc5e94..566537510c2648 100644 --- a/net/ipv6/raw.c +++ b/net/ipv6/raw.c @@ -589,7 +589,11 @@ static int rawv6_push_pending_frames(struct sock *sk, struct flowi6 *fl6, } offset += skb_transport_offset(skb); - BUG_ON(skb_copy_bits(skb, offset, &csum, 2)); + err = skb_copy_bits(skb, offset, &csum, 2); + if (err < 0) { + ip6_flush_pending_frames(sk); + goto out; + } /* in case cksum was not initialized */ if (unlikely(csum)) @@ -774,6 +778,7 @@ static int rawv6_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) memset(&fl6, 0, sizeof(fl6)); fl6.flowi6_mark = sk->sk_mark; + fl6.flowi6_uid = sk->sk_uid; ipc6.hlimit = -1; ipc6.tclass = -1; diff --git a/net/ipv6/route.c b/net/ipv6/route.c index 1b57e11e6e0db4..79d1781bb17161 100644 --- a/net/ipv6/route.c +++ b/net/ipv6/route.c @@ -1408,7 +1408,7 @@ static void ip6_rt_update_pmtu(struct dst_entry *dst, struct sock *sk, } void ip6_update_pmtu(struct sk_buff *skb, struct net *net, __be32 mtu, - int oif, u32 mark) + int oif, u32 mark, kuid_t uid) { const struct ipv6hdr *iph = (struct ipv6hdr *) skb->data; struct dst_entry *dst; @@ -1420,6 +1420,7 @@ void ip6_update_pmtu(struct sk_buff *skb, struct net *net, __be32 mtu, fl6.daddr = iph->daddr; fl6.saddr = iph->saddr; fl6.flowlabel = ip6_flowinfo(iph); + fl6.flowi6_uid = uid; dst = ip6_route_output(net, NULL, &fl6); if (!dst->error) @@ -1433,7 +1434,7 @@ void ip6_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, __be32 mtu) struct dst_entry *dst; ip6_update_pmtu(skb, sock_net(sk), mtu, - sk->sk_bound_dev_if, sk->sk_mark); + sk->sk_bound_dev_if, sk->sk_mark, sk->sk_uid); dst = __sk_dst_get(sk); if (!dst || !dst->obsolete || @@ -1525,7 +1526,8 @@ static struct dst_entry *ip6_route_redirect(struct net *net, flags, __ip6_route_redirect); } -void ip6_redirect(struct sk_buff *skb, struct net *net, int oif, u32 mark) +void ip6_redirect(struct sk_buff *skb, struct net *net, int oif, u32 mark, + kuid_t uid) { const struct ipv6hdr *iph = (struct ipv6hdr *) skb->data; struct dst_entry *dst; @@ -1538,6 +1540,7 @@ void ip6_redirect(struct sk_buff *skb, struct net *net, int oif, u32 mark) fl6.daddr = iph->daddr; fl6.saddr = iph->saddr; fl6.flowlabel = ip6_flowinfo(iph); + fl6.flowi6_uid = uid; dst = ip6_route_redirect(net, &fl6, &ipv6_hdr(skb)->saddr); rt6_do_redirect(dst, NULL, skb); @@ -1559,6 +1562,7 @@ void ip6_redirect_no_header(struct sk_buff *skb, struct net *net, int oif, fl6.flowi6_mark = mark; fl6.daddr = msg->dest; fl6.saddr = iph->daddr; + fl6.flowi6_uid = sock_net_uid(net, NULL); dst = ip6_route_redirect(net, &fl6, &iph->saddr); rt6_do_redirect(dst, NULL, skb); @@ -1567,7 +1571,8 @@ void ip6_redirect_no_header(struct sk_buff *skb, struct net *net, int oif, void ip6_sk_redirect(struct sk_buff *skb, struct sock *sk) { - ip6_redirect(skb, sock_net(sk), sk->sk_bound_dev_if, sk->sk_mark); + ip6_redirect(skb, sock_net(sk), sk->sk_bound_dev_if, sk->sk_mark, + sk->sk_uid); } EXPORT_SYMBOL_GPL(ip6_sk_redirect); @@ -2335,8 +2340,7 @@ static struct rt6_info *rt6_get_route_info(struct net *net, const struct in6_addr *gwaddr, struct net_device *dev) { - u32 tb_id = l3mdev_fib_table(dev) ? : RT6_TABLE_INFO; - int ifindex = dev->ifindex; + u32 tb_id = l3mdev_fib_table(dev) ? : addrconf_rt_table(dev, RT6_TABLE_INFO); struct fib6_node *fn; struct rt6_info *rt = NULL; struct fib6_table *table; @@ -2351,7 +2355,7 @@ static struct rt6_info *rt6_get_route_info(struct net *net, goto out; for (rt = fn->leaf; rt; rt = rt->dst.rt6_next) { - if (rt->dst.dev->ifindex != ifindex) + if (rt->dst.dev->ifindex != dev->ifindex) continue; if ((rt->rt6i_flags & (RTF_ROUTEINFO|RTF_GATEWAY)) != (RTF_ROUTEINFO|RTF_GATEWAY)) continue; @@ -2382,7 +2386,7 @@ static struct rt6_info *rt6_add_route_info(struct net *net, .fc_nlinfo.nl_net = net, }; - cfg.fc_table = l3mdev_fib_table(dev) ? : RT6_TABLE_INFO, + cfg.fc_table = l3mdev_fib_table(dev) ? : addrconf_rt_table(dev, RT6_TABLE_INFO), cfg.fc_dst = *prefix; cfg.fc_gateway = *gwaddr; @@ -2398,7 +2402,7 @@ static struct rt6_info *rt6_add_route_info(struct net *net, struct rt6_info *rt6_get_dflt_router(const struct in6_addr *addr, struct net_device *dev) { - u32 tb_id = l3mdev_fib_table(dev) ? : RT6_TABLE_DFLT; + u32 tb_id = l3mdev_fib_table(dev) ? : addrconf_rt_table(dev, RT6_TABLE_MAIN); struct rt6_info *rt; struct fib6_table *table; @@ -2424,7 +2428,7 @@ struct rt6_info *rt6_add_dflt_router(const struct in6_addr *gwaddr, unsigned int pref) { struct fib6_config cfg = { - .fc_table = l3mdev_fib_table(dev) ? : RT6_TABLE_DFLT, + .fc_table = l3mdev_fib_table(dev) ? : addrconf_rt_table(dev, RT6_TABLE_DFLT), .fc_metric = IP6_RT_PRIO_USER, .fc_ifindex = dev->ifindex, .fc_flags = RTF_GATEWAY | RTF_ADDRCONF | RTF_DEFAULT | @@ -2447,43 +2451,16 @@ struct rt6_info *rt6_add_dflt_router(const struct in6_addr *gwaddr, return rt6_get_dflt_router(gwaddr, dev); } -static void __rt6_purge_dflt_routers(struct fib6_table *table) -{ - struct rt6_info *rt; - -restart: - read_lock_bh(&table->tb6_lock); - for (rt = table->tb6_root.leaf; rt; rt = rt->dst.rt6_next) { - if (rt->rt6i_flags & (RTF_DEFAULT | RTF_ADDRCONF) && - (!rt->rt6i_idev || rt->rt6i_idev->cnf.accept_ra != 2)) { - dst_hold(&rt->dst); - read_unlock_bh(&table->tb6_lock); - ip6_del_rt(rt); - goto restart; - } - } - read_unlock_bh(&table->tb6_lock); - - table->flags &= ~RT6_TABLE_HAS_DFLT_ROUTER; +int rt6_addrconf_purge(struct rt6_info *rt, void *arg) { + if (rt->rt6i_flags & (RTF_DEFAULT | RTF_ADDRCONF) && + (!rt->rt6i_idev || rt->rt6i_idev->cnf.accept_ra != 2)) + return -1; + return 0; } void rt6_purge_dflt_routers(struct net *net) { - struct fib6_table *table; - struct hlist_head *head; - unsigned int h; - - rcu_read_lock(); - - for (h = 0; h < FIB6_TABLE_HASHSZ; h++) { - head = &net->ipv6.fib_table_hash[h]; - hlist_for_each_entry_rcu(table, head, tb6_hlist) { - if (table->flags & RT6_TABLE_HAS_DFLT_ROUTER) - __rt6_purge_dflt_routers(table); - } - } - - rcu_read_unlock(); + fib6_clean_all(net, rt6_addrconf_purge, NULL); } static void rtmsg_to_fib6_config(struct net *net, @@ -2801,6 +2778,7 @@ static const struct nla_policy rtm_ipv6_policy[RTA_MAX+1] = { [RTA_ENCAP_TYPE] = { .type = NLA_U16 }, [RTA_ENCAP] = { .type = NLA_NESTED }, [RTA_EXPIRES] = { .type = NLA_U32 }, + [RTA_UID] = { .type = NLA_U32 }, }; static int rtm_to_fib6_config(struct sk_buff *skb, struct nlmsghdr *nlh, @@ -2885,6 +2863,11 @@ static int rtm_to_fib6_config(struct sk_buff *skb, struct nlmsghdr *nlh, if (tb[RTA_MULTIPATH]) { cfg->fc_mp = nla_data(tb[RTA_MULTIPATH]); cfg->fc_mp_len = nla_len(tb[RTA_MULTIPATH]); + + err = lwtunnel_valid_encap_type_attr(cfg->fc_mp, + cfg->fc_mp_len); + if (err < 0) + goto errout; } if (tb[RTA_PREF]) { @@ -2898,9 +2881,14 @@ static int rtm_to_fib6_config(struct sk_buff *skb, struct nlmsghdr *nlh, if (tb[RTA_ENCAP]) cfg->fc_encap = tb[RTA_ENCAP]; - if (tb[RTA_ENCAP_TYPE]) + if (tb[RTA_ENCAP_TYPE]) { cfg->fc_encap_type = nla_get_u16(tb[RTA_ENCAP_TYPE]); + err = lwtunnel_valid_encap_type(cfg->fc_encap_type); + if (err < 0) + goto errout; + } + if (tb[RTA_EXPIRES]) { unsigned long timeout = addrconf_timeout_fixup(nla_get_u32(tb[RTA_EXPIRES]), HZ); @@ -3306,7 +3294,8 @@ static int rt6_fill_node(struct net *net, if (nla_put_u8(skb, RTA_PREF, IPV6_EXTRACT_PREF(rt->rt6i_flags))) goto nla_put_failure; - lwtunnel_fill_encap(skb, rt->dst.lwtstate); + if (lwtunnel_fill_encap(skb, rt->dst.lwtstate) < 0) + goto nla_put_failure; nlmsg_end(skb, nlh); return 0; @@ -3375,6 +3364,12 @@ static int inet6_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh) if (tb[RTA_MARK]) fl6.flowi6_mark = nla_get_u32(tb[RTA_MARK]); + if (tb[RTA_UID]) + fl6.flowi6_uid = make_kuid(current_user_ns(), + nla_get_u32(tb[RTA_UID])); + else + fl6.flowi6_uid = iif ? INVALID_UID : current_uid(); + if (iif) { struct net_device *dev; int flags = 0; diff --git a/net/ipv6/syncookies.c b/net/ipv6/syncookies.c index 59c483937aec1c..97830a6a9cbb25 100644 --- a/net/ipv6/syncookies.c +++ b/net/ipv6/syncookies.c @@ -227,6 +227,7 @@ struct sock *cookie_v6_check(struct sock *sk, struct sk_buff *skb) fl6.flowi6_mark = ireq->ir_mark; fl6.fl6_dport = ireq->ir_rmt_port; fl6.fl6_sport = inet_sk(sk)->inet_sport; + fl6.flowi6_uid = sk->sk_uid; security_req_classify_flow(req, flowi6_to_flowi(&fl6)); dst = ip6_dst_lookup_flow(sk, &fl6, final_p); diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c index b9f1fee9a8862a..28ec0a2e7b7295 100644 --- a/net/ipv6/tcp_ipv6.c +++ b/net/ipv6/tcp_ipv6.c @@ -233,6 +233,7 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr, fl6.flowi6_mark = sk->sk_mark; fl6.fl6_dport = usin->sin6_port; fl6.fl6_sport = inet->inet_sport; + fl6.flowi6_uid = sk->sk_uid; opt = rcu_dereference_protected(np->opt, lockdep_sock_is_held(sk)); final_p = fl6_update_dst(&fl6, opt, &final); @@ -828,6 +829,7 @@ static void tcp_v6_send_response(const struct sock *sk, struct sk_buff *skb, u32 fl6.flowi6_mark = IP6_REPLY_MARK(net, skb->mark); fl6.fl6_dport = t1->dest; fl6.fl6_sport = t1->source; + fl6.flowi6_uid = sock_net_uid(net, sk && sk_fullsock(sk) ? sk : NULL); security_skb_classify_flow(skb, flowi6_to_flowi(&fl6)); /* Pass a socket to ip6_dst_lookup either it is for RST diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c index e4a8000d59ad6f..f6fbd257cf7ec8 100644 --- a/net/ipv6/udp.c +++ b/net/ipv6/udp.c @@ -1156,6 +1156,7 @@ int udpv6_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) fl6.flowi6_oif = np->sticky_pktinfo.ipi6_ifindex; fl6.flowi6_mark = sk->sk_mark; + fl6.flowi6_uid = sk->sk_uid; sockc.tsflags = sk->sk_tsflags; if (msg->msg_controllen) { diff --git a/net/iucv/af_iucv.c b/net/iucv/af_iucv.c index 02b45a8e8b3572..91cbbf1c3f82da 100644 --- a/net/iucv/af_iucv.c +++ b/net/iucv/af_iucv.c @@ -1036,7 +1036,8 @@ static int iucv_sock_sendmsg(struct socket *sock, struct msghdr *msg, { struct sock *sk = sock->sk; struct iucv_sock *iucv = iucv_sk(sk); - size_t headroom, linear; + size_t headroom = 0; + size_t linear; struct sk_buff *skb; struct iucv_message txmsg = {0}; struct cmsghdr *cmsg; @@ -1114,18 +1115,20 @@ static int iucv_sock_sendmsg(struct socket *sock, struct msghdr *msg, * this is fine for SOCK_SEQPACKET (unless we want to support * segmented records using the MSG_EOR flag), but * for SOCK_STREAM we might want to improve it in future */ - headroom = (iucv->transport == AF_IUCV_TRANS_HIPER) - ? sizeof(struct af_iucv_trans_hdr) + ETH_HLEN : 0; - if (headroom + len < PAGE_SIZE) { + if (iucv->transport == AF_IUCV_TRANS_HIPER) { + headroom = sizeof(struct af_iucv_trans_hdr) + ETH_HLEN; linear = len; } else { - /* In nonlinear "classic" iucv skb, - * reserve space for iucv_array - */ - if (iucv->transport != AF_IUCV_TRANS_HIPER) - headroom += sizeof(struct iucv_array) * - (MAX_SKB_FRAGS + 1); - linear = PAGE_SIZE - headroom; + if (len < PAGE_SIZE) { + linear = len; + } else { + /* In nonlinear "classic" iucv skb, + * reserve space for iucv_array + */ + headroom = sizeof(struct iucv_array) * + (MAX_SKB_FRAGS + 1); + linear = PAGE_SIZE - headroom; + } } skb = sock_alloc_send_pskb(sk, headroom + linear, len - linear, noblock, &err, 0); diff --git a/net/l2tp/l2tp_ip6.c b/net/l2tp/l2tp_ip6.c index aa821cb639e541..f092ac441fdda5 100644 --- a/net/l2tp/l2tp_ip6.c +++ b/net/l2tp/l2tp_ip6.c @@ -525,6 +525,7 @@ static int l2tp_ip6_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) memset(&fl6, 0, sizeof(fl6)); fl6.flowi6_mark = sk->sk_mark; + fl6.flowi6_uid = sk->sk_uid; ipc6.hlimit = -1; ipc6.tclass = -1; diff --git a/net/mac80211/agg-rx.c b/net/mac80211/agg-rx.c index f6749dced021bc..3b5fd4188f2ac7 100644 --- a/net/mac80211/agg-rx.c +++ b/net/mac80211/agg-rx.c @@ -315,11 +315,7 @@ void __ieee80211_start_rx_ba_session(struct sta_info *sta, mutex_lock(&sta->ampdu_mlme.mtx); if (test_bit(tid, sta->ampdu_mlme.agg_session_valid)) { - tid_agg_rx = rcu_dereference_protected( - sta->ampdu_mlme.tid_rx[tid], - lockdep_is_held(&sta->ampdu_mlme.mtx)); - - if (tid_agg_rx->dialog_token == dialog_token) { + if (sta->ampdu_mlme.tid_rx_token[tid] == dialog_token) { ht_dbg_ratelimited(sta->sdata, "updated AddBA Req from %pM on tid %u\n", sta->sta.addr, tid); @@ -396,7 +392,6 @@ void __ieee80211_start_rx_ba_session(struct sta_info *sta, } /* update data */ - tid_agg_rx->dialog_token = dialog_token; tid_agg_rx->ssn = start_seq_num; tid_agg_rx->head_seq_num = start_seq_num; tid_agg_rx->buf_size = buf_size; @@ -418,6 +413,7 @@ void __ieee80211_start_rx_ba_session(struct sta_info *sta, if (status == WLAN_STATUS_SUCCESS) { __set_bit(tid, sta->ampdu_mlme.agg_session_valid); __clear_bit(tid, sta->ampdu_mlme.unexpected_agg); + sta->ampdu_mlme.tid_rx_token[tid] = dialog_token; } mutex_unlock(&sta->ampdu_mlme.mtx); diff --git a/net/mac80211/debugfs_sta.c b/net/mac80211/debugfs_sta.c index a2fcdb47a0e684..14ec63a026693c 100644 --- a/net/mac80211/debugfs_sta.c +++ b/net/mac80211/debugfs_sta.c @@ -205,7 +205,7 @@ static ssize_t sta_agg_status_read(struct file *file, char __user *userbuf, p += scnprintf(p, sizeof(buf) + buf - p, "%02d", i); p += scnprintf(p, sizeof(buf) + buf - p, "\t\t%x", !!tid_rx); p += scnprintf(p, sizeof(buf) + buf - p, "\t%#.2x", - tid_rx ? tid_rx->dialog_token : 0); + tid_rx ? sta->ampdu_mlme.tid_rx_token[i] : 0); p += scnprintf(p, sizeof(buf) + buf - p, "\t%#.3x", tid_rx ? tid_rx->ssn : 0); diff --git a/net/mac80211/mesh.c b/net/mac80211/mesh.c index 42120d965263d2..50e1b7f78bd496 100644 --- a/net/mac80211/mesh.c +++ b/net/mac80211/mesh.c @@ -339,7 +339,7 @@ int mesh_add_vendor_ies(struct ieee80211_sub_if_data *sdata, /* fast-forward to vendor IEs */ offset = ieee80211_ie_split_vendor(ifmsh->ie, ifmsh->ie_len, 0); - if (offset) { + if (offset < ifmsh->ie_len) { len = ifmsh->ie_len - offset; data = ifmsh->ie + offset; if (skb_tailroom(skb) < len) diff --git a/net/mac80211/mlme.c b/net/mac80211/mlme.c index 7486f2dab4ba70..1118c61f835dde 100644 --- a/net/mac80211/mlme.c +++ b/net/mac80211/mlme.c @@ -2510,7 +2510,7 @@ static void ieee80211_destroy_auth_data(struct ieee80211_sub_if_data *sdata, } static void ieee80211_destroy_assoc_data(struct ieee80211_sub_if_data *sdata, - bool assoc) + bool assoc, bool abandon) { struct ieee80211_mgd_assoc_data *assoc_data = sdata->u.mgd.assoc_data; @@ -2533,6 +2533,9 @@ static void ieee80211_destroy_assoc_data(struct ieee80211_sub_if_data *sdata, mutex_lock(&sdata->local->mtx); ieee80211_vif_release_channel(sdata); mutex_unlock(&sdata->local->mtx); + + if (abandon) + cfg80211_abandon_assoc(sdata->dev, assoc_data->bss); } kfree(assoc_data); @@ -2762,7 +2765,7 @@ static void ieee80211_rx_mgmt_deauth(struct ieee80211_sub_if_data *sdata, bssid, reason_code, ieee80211_get_reason_code_string(reason_code)); - ieee80211_destroy_assoc_data(sdata, false); + ieee80211_destroy_assoc_data(sdata, false, true); cfg80211_rx_mlme_mgmt(sdata->dev, (u8 *)mgmt, len); return; @@ -3167,14 +3170,14 @@ static void ieee80211_rx_mgmt_assoc_resp(struct ieee80211_sub_if_data *sdata, if (status_code != WLAN_STATUS_SUCCESS) { sdata_info(sdata, "%pM denied association (code=%d)\n", mgmt->sa, status_code); - ieee80211_destroy_assoc_data(sdata, false); + ieee80211_destroy_assoc_data(sdata, false, false); event.u.mlme.status = MLME_DENIED; event.u.mlme.reason = status_code; drv_event_callback(sdata->local, sdata, &event); } else { if (!ieee80211_assoc_success(sdata, bss, mgmt, len)) { /* oops -- internal error -- send timeout for now */ - ieee80211_destroy_assoc_data(sdata, false); + ieee80211_destroy_assoc_data(sdata, false, false); cfg80211_assoc_timeout(sdata->dev, bss); return; } @@ -3187,7 +3190,7 @@ static void ieee80211_rx_mgmt_assoc_resp(struct ieee80211_sub_if_data *sdata, * recalc after assoc_data is NULL but before associated * is set can cause the interface to go idle */ - ieee80211_destroy_assoc_data(sdata, true); + ieee80211_destroy_assoc_data(sdata, true, false); /* get uapsd queues configuration */ uapsd_queues = 0; @@ -3886,7 +3889,7 @@ void ieee80211_sta_work(struct ieee80211_sub_if_data *sdata) .u.mlme.status = MLME_TIMEOUT, }; - ieee80211_destroy_assoc_data(sdata, false); + ieee80211_destroy_assoc_data(sdata, false, false); cfg80211_assoc_timeout(sdata->dev, bss); drv_event_callback(sdata->local, sdata, &event); } @@ -4025,7 +4028,7 @@ void ieee80211_mgd_quiesce(struct ieee80211_sub_if_data *sdata) WLAN_REASON_DEAUTH_LEAVING, false, frame_buf); if (ifmgd->assoc_data) - ieee80211_destroy_assoc_data(sdata, false); + ieee80211_destroy_assoc_data(sdata, false, true); if (ifmgd->auth_data) ieee80211_destroy_auth_data(sdata, false); cfg80211_tx_mlme_mgmt(sdata->dev, frame_buf, @@ -4907,7 +4910,7 @@ int ieee80211_mgd_deauth(struct ieee80211_sub_if_data *sdata, IEEE80211_STYPE_DEAUTH, req->reason_code, tx, frame_buf); - ieee80211_destroy_assoc_data(sdata, false); + ieee80211_destroy_assoc_data(sdata, false, true); ieee80211_report_disconnect(sdata, frame_buf, sizeof(frame_buf), true, req->reason_code); @@ -4982,7 +4985,7 @@ void ieee80211_mgd_stop(struct ieee80211_sub_if_data *sdata) sdata_lock(sdata); if (ifmgd->assoc_data) { struct cfg80211_bss *bss = ifmgd->assoc_data->bss; - ieee80211_destroy_assoc_data(sdata, false); + ieee80211_destroy_assoc_data(sdata, false, false); cfg80211_assoc_timeout(sdata->dev, bss); } if (ifmgd->auth_data) diff --git a/net/mac80211/rx.c b/net/mac80211/rx.c index a47bbc973f2dbc..2384b4aae0648f 100644 --- a/net/mac80211/rx.c +++ b/net/mac80211/rx.c @@ -3939,21 +3939,31 @@ static bool ieee80211_invoke_fast_rx(struct ieee80211_rx_data *rx, u64_stats_update_end(&stats->syncp); if (fast_rx->internal_forward) { - struct sta_info *dsta = sta_info_get(rx->sdata, skb->data); + struct sk_buff *xmit_skb = NULL; + bool multicast = is_multicast_ether_addr(skb->data); - if (dsta) { + if (multicast) { + xmit_skb = skb_copy(skb, GFP_ATOMIC); + } else if (sta_info_get(rx->sdata, skb->data)) { + xmit_skb = skb; + skb = NULL; + } + + if (xmit_skb) { /* * Send to wireless media and increase priority by 256 * to keep the received priority instead of * reclassifying the frame (see cfg80211_classify8021d). */ - skb->priority += 256; - skb->protocol = htons(ETH_P_802_3); - skb_reset_network_header(skb); - skb_reset_mac_header(skb); - dev_queue_xmit(skb); - return true; + xmit_skb->priority += 256; + xmit_skb->protocol = htons(ETH_P_802_3); + skb_reset_network_header(xmit_skb); + skb_reset_mac_header(xmit_skb); + dev_queue_xmit(xmit_skb); } + + if (!skb) + return true; } /* deliver to local stack */ diff --git a/net/mac80211/sta_info.h b/net/mac80211/sta_info.h index ed5fcb984a01a5..dd06ef0b886145 100644 --- a/net/mac80211/sta_info.h +++ b/net/mac80211/sta_info.h @@ -184,7 +184,6 @@ struct tid_ampdu_tx { * @ssn: Starting Sequence Number expected to be aggregated. * @buf_size: buffer size for incoming A-MPDUs * @timeout: reset timer value (in TUs). - * @dialog_token: dialog token for aggregation session * @rcu_head: RCU head used for freeing this struct * @reorder_lock: serializes access to reorder buffer, see below. * @auto_seq: used for offloaded BA sessions to automatically pick head_seq_and @@ -213,7 +212,6 @@ struct tid_ampdu_rx { u16 ssn; u16 buf_size; u16 timeout; - u8 dialog_token; bool auto_seq; bool removed; }; @@ -225,6 +223,7 @@ struct tid_ampdu_rx { * to tid_tx[idx], which are protected by the sta spinlock) * tid_start_tx is also protected by sta->lock. * @tid_rx: aggregation info for Rx per TID -- RCU protected + * @tid_rx_token: dialog tokens for valid aggregation sessions * @tid_rx_timer_expired: bitmap indicating on which TIDs the * RX timer expired until the work for it runs * @tid_rx_stop_requested: bitmap indicating which BA sessions per TID the @@ -243,6 +242,7 @@ struct sta_ampdu_mlme { struct mutex mtx; /* rx */ struct tid_ampdu_rx __rcu *tid_rx[IEEE80211_NUM_TIDS]; + u8 tid_rx_token[IEEE80211_NUM_TIDS]; unsigned long tid_rx_timer_expired[BITS_TO_LONGS(IEEE80211_NUM_TIDS)]; unsigned long tid_rx_stop_requested[BITS_TO_LONGS(IEEE80211_NUM_TIDS)]; unsigned long agg_session_valid[BITS_TO_LONGS(IEEE80211_NUM_TIDS)]; diff --git a/net/mac80211/tx.c b/net/mac80211/tx.c index bd5f4be89435eb..dd190ff3daea27 100644 --- a/net/mac80211/tx.c +++ b/net/mac80211/tx.c @@ -3262,7 +3262,7 @@ static bool ieee80211_xmit_fast(struct ieee80211_sub_if_data *sdata, int extra_head = fast_tx->hdr_len - (ETH_HLEN - 2); int hw_headroom = sdata->local->hw.extra_tx_headroom; struct ethhdr eth; - struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); + struct ieee80211_tx_info *info; struct ieee80211_hdr *hdr = (void *)fast_tx->hdr; struct ieee80211_tx_data tx; ieee80211_tx_result r; @@ -3326,6 +3326,7 @@ static bool ieee80211_xmit_fast(struct ieee80211_sub_if_data *sdata, memcpy(skb->data + fast_tx->da_offs, eth.h_dest, ETH_ALEN); memcpy(skb->data + fast_tx->sa_offs, eth.h_source, ETH_ALEN); + info = IEEE80211_SKB_CB(skb); memset(info, 0, sizeof(*info)); info->band = fast_tx->band; info->control.vif = &sdata->vif; diff --git a/net/mpls/af_mpls.c b/net/mpls/af_mpls.c index 15fe97644ffe04..5b77377e5a1547 100644 --- a/net/mpls/af_mpls.c +++ b/net/mpls/af_mpls.c @@ -98,18 +98,19 @@ bool mpls_pkt_too_big(const struct sk_buff *skb, unsigned int mtu) } EXPORT_SYMBOL_GPL(mpls_pkt_too_big); -static u32 mpls_multipath_hash(struct mpls_route *rt, - struct sk_buff *skb, bool bos) +static u32 mpls_multipath_hash(struct mpls_route *rt, struct sk_buff *skb) { struct mpls_entry_decoded dec; + unsigned int mpls_hdr_len = 0; struct mpls_shim_hdr *hdr; bool eli_seen = false; int label_index; u32 hash = 0; - for (label_index = 0; label_index < MAX_MP_SELECT_LABELS && !bos; + for (label_index = 0; label_index < MAX_MP_SELECT_LABELS; label_index++) { - if (!pskb_may_pull(skb, sizeof(*hdr) * label_index)) + mpls_hdr_len += sizeof(*hdr); + if (!pskb_may_pull(skb, mpls_hdr_len)) break; /* Read and decode the current label */ @@ -134,37 +135,38 @@ static u32 mpls_multipath_hash(struct mpls_route *rt, eli_seen = true; } - bos = dec.bos; - if (bos && pskb_may_pull(skb, sizeof(*hdr) * label_index + - sizeof(struct iphdr))) { + if (!dec.bos) + continue; + + /* found bottom label; does skb have room for a header? */ + if (pskb_may_pull(skb, mpls_hdr_len + sizeof(struct iphdr))) { const struct iphdr *v4hdr; - v4hdr = (const struct iphdr *)(mpls_hdr(skb) + - label_index); + v4hdr = (const struct iphdr *)(hdr + 1); if (v4hdr->version == 4) { hash = jhash_3words(ntohl(v4hdr->saddr), ntohl(v4hdr->daddr), v4hdr->protocol, hash); } else if (v4hdr->version == 6 && - pskb_may_pull(skb, sizeof(*hdr) * label_index + - sizeof(struct ipv6hdr))) { + pskb_may_pull(skb, mpls_hdr_len + + sizeof(struct ipv6hdr))) { const struct ipv6hdr *v6hdr; - v6hdr = (const struct ipv6hdr *)(mpls_hdr(skb) + - label_index); - + v6hdr = (const struct ipv6hdr *)(hdr + 1); hash = __ipv6_addr_jhash(&v6hdr->saddr, hash); hash = __ipv6_addr_jhash(&v6hdr->daddr, hash); hash = jhash_1word(v6hdr->nexthdr, hash); } } + + break; } return hash; } static struct mpls_nh *mpls_select_multipath(struct mpls_route *rt, - struct sk_buff *skb, bool bos) + struct sk_buff *skb) { int alive = ACCESS_ONCE(rt->rt_nhn_alive); u32 hash = 0; @@ -180,7 +182,7 @@ static struct mpls_nh *mpls_select_multipath(struct mpls_route *rt, if (alive <= 0) return NULL; - hash = mpls_multipath_hash(rt, skb, bos); + hash = mpls_multipath_hash(rt, skb); nh_index = hash % alive; if (alive == rt->rt_nhn) goto out; @@ -278,17 +280,11 @@ static int mpls_forward(struct sk_buff *skb, struct net_device *dev, hdr = mpls_hdr(skb); dec = mpls_entry_decode(hdr); - /* Pop the label */ - skb_pull(skb, sizeof(*hdr)); - skb_reset_network_header(skb); - - skb_orphan(skb); - rt = mpls_route_input_rcu(net, dec.label); if (!rt) goto drop; - nh = mpls_select_multipath(rt, skb, dec.bos); + nh = mpls_select_multipath(rt, skb); if (!nh) goto drop; @@ -297,6 +293,12 @@ static int mpls_forward(struct sk_buff *skb, struct net_device *dev, if (!mpls_output_possible(out_dev)) goto drop; + /* Pop the label */ + skb_pull(skb, sizeof(*hdr)); + skb_reset_network_header(skb); + + skb_orphan(skb); + if (skb_warn_if_lro(skb)) goto drop; diff --git a/net/mpls/mpls_iptunnel.c b/net/mpls/mpls_iptunnel.c index cf52cf30ac4b50..bc9aaf58c7ccfd 100644 --- a/net/mpls/mpls_iptunnel.c +++ b/net/mpls/mpls_iptunnel.c @@ -218,6 +218,7 @@ static const struct lwtunnel_encap_ops mpls_iptun_ops = { .fill_encap = mpls_fill_encap_info, .get_encap_size = mpls_encap_nlsize, .cmp_encap = mpls_encap_cmp, + .owner = THIS_MODULE, }; static int __init mpls_iptunnel_init(void) diff --git a/net/netfilter/Kconfig b/net/netfilter/Kconfig index e8d56d9a4df2cc..177d3ae5fda88e 100644 --- a/net/netfilter/Kconfig +++ b/net/netfilter/Kconfig @@ -1316,6 +1316,8 @@ config NETFILTER_XT_MATCH_OWNER based on who created the socket: the user or group. It is also possible to check whether a socket actually exists. + Conflicts with '"quota, tag, uid" match' + config NETFILTER_XT_MATCH_POLICY tristate 'IPsec "policy" match support' depends on XFRM @@ -1349,6 +1351,22 @@ config NETFILTER_XT_MATCH_PKTTYPE To compile it as a module, choose M here. If unsure, say N. +config NETFILTER_XT_MATCH_QTAGUID + bool '"quota, tag, owner" match and stats support' + depends on NETFILTER_XT_MATCH_SOCKET + depends on NETFILTER_XT_MATCH_OWNER=n + help + This option replaces the `owner' match. In addition to matching + on uid, it keeps stats based on a tag assigned to a socket. + The full tag is comprised of a UID and an accounting tag. + The tags are assignable to sockets from user space (e.g. a download + manager can assign the socket to another UID for accounting). + Stats and control are done via /proc/net/xt_qtaguid/. + It replaces owner as it takes the same arguments, but should + really be recognized by the iptables tool. + + If unsure, say `N'. + config NETFILTER_XT_MATCH_QUOTA tristate '"quota" match support' depends on NETFILTER_ADVANCED @@ -1359,6 +1377,29 @@ config NETFILTER_XT_MATCH_QUOTA If you want to compile it as a module, say M here and read . If unsure, say `N'. +config NETFILTER_XT_MATCH_QUOTA2 + tristate '"quota2" match support' + depends on NETFILTER_ADVANCED + help + This option adds a `quota2' match, which allows to match on a + byte counter correctly and not per CPU. + It allows naming the quotas. + This is based on http://xtables-addons.git.sourceforge.net + + If you want to compile it as a module, say M here and read + . If unsure, say `N'. + +config NETFILTER_XT_MATCH_QUOTA2_LOG + bool '"quota2" Netfilter LOG support' + depends on NETFILTER_XT_MATCH_QUOTA2 + default n + help + This option allows `quota2' to log ONCE when a quota limit + is passed. It logs via NETLINK using the NETLINK_NFLOG family. + It logs similarly to how ipt_ULOG would without data. + + If unsure, say `N'. + config NETFILTER_XT_MATCH_RATEEST tristate '"rateest" match support' depends on NETFILTER_ADVANCED diff --git a/net/netfilter/Makefile b/net/netfilter/Makefile index c23c3c84416f7e..54ba5aa1f9bf9e 100644 --- a/net/netfilter/Makefile +++ b/net/netfilter/Makefile @@ -167,7 +167,9 @@ obj-$(CONFIG_NETFILTER_XT_MATCH_CGROUP) += xt_cgroup.o obj-$(CONFIG_NETFILTER_XT_MATCH_PHYSDEV) += xt_physdev.o obj-$(CONFIG_NETFILTER_XT_MATCH_PKTTYPE) += xt_pkttype.o obj-$(CONFIG_NETFILTER_XT_MATCH_POLICY) += xt_policy.o +obj-$(CONFIG_NETFILTER_XT_MATCH_QTAGUID) += xt_qtaguid_print.o xt_qtaguid.o obj-$(CONFIG_NETFILTER_XT_MATCH_QUOTA) += xt_quota.o +obj-$(CONFIG_NETFILTER_XT_MATCH_QUOTA2) += xt_quota2.o obj-$(CONFIG_NETFILTER_XT_MATCH_RATEEST) += xt_rateest.o obj-$(CONFIG_NETFILTER_XT_MATCH_REALM) += xt_realm.o obj-$(CONFIG_NETFILTER_XT_MATCH_RECENT) += xt_recent.o diff --git a/net/netfilter/xt_IDLETIMER.c b/net/netfilter/xt_IDLETIMER.c index daf45da448fab4..4a2d853a90b2a9 100644 --- a/net/netfilter/xt_IDLETIMER.c +++ b/net/netfilter/xt_IDLETIMER.c @@ -5,6 +5,7 @@ * After timer expires a kevent will be sent. * * Copyright (C) 2004, 2010 Nokia Corporation + * * Written by Timo Teras * * Converted to x_tables and reworked for upstream inclusion @@ -38,8 +39,16 @@ #include #include #include +#include #include #include +#include +#include +#include +#include +#include +#include +#include struct idletimer_tg_attr { struct attribute attr; @@ -55,14 +64,110 @@ struct idletimer_tg { struct kobject *kobj; struct idletimer_tg_attr attr; + struct timespec delayed_timer_trigger; + struct timespec last_modified_timer; + struct timespec last_suspend_time; + struct notifier_block pm_nb; + + int timeout; unsigned int refcnt; + bool work_pending; + bool send_nl_msg; + bool active; + uid_t uid; }; static LIST_HEAD(idletimer_tg_list); static DEFINE_MUTEX(list_mutex); +static DEFINE_SPINLOCK(timestamp_lock); static struct kobject *idletimer_tg_kobj; +static bool check_for_delayed_trigger(struct idletimer_tg *timer, + struct timespec *ts) +{ + bool state; + struct timespec temp; + spin_lock_bh(×tamp_lock); + timer->work_pending = false; + if ((ts->tv_sec - timer->last_modified_timer.tv_sec) > timer->timeout || + timer->delayed_timer_trigger.tv_sec != 0) { + state = false; + temp.tv_sec = timer->timeout; + temp.tv_nsec = 0; + if (timer->delayed_timer_trigger.tv_sec != 0) { + temp = timespec_add(timer->delayed_timer_trigger, temp); + ts->tv_sec = temp.tv_sec; + ts->tv_nsec = temp.tv_nsec; + timer->delayed_timer_trigger.tv_sec = 0; + timer->work_pending = true; + schedule_work(&timer->work); + } else { + temp = timespec_add(timer->last_modified_timer, temp); + ts->tv_sec = temp.tv_sec; + ts->tv_nsec = temp.tv_nsec; + } + } else { + state = timer->active; + } + spin_unlock_bh(×tamp_lock); + return state; +} + +static void notify_netlink_uevent(const char *iface, struct idletimer_tg *timer) +{ + char iface_msg[NLMSG_MAX_SIZE]; + char state_msg[NLMSG_MAX_SIZE]; + char timestamp_msg[NLMSG_MAX_SIZE]; + char uid_msg[NLMSG_MAX_SIZE]; + char *envp[] = { iface_msg, state_msg, timestamp_msg, uid_msg, NULL }; + int res; + struct timespec ts; + uint64_t time_ns; + bool state; + + res = snprintf(iface_msg, NLMSG_MAX_SIZE, "INTERFACE=%s", + iface); + if (NLMSG_MAX_SIZE <= res) { + pr_err("message too long (%d)", res); + return; + } + + get_monotonic_boottime(&ts); + state = check_for_delayed_trigger(timer, &ts); + res = snprintf(state_msg, NLMSG_MAX_SIZE, "STATE=%s", + state ? "active" : "inactive"); + + if (NLMSG_MAX_SIZE <= res) { + pr_err("message too long (%d)", res); + return; + } + + if (state) { + res = snprintf(uid_msg, NLMSG_MAX_SIZE, "UID=%u", timer->uid); + if (NLMSG_MAX_SIZE <= res) + pr_err("message too long (%d)", res); + } else { + res = snprintf(uid_msg, NLMSG_MAX_SIZE, "UID="); + if (NLMSG_MAX_SIZE <= res) + pr_err("message too long (%d)", res); + } + + time_ns = timespec_to_ns(&ts); + res = snprintf(timestamp_msg, NLMSG_MAX_SIZE, "TIME_NS=%llu", time_ns); + if (NLMSG_MAX_SIZE <= res) { + timestamp_msg[0] = '\0'; + pr_err("message too long (%d)", res); + } + + pr_debug("putting nlmsg: <%s> <%s> <%s> <%s>\n", iface_msg, state_msg, + timestamp_msg, uid_msg); + kobject_uevent_env(idletimer_tg_kobj, KOBJ_CHANGE, envp); + return; + + +} + static struct idletimer_tg *__idletimer_tg_find_by_label(const char *label) { @@ -83,6 +188,7 @@ static ssize_t idletimer_tg_show(struct kobject *kobj, struct attribute *attr, { struct idletimer_tg *timer; unsigned long expires = 0; + unsigned long now = jiffies; mutex_lock(&list_mutex); @@ -92,11 +198,15 @@ static ssize_t idletimer_tg_show(struct kobject *kobj, struct attribute *attr, mutex_unlock(&list_mutex); - if (time_after(expires, jiffies)) + if (time_after(expires, now)) return sprintf(buf, "%u\n", - jiffies_to_msecs(expires - jiffies) / 1000); + jiffies_to_msecs(expires - now) / 1000); - return sprintf(buf, "0\n"); + if (timer->send_nl_msg) + return sprintf(buf, "0 %d\n", + jiffies_to_msecs(now - expires) / 1000); + else + return sprintf(buf, "0\n"); } static void idletimer_tg_work(struct work_struct *work) @@ -105,6 +215,9 @@ static void idletimer_tg_work(struct work_struct *work) work); sysfs_notify(idletimer_tg_kobj, NULL, timer->attr.attr.name); + + if (timer->send_nl_msg) + notify_netlink_uevent(timer->attr.attr.name, timer); } static void idletimer_tg_expired(unsigned long data) @@ -112,8 +225,55 @@ static void idletimer_tg_expired(unsigned long data) struct idletimer_tg *timer = (struct idletimer_tg *) data; pr_debug("timer %s expired\n", timer->attr.attr.name); - + spin_lock_bh(×tamp_lock); + timer->active = false; + timer->work_pending = true; schedule_work(&timer->work); + spin_unlock_bh(×tamp_lock); +} + +static int idletimer_resume(struct notifier_block *notifier, + unsigned long pm_event, void *unused) +{ + struct timespec ts; + unsigned long time_diff, now = jiffies; + struct idletimer_tg *timer = container_of(notifier, + struct idletimer_tg, pm_nb); + if (!timer) + return NOTIFY_DONE; + switch (pm_event) { + case PM_SUSPEND_PREPARE: + get_monotonic_boottime(&timer->last_suspend_time); + break; + case PM_POST_SUSPEND: + spin_lock_bh(×tamp_lock); + if (!timer->active) { + spin_unlock_bh(×tamp_lock); + break; + } + /* since jiffies are not updated when suspended now represents + * the time it would have suspended */ + if (time_after(timer->timer.expires, now)) { + get_monotonic_boottime(&ts); + ts = timespec_sub(ts, timer->last_suspend_time); + time_diff = timespec_to_jiffies(&ts); + if (timer->timer.expires > (time_diff + now)) { + mod_timer_pending(&timer->timer, + (timer->timer.expires - time_diff)); + } else { + del_timer(&timer->timer); + timer->timer.expires = 0; + timer->active = false; + timer->work_pending = true; + schedule_work(&timer->work); + } + } + spin_unlock_bh(×tamp_lock); + break; + default: + break; + } + return NOTIFY_DONE; } static int idletimer_tg_create(struct idletimer_tg_info *info) @@ -146,6 +306,21 @@ static int idletimer_tg_create(struct idletimer_tg_info *info) setup_timer(&info->timer->timer, idletimer_tg_expired, (unsigned long) info->timer); info->timer->refcnt = 1; + info->timer->send_nl_msg = (info->send_nl_msg == 0) ? false : true; + info->timer->active = true; + info->timer->timeout = info->timeout; + + info->timer->delayed_timer_trigger.tv_sec = 0; + info->timer->delayed_timer_trigger.tv_nsec = 0; + info->timer->work_pending = false; + info->timer->uid = 0; + get_monotonic_boottime(&info->timer->last_modified_timer); + + info->timer->pm_nb.notifier_call = idletimer_resume; + ret = register_pm_notifier(&info->timer->pm_nb); + if (ret) + printk(KERN_WARNING "[%s] Failed to register pm notifier %d\n", + __func__, ret); mod_timer(&info->timer->timer, msecs_to_jiffies(info->timeout * 1000) + jiffies); @@ -162,6 +337,42 @@ static int idletimer_tg_create(struct idletimer_tg_info *info) return ret; } +static void reset_timer(const struct idletimer_tg_info *info, + struct sk_buff *skb) +{ + unsigned long now = jiffies; + struct idletimer_tg *timer = info->timer; + bool timer_prev; + + spin_lock_bh(×tamp_lock); + timer_prev = timer->active; + timer->active = true; + /* timer_prev is used to guard overflow problem in time_before*/ + if (!timer_prev || time_before(timer->timer.expires, now)) { + pr_debug("Starting Checkentry timer (Expired, Jiffies): %lu, %lu\n", + timer->timer.expires, now); + + /* Stores the uid resposible for waking up the radio */ + if (skb && (skb->sk)) { + timer->uid = from_kuid_munged(current_user_ns(), + sock_i_uid(skb->sk)); + } + + /* checks if there is a pending inactive notification*/ + if (timer->work_pending) + timer->delayed_timer_trigger = timer->last_modified_timer; + else { + timer->work_pending = true; + schedule_work(&timer->work); + } + } + + get_monotonic_boottime(&timer->last_modified_timer); + mod_timer(&timer->timer, + msecs_to_jiffies(info->timeout * 1000) + now); + spin_unlock_bh(×tamp_lock); +} + /* * The actual xt_tables plugin. */ @@ -169,15 +380,23 @@ static unsigned int idletimer_tg_target(struct sk_buff *skb, const struct xt_action_param *par) { const struct idletimer_tg_info *info = par->targinfo; + unsigned long now = jiffies; pr_debug("resetting timer %s, timeout period %u\n", info->label, info->timeout); BUG_ON(!info->timer); - mod_timer(&info->timer->timer, - msecs_to_jiffies(info->timeout * 1000) + jiffies); + info->timer->active = true; + if (time_before(info->timer->timer.expires, now)) { + schedule_work(&info->timer->work); + pr_debug("Starting timer %s (Expired, Jiffies): %lu, %lu\n", + info->label, info->timer->timer.expires, now); + } + + /* TODO: Avoid modifying timers on each packet */ + reset_timer(info, skb); return XT_CONTINUE; } @@ -186,7 +405,7 @@ static int idletimer_tg_checkentry(const struct xt_tgchk_param *par) struct idletimer_tg_info *info = par->targinfo; int ret; - pr_debug("checkentry targinfo%s\n", info->label); + pr_debug("checkentry targinfo %s\n", info->label); if (info->timeout == 0) { pr_debug("timeout value is zero\n"); @@ -205,9 +424,7 @@ static int idletimer_tg_checkentry(const struct xt_tgchk_param *par) info->timer = __idletimer_tg_find_by_label(info->label); if (info->timer) { info->timer->refcnt++; - mod_timer(&info->timer->timer, - msecs_to_jiffies(info->timeout * 1000) + jiffies); - + reset_timer(info, NULL); pr_debug("increased refcnt of timer %s to %u\n", info->label, info->timer->refcnt); } else { @@ -220,6 +437,7 @@ static int idletimer_tg_checkentry(const struct xt_tgchk_param *par) } mutex_unlock(&list_mutex); + return 0; } @@ -238,11 +456,12 @@ static void idletimer_tg_destroy(const struct xt_tgdtor_param *par) del_timer_sync(&info->timer->timer); cancel_work_sync(&info->timer->work); sysfs_remove_file(idletimer_tg_kobj, &info->timer->attr.attr); + unregister_pm_notifier(&info->timer->pm_nb); kfree(info->timer->attr.attr.name); kfree(info->timer); } else { pr_debug("decreased refcnt of timer %s to %u\n", - info->label, info->timer->refcnt); + info->label, info->timer->refcnt); } mutex_unlock(&list_mutex); @@ -250,6 +469,7 @@ static void idletimer_tg_destroy(const struct xt_tgdtor_param *par) static struct xt_target idletimer_tg __read_mostly = { .name = "IDLETIMER", + .revision = 1, .family = NFPROTO_UNSPEC, .target = idletimer_tg_target, .targetsize = sizeof(struct idletimer_tg_info), @@ -315,3 +535,4 @@ MODULE_DESCRIPTION("Xtables: idle time monitor"); MODULE_LICENSE("GPL v2"); MODULE_ALIAS("ipt_IDLETIMER"); MODULE_ALIAS("ip6t_IDLETIMER"); +MODULE_ALIAS("arpt_IDLETIMER"); diff --git a/net/netfilter/xt_qtaguid.c b/net/netfilter/xt_qtaguid.c new file mode 100644 index 00000000000000..3bf0c59dab2f54 --- /dev/null +++ b/net/netfilter/xt_qtaguid.c @@ -0,0 +1,3032 @@ +/* + * Kernel iptables module to track stats for packets based on user tags. + * + * (C) 2011 Google, Inc + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +/* + * There are run-time debug flags enabled via the debug_mask module param, or + * via the DEFAULT_DEBUG_MASK. See xt_qtaguid_internal.h. + */ +#define DEBUG + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(CONFIG_IP6_NF_IPTABLES) || defined(CONFIG_IP6_NF_IPTABLES_MODULE) +#include +#endif + +#include +#include "xt_qtaguid_internal.h" +#include "xt_qtaguid_print.h" +#include "../../fs/proc/internal.h" + +/* + * We only use the xt_socket funcs within a similar context to avoid unexpected + * return values. + */ +#define XT_SOCKET_SUPPORTED_HOOKS \ + ((1 << NF_INET_PRE_ROUTING) | (1 << NF_INET_LOCAL_IN)) + + +static const char *module_procdirname = "xt_qtaguid"; +static struct proc_dir_entry *xt_qtaguid_procdir; + +static unsigned int proc_iface_perms = S_IRUGO; +module_param_named(iface_perms, proc_iface_perms, uint, S_IRUGO | S_IWUSR); + +static struct proc_dir_entry *xt_qtaguid_stats_file; +static unsigned int proc_stats_perms = S_IRUGO; +module_param_named(stats_perms, proc_stats_perms, uint, S_IRUGO | S_IWUSR); + +static struct proc_dir_entry *xt_qtaguid_ctrl_file; + +/* Everybody can write. But proc_ctrl_write_limited is true by default which + * limits what can be controlled. See the can_*() functions. + */ +static unsigned int proc_ctrl_perms = S_IRUGO | S_IWUGO; +module_param_named(ctrl_perms, proc_ctrl_perms, uint, S_IRUGO | S_IWUSR); + +/* Limited by default, so the gid of the ctrl and stats proc entries + * will limit what can be done. See the can_*() functions. + */ +static bool proc_stats_readall_limited = true; +static bool proc_ctrl_write_limited = true; + +module_param_named(stats_readall_limited, proc_stats_readall_limited, bool, + S_IRUGO | S_IWUSR); +module_param_named(ctrl_write_limited, proc_ctrl_write_limited, bool, + S_IRUGO | S_IWUSR); + +/* + * Limit the number of active tags (via socket tags) for a given UID. + * Multiple processes could share the UID. + */ +static int max_sock_tags = DEFAULT_MAX_SOCK_TAGS; +module_param(max_sock_tags, int, S_IRUGO | S_IWUSR); + +/* + * After the kernel has initiallized this module, it is still possible + * to make it passive. + * Setting passive to Y: + * - the iface stats handling will not act on notifications. + * - iptables matches will never match. + * - ctrl commands silently succeed. + * - stats are always empty. + * This is mostly usefull when a bug is suspected. + */ +static bool module_passive; +module_param_named(passive, module_passive, bool, S_IRUGO | S_IWUSR); + +/* + * Control how qtaguid data is tracked per proc/uid. + * Setting tag_tracking_passive to Y: + * - don't create proc specific structs to track tags + * - don't check that active tag stats exceed some limits. + * - don't clean up socket tags on process exits. + * This is mostly usefull when a bug is suspected. + */ +static bool qtu_proc_handling_passive; +module_param_named(tag_tracking_passive, qtu_proc_handling_passive, bool, + S_IRUGO | S_IWUSR); + +#define QTU_DEV_NAME "xt_qtaguid" + +uint qtaguid_debug_mask = DEFAULT_DEBUG_MASK; +module_param_named(debug_mask, qtaguid_debug_mask, uint, S_IRUGO | S_IWUSR); + +/*---------------------------------------------------------------------------*/ +static const char *iface_stat_procdirname = "iface_stat"; +static struct proc_dir_entry *iface_stat_procdir; +/* + * The iface_stat_all* will go away once userspace gets use to the new fields + * that have a format line. + */ +static const char *iface_stat_all_procfilename = "iface_stat_all"; +static struct proc_dir_entry *iface_stat_all_procfile; +static const char *iface_stat_fmt_procfilename = "iface_stat_fmt"; +static struct proc_dir_entry *iface_stat_fmt_procfile; + + +static LIST_HEAD(iface_stat_list); +static DEFINE_SPINLOCK(iface_stat_list_lock); + +static struct rb_root sock_tag_tree = RB_ROOT; +static DEFINE_SPINLOCK(sock_tag_list_lock); + +static struct rb_root tag_counter_set_tree = RB_ROOT; +static DEFINE_SPINLOCK(tag_counter_set_list_lock); + +static struct rb_root uid_tag_data_tree = RB_ROOT; +static DEFINE_SPINLOCK(uid_tag_data_tree_lock); + +static struct rb_root proc_qtu_data_tree = RB_ROOT; +/* No proc_qtu_data_tree_lock; use uid_tag_data_tree_lock */ + +static struct qtaguid_event_counts qtu_events; +/*----------------------------------------------*/ +static bool can_manipulate_uids(void) +{ + /* root pwnd */ + return in_egroup_p(xt_qtaguid_ctrl_file->gid) + || unlikely(!from_kuid(&init_user_ns, current_fsuid())) || unlikely(!proc_ctrl_write_limited) + || unlikely(uid_eq(current_fsuid(), xt_qtaguid_ctrl_file->uid)); +} + +static bool can_impersonate_uid(kuid_t uid) +{ + return uid_eq(uid, current_fsuid()) || can_manipulate_uids(); +} + +static bool can_read_other_uid_stats(kuid_t uid) +{ + /* root pwnd */ + return in_egroup_p(xt_qtaguid_stats_file->gid) + || unlikely(!from_kuid(&init_user_ns, current_fsuid())) || uid_eq(uid, current_fsuid()) + || unlikely(!proc_stats_readall_limited) + || unlikely(uid_eq(current_fsuid(), xt_qtaguid_ctrl_file->uid)); +} + +static inline void dc_add_byte_packets(struct data_counters *counters, int set, + enum ifs_tx_rx direction, + enum ifs_proto ifs_proto, + int bytes, + int packets) +{ + counters->bpc[set][direction][ifs_proto].bytes += bytes; + counters->bpc[set][direction][ifs_proto].packets += packets; +} + +static struct tag_node *tag_node_tree_search(struct rb_root *root, tag_t tag) +{ + struct rb_node *node = root->rb_node; + + while (node) { + struct tag_node *data = rb_entry(node, struct tag_node, node); + int result; + RB_DEBUG("qtaguid: tag_node_tree_search(0x%llx): " + " node=%p data=%p\n", tag, node, data); + result = tag_compare(tag, data->tag); + RB_DEBUG("qtaguid: tag_node_tree_search(0x%llx): " + " data.tag=0x%llx (uid=%u) res=%d\n", + tag, data->tag, get_uid_from_tag(data->tag), result); + if (result < 0) + node = node->rb_left; + else if (result > 0) + node = node->rb_right; + else + return data; + } + return NULL; +} + +static void tag_node_tree_insert(struct tag_node *data, struct rb_root *root) +{ + struct rb_node **new = &(root->rb_node), *parent = NULL; + + /* Figure out where to put new node */ + while (*new) { + struct tag_node *this = rb_entry(*new, struct tag_node, + node); + int result = tag_compare(data->tag, this->tag); + RB_DEBUG("qtaguid: %s(): tag=0x%llx" + " (uid=%u)\n", __func__, + this->tag, + get_uid_from_tag(this->tag)); + parent = *new; + if (result < 0) + new = &((*new)->rb_left); + else if (result > 0) + new = &((*new)->rb_right); + else + BUG(); + } + + /* Add new node and rebalance tree. */ + rb_link_node(&data->node, parent, new); + rb_insert_color(&data->node, root); +} + +static void tag_stat_tree_insert(struct tag_stat *data, struct rb_root *root) +{ + tag_node_tree_insert(&data->tn, root); +} + +static struct tag_stat *tag_stat_tree_search(struct rb_root *root, tag_t tag) +{ + struct tag_node *node = tag_node_tree_search(root, tag); + if (!node) + return NULL; + return rb_entry(&node->node, struct tag_stat, tn.node); +} + +static void tag_counter_set_tree_insert(struct tag_counter_set *data, + struct rb_root *root) +{ + tag_node_tree_insert(&data->tn, root); +} + +static struct tag_counter_set *tag_counter_set_tree_search(struct rb_root *root, + tag_t tag) +{ + struct tag_node *node = tag_node_tree_search(root, tag); + if (!node) + return NULL; + return rb_entry(&node->node, struct tag_counter_set, tn.node); + +} + +static void tag_ref_tree_insert(struct tag_ref *data, struct rb_root *root) +{ + tag_node_tree_insert(&data->tn, root); +} + +static struct tag_ref *tag_ref_tree_search(struct rb_root *root, tag_t tag) +{ + struct tag_node *node = tag_node_tree_search(root, tag); + if (!node) + return NULL; + return rb_entry(&node->node, struct tag_ref, tn.node); +} + +static struct sock_tag *sock_tag_tree_search(struct rb_root *root, + const struct sock *sk) +{ + struct rb_node *node = root->rb_node; + + while (node) { + struct sock_tag *data = rb_entry(node, struct sock_tag, + sock_node); + if (sk < data->sk) + node = node->rb_left; + else if (sk > data->sk) + node = node->rb_right; + else + return data; + } + return NULL; +} + +static void sock_tag_tree_insert(struct sock_tag *data, struct rb_root *root) +{ + struct rb_node **new = &(root->rb_node), *parent = NULL; + + /* Figure out where to put new node */ + while (*new) { + struct sock_tag *this = rb_entry(*new, struct sock_tag, + sock_node); + parent = *new; + if (data->sk < this->sk) + new = &((*new)->rb_left); + else if (data->sk > this->sk) + new = &((*new)->rb_right); + else + BUG(); + } + + /* Add new node and rebalance tree. */ + rb_link_node(&data->sock_node, parent, new); + rb_insert_color(&data->sock_node, root); +} + +static void sock_tag_tree_erase(struct rb_root *st_to_free_tree) +{ + struct rb_node *node; + struct sock_tag *st_entry; + + node = rb_first(st_to_free_tree); + while (node) { + st_entry = rb_entry(node, struct sock_tag, sock_node); + node = rb_next(node); + CT_DEBUG("qtaguid: %s(): " + "erase st: sk=%p tag=0x%llx (uid=%u)\n", __func__, + st_entry->sk, + st_entry->tag, + get_uid_from_tag(st_entry->tag)); + rb_erase(&st_entry->sock_node, st_to_free_tree); + sockfd_put(st_entry->socket); + kfree(st_entry); + } +} + +static struct proc_qtu_data *proc_qtu_data_tree_search(struct rb_root *root, + const pid_t pid) +{ + struct rb_node *node = root->rb_node; + + while (node) { + struct proc_qtu_data *data = rb_entry(node, + struct proc_qtu_data, + node); + if (pid < data->pid) + node = node->rb_left; + else if (pid > data->pid) + node = node->rb_right; + else + return data; + } + return NULL; +} + +static void proc_qtu_data_tree_insert(struct proc_qtu_data *data, + struct rb_root *root) +{ + struct rb_node **new = &(root->rb_node), *parent = NULL; + + /* Figure out where to put new node */ + while (*new) { + struct proc_qtu_data *this = rb_entry(*new, + struct proc_qtu_data, + node); + parent = *new; + if (data->pid < this->pid) + new = &((*new)->rb_left); + else if (data->pid > this->pid) + new = &((*new)->rb_right); + else + BUG(); + } + + /* Add new node and rebalance tree. */ + rb_link_node(&data->node, parent, new); + rb_insert_color(&data->node, root); +} + +static void uid_tag_data_tree_insert(struct uid_tag_data *data, + struct rb_root *root) +{ + struct rb_node **new = &(root->rb_node), *parent = NULL; + + /* Figure out where to put new node */ + while (*new) { + struct uid_tag_data *this = rb_entry(*new, + struct uid_tag_data, + node); + parent = *new; + if (data->uid < this->uid) + new = &((*new)->rb_left); + else if (data->uid > this->uid) + new = &((*new)->rb_right); + else + BUG(); + } + + /* Add new node and rebalance tree. */ + rb_link_node(&data->node, parent, new); + rb_insert_color(&data->node, root); +} + +static struct uid_tag_data *uid_tag_data_tree_search(struct rb_root *root, + uid_t uid) +{ + struct rb_node *node = root->rb_node; + + while (node) { + struct uid_tag_data *data = rb_entry(node, + struct uid_tag_data, + node); + if (uid < data->uid) + node = node->rb_left; + else if (uid > data->uid) + node = node->rb_right; + else + return data; + } + return NULL; +} + +/* + * Allocates a new uid_tag_data struct if needed. + * Returns a pointer to the found or allocated uid_tag_data. + * Returns a PTR_ERR on failures, and lock is not held. + * If found is not NULL: + * sets *found to true if not allocated. + * sets *found to false if allocated. + */ +struct uid_tag_data *get_uid_data(uid_t uid, bool *found_res) +{ + struct uid_tag_data *utd_entry; + + /* Look for top level uid_tag_data for the UID */ + utd_entry = uid_tag_data_tree_search(&uid_tag_data_tree, uid); + DR_DEBUG("qtaguid: get_uid_data(%u) utd=%p\n", uid, utd_entry); + + if (found_res) + *found_res = utd_entry; + if (utd_entry) + return utd_entry; + + utd_entry = kzalloc(sizeof(*utd_entry), GFP_ATOMIC); + if (!utd_entry) { + pr_err("qtaguid: get_uid_data(%u): " + "tag data alloc failed\n", uid); + return ERR_PTR(-ENOMEM); + } + + utd_entry->uid = uid; + utd_entry->tag_ref_tree = RB_ROOT; + uid_tag_data_tree_insert(utd_entry, &uid_tag_data_tree); + DR_DEBUG("qtaguid: get_uid_data(%u) new utd=%p\n", uid, utd_entry); + return utd_entry; +} + +/* Never returns NULL. Either PTR_ERR or a valid ptr. */ +static struct tag_ref *new_tag_ref(tag_t new_tag, + struct uid_tag_data *utd_entry) +{ + struct tag_ref *tr_entry; + int res; + + if (utd_entry->num_active_tags + 1 > max_sock_tags) { + pr_info("qtaguid: new_tag_ref(0x%llx): " + "tag ref alloc quota exceeded. max=%d\n", + new_tag, max_sock_tags); + res = -EMFILE; + goto err_res; + + } + + tr_entry = kzalloc(sizeof(*tr_entry), GFP_ATOMIC); + if (!tr_entry) { + pr_err("qtaguid: new_tag_ref(0x%llx): " + "tag ref alloc failed\n", + new_tag); + res = -ENOMEM; + goto err_res; + } + tr_entry->tn.tag = new_tag; + /* tr_entry->num_sock_tags handled by caller */ + utd_entry->num_active_tags++; + tag_ref_tree_insert(tr_entry, &utd_entry->tag_ref_tree); + DR_DEBUG("qtaguid: new_tag_ref(0x%llx): " + " inserted new tag ref %p\n", + new_tag, tr_entry); + return tr_entry; + +err_res: + return ERR_PTR(res); +} + +static struct tag_ref *lookup_tag_ref(tag_t full_tag, + struct uid_tag_data **utd_res) +{ + struct uid_tag_data *utd_entry; + struct tag_ref *tr_entry; + bool found_utd; + uid_t uid = get_uid_from_tag(full_tag); + + DR_DEBUG("qtaguid: lookup_tag_ref(tag=0x%llx (uid=%u))\n", + full_tag, uid); + + utd_entry = get_uid_data(uid, &found_utd); + if (IS_ERR_OR_NULL(utd_entry)) { + if (utd_res) + *utd_res = utd_entry; + return NULL; + } + + tr_entry = tag_ref_tree_search(&utd_entry->tag_ref_tree, full_tag); + if (utd_res) + *utd_res = utd_entry; + DR_DEBUG("qtaguid: lookup_tag_ref(0x%llx) utd_entry=%p tr_entry=%p\n", + full_tag, utd_entry, tr_entry); + return tr_entry; +} + +/* Never returns NULL. Either PTR_ERR or a valid ptr. */ +static struct tag_ref *get_tag_ref(tag_t full_tag, + struct uid_tag_data **utd_res) +{ + struct uid_tag_data *utd_entry; + struct tag_ref *tr_entry; + + DR_DEBUG("qtaguid: get_tag_ref(0x%llx)\n", + full_tag); + spin_lock_bh(&uid_tag_data_tree_lock); + tr_entry = lookup_tag_ref(full_tag, &utd_entry); + BUG_ON(IS_ERR_OR_NULL(utd_entry)); + if (!tr_entry) + tr_entry = new_tag_ref(full_tag, utd_entry); + + spin_unlock_bh(&uid_tag_data_tree_lock); + if (utd_res) + *utd_res = utd_entry; + DR_DEBUG("qtaguid: get_tag_ref(0x%llx) utd=%p tr=%p\n", + full_tag, utd_entry, tr_entry); + return tr_entry; +} + +/* Checks and maybe frees the UID Tag Data entry */ +static void put_utd_entry(struct uid_tag_data *utd_entry) +{ + /* Are we done with the UID tag data entry? */ + if (RB_EMPTY_ROOT(&utd_entry->tag_ref_tree) && + !utd_entry->num_pqd) { + DR_DEBUG("qtaguid: %s(): " + "erase utd_entry=%p uid=%u " + "by pid=%u tgid=%u uid=%u\n", __func__, + utd_entry, utd_entry->uid, + current->pid, current->tgid, from_kuid(&init_user_ns, current_fsuid())); + BUG_ON(utd_entry->num_active_tags); + rb_erase(&utd_entry->node, &uid_tag_data_tree); + kfree(utd_entry); + } else { + DR_DEBUG("qtaguid: %s(): " + "utd_entry=%p still has %d tags %d proc_qtu_data\n", + __func__, utd_entry, utd_entry->num_active_tags, + utd_entry->num_pqd); + BUG_ON(!(utd_entry->num_active_tags || + utd_entry->num_pqd)); + } +} + +/* + * If no sock_tags are using this tag_ref, + * decrements refcount of utd_entry, removes tr_entry + * from utd_entry->tag_ref_tree and frees. + */ +static void free_tag_ref_from_utd_entry(struct tag_ref *tr_entry, + struct uid_tag_data *utd_entry) +{ + DR_DEBUG("qtaguid: %s(): %p tag=0x%llx (uid=%u)\n", __func__, + tr_entry, tr_entry->tn.tag, + get_uid_from_tag(tr_entry->tn.tag)); + if (!tr_entry->num_sock_tags) { + BUG_ON(!utd_entry->num_active_tags); + utd_entry->num_active_tags--; + rb_erase(&tr_entry->tn.node, &utd_entry->tag_ref_tree); + DR_DEBUG("qtaguid: %s(): erased %p\n", __func__, tr_entry); + kfree(tr_entry); + } +} + +static void put_tag_ref_tree(tag_t full_tag, struct uid_tag_data *utd_entry) +{ + struct rb_node *node; + struct tag_ref *tr_entry; + tag_t acct_tag; + + DR_DEBUG("qtaguid: %s(tag=0x%llx (uid=%u))\n", __func__, + full_tag, get_uid_from_tag(full_tag)); + acct_tag = get_atag_from_tag(full_tag); + node = rb_first(&utd_entry->tag_ref_tree); + while (node) { + tr_entry = rb_entry(node, struct tag_ref, tn.node); + node = rb_next(node); + if (!acct_tag || tr_entry->tn.tag == full_tag) + free_tag_ref_from_utd_entry(tr_entry, utd_entry); + } +} + +static ssize_t read_proc_u64(struct file *file, char __user *buf, + size_t size, loff_t *ppos) +{ + uint64_t *valuep = PDE_DATA(file_inode(file)); + char tmp[24]; + size_t tmp_size; + + tmp_size = scnprintf(tmp, sizeof(tmp), "%llu\n", *valuep); + return simple_read_from_buffer(buf, size, ppos, tmp, tmp_size); +} + +static ssize_t read_proc_bool(struct file *file, char __user *buf, + size_t size, loff_t *ppos) +{ + bool *valuep = PDE_DATA(file_inode(file)); + char tmp[24]; + size_t tmp_size; + + tmp_size = scnprintf(tmp, sizeof(tmp), "%u\n", *valuep); + return simple_read_from_buffer(buf, size, ppos, tmp, tmp_size); +} + +static int get_active_counter_set(tag_t tag) +{ + int active_set = 0; + struct tag_counter_set *tcs; + + MT_DEBUG("qtaguid: get_active_counter_set(tag=0x%llx)" + " (uid=%u)\n", + tag, get_uid_from_tag(tag)); + /* For now we only handle UID tags for active sets */ + tag = get_utag_from_tag(tag); + spin_lock_bh(&tag_counter_set_list_lock); + tcs = tag_counter_set_tree_search(&tag_counter_set_tree, tag); + if (tcs) + active_set = tcs->active_set; + spin_unlock_bh(&tag_counter_set_list_lock); + return active_set; +} + +/* + * Find the entry for tracking the specified interface. + * Caller must hold iface_stat_list_lock + */ +static struct iface_stat *get_iface_entry(const char *ifname) +{ + struct iface_stat *iface_entry; + + /* Find the entry for tracking the specified tag within the interface */ + if (ifname == NULL) { + pr_info("qtaguid: iface_stat: get() NULL device name\n"); + return NULL; + } + + /* Iterate over interfaces */ + list_for_each_entry(iface_entry, &iface_stat_list, list) { + if (!strcmp(ifname, iface_entry->ifname)) + goto done; + } + iface_entry = NULL; +done: + return iface_entry; +} + +/* This is for fmt2 only */ +static void pp_iface_stat_header(struct seq_file *m) +{ + seq_puts(m, + "ifname " + "total_skb_rx_bytes total_skb_rx_packets " + "total_skb_tx_bytes total_skb_tx_packets " + "rx_tcp_bytes rx_tcp_packets " + "rx_udp_bytes rx_udp_packets " + "rx_other_bytes rx_other_packets " + "tx_tcp_bytes tx_tcp_packets " + "tx_udp_bytes tx_udp_packets " + "tx_other_bytes tx_other_packets\n" + ); +} + +static void pp_iface_stat_line(struct seq_file *m, + struct iface_stat *iface_entry) +{ + struct data_counters *cnts; + int cnt_set = 0; /* We only use one set for the device */ + cnts = &iface_entry->totals_via_skb; + seq_printf(m, "%s %llu %llu %llu %llu %llu %llu %llu %llu " + "%llu %llu %llu %llu %llu %llu %llu %llu\n", + iface_entry->ifname, + dc_sum_bytes(cnts, cnt_set, IFS_RX), + dc_sum_packets(cnts, cnt_set, IFS_RX), + dc_sum_bytes(cnts, cnt_set, IFS_TX), + dc_sum_packets(cnts, cnt_set, IFS_TX), + cnts->bpc[cnt_set][IFS_RX][IFS_TCP].bytes, + cnts->bpc[cnt_set][IFS_RX][IFS_TCP].packets, + cnts->bpc[cnt_set][IFS_RX][IFS_UDP].bytes, + cnts->bpc[cnt_set][IFS_RX][IFS_UDP].packets, + cnts->bpc[cnt_set][IFS_RX][IFS_PROTO_OTHER].bytes, + cnts->bpc[cnt_set][IFS_RX][IFS_PROTO_OTHER].packets, + cnts->bpc[cnt_set][IFS_TX][IFS_TCP].bytes, + cnts->bpc[cnt_set][IFS_TX][IFS_TCP].packets, + cnts->bpc[cnt_set][IFS_TX][IFS_UDP].bytes, + cnts->bpc[cnt_set][IFS_TX][IFS_UDP].packets, + cnts->bpc[cnt_set][IFS_TX][IFS_PROTO_OTHER].bytes, + cnts->bpc[cnt_set][IFS_TX][IFS_PROTO_OTHER].packets); +} + +struct proc_iface_stat_fmt_info { + int fmt; +}; + +static void *iface_stat_fmt_proc_start(struct seq_file *m, loff_t *pos) +{ + struct proc_iface_stat_fmt_info *p = m->private; + loff_t n = *pos; + + /* + * This lock will prevent iface_stat_update() from changing active, + * and in turn prevent an interface from unregistering itself. + */ + spin_lock_bh(&iface_stat_list_lock); + + if (unlikely(module_passive)) + return NULL; + + if (!n && p->fmt == 2) + pp_iface_stat_header(m); + + return seq_list_start(&iface_stat_list, n); +} + +static void *iface_stat_fmt_proc_next(struct seq_file *m, void *p, loff_t *pos) +{ + return seq_list_next(p, &iface_stat_list, pos); +} + +static void iface_stat_fmt_proc_stop(struct seq_file *m, void *p) +{ + spin_unlock_bh(&iface_stat_list_lock); +} + +static int iface_stat_fmt_proc_show(struct seq_file *m, void *v) +{ + struct proc_iface_stat_fmt_info *p = m->private; + struct iface_stat *iface_entry; + struct rtnl_link_stats64 dev_stats, *stats; + struct rtnl_link_stats64 no_dev_stats = {0}; + + + CT_DEBUG("qtaguid:proc iface_stat_fmt pid=%u tgid=%u uid=%u\n", + current->pid, current->tgid, from_kuid(&init_user_ns, current_fsuid())); + + iface_entry = list_entry(v, struct iface_stat, list); + + if (iface_entry->active) { + stats = dev_get_stats(iface_entry->net_dev, + &dev_stats); + } else { + stats = &no_dev_stats; + } + /* + * If the meaning of the data changes, then update the fmtX + * string. + */ + if (p->fmt == 1) { + seq_printf(m, "%s %d %llu %llu %llu %llu %llu %llu %llu %llu\n", + iface_entry->ifname, + iface_entry->active, + iface_entry->totals_via_dev[IFS_RX].bytes, + iface_entry->totals_via_dev[IFS_RX].packets, + iface_entry->totals_via_dev[IFS_TX].bytes, + iface_entry->totals_via_dev[IFS_TX].packets, + stats->rx_bytes, stats->rx_packets, + stats->tx_bytes, stats->tx_packets + ); + } else { + pp_iface_stat_line(m, iface_entry); + } + return 0; +} + +static const struct file_operations read_u64_fops = { + .read = read_proc_u64, + .llseek = default_llseek, +}; + +static const struct file_operations read_bool_fops = { + .read = read_proc_bool, + .llseek = default_llseek, +}; + +static void iface_create_proc_worker(struct work_struct *work) +{ + struct proc_dir_entry *proc_entry; + struct iface_stat_work *isw = container_of(work, struct iface_stat_work, + iface_work); + struct iface_stat *new_iface = isw->iface_entry; + + /* iface_entries are not deleted, so safe to manipulate. */ + proc_entry = proc_mkdir(new_iface->ifname, iface_stat_procdir); + if (IS_ERR_OR_NULL(proc_entry)) { + pr_err("qtaguid: iface_stat: create_proc(): alloc failed.\n"); + kfree(isw); + return; + } + + new_iface->proc_ptr = proc_entry; + + proc_create_data("tx_bytes", proc_iface_perms, proc_entry, + &read_u64_fops, + &new_iface->totals_via_dev[IFS_TX].bytes); + proc_create_data("rx_bytes", proc_iface_perms, proc_entry, + &read_u64_fops, + &new_iface->totals_via_dev[IFS_RX].bytes); + proc_create_data("tx_packets", proc_iface_perms, proc_entry, + &read_u64_fops, + &new_iface->totals_via_dev[IFS_TX].packets); + proc_create_data("rx_packets", proc_iface_perms, proc_entry, + &read_u64_fops, + &new_iface->totals_via_dev[IFS_RX].packets); + proc_create_data("active", proc_iface_perms, proc_entry, + &read_bool_fops, &new_iface->active); + + IF_DEBUG("qtaguid: iface_stat: create_proc(): done " + "entry=%p dev=%s\n", new_iface, new_iface->ifname); + kfree(isw); +} + +/* + * Will set the entry's active state, and + * update the net_dev accordingly also. + */ +static void _iface_stat_set_active(struct iface_stat *entry, + struct net_device *net_dev, + bool activate) +{ + if (activate) { + entry->net_dev = net_dev; + entry->active = true; + IF_DEBUG("qtaguid: %s(%s): " + "enable tracking. rfcnt=%d\n", __func__, + entry->ifname, + __this_cpu_read(*net_dev->pcpu_refcnt)); + } else { + entry->active = false; + entry->net_dev = NULL; + IF_DEBUG("qtaguid: %s(%s): " + "disable tracking. rfcnt=%d\n", __func__, + entry->ifname, + __this_cpu_read(*net_dev->pcpu_refcnt)); + + } +} + +/* Caller must hold iface_stat_list_lock */ +static struct iface_stat *iface_alloc(struct net_device *net_dev) +{ + struct iface_stat *new_iface; + struct iface_stat_work *isw; + + new_iface = kzalloc(sizeof(*new_iface), GFP_ATOMIC); + if (new_iface == NULL) { + pr_err("qtaguid: iface_stat: create(%s): " + "iface_stat alloc failed\n", net_dev->name); + return NULL; + } + new_iface->ifname = kstrdup(net_dev->name, GFP_ATOMIC); + if (new_iface->ifname == NULL) { + pr_err("qtaguid: iface_stat: create(%s): " + "ifname alloc failed\n", net_dev->name); + kfree(new_iface); + return NULL; + } + spin_lock_init(&new_iface->tag_stat_list_lock); + new_iface->tag_stat_tree = RB_ROOT; + _iface_stat_set_active(new_iface, net_dev, true); + + /* + * ipv6 notifier chains are atomic :( + * No create_proc_read_entry() for you! + */ + isw = kmalloc(sizeof(*isw), GFP_ATOMIC); + if (!isw) { + pr_err("qtaguid: iface_stat: create(%s): " + "work alloc failed\n", new_iface->ifname); + _iface_stat_set_active(new_iface, net_dev, false); + kfree(new_iface->ifname); + kfree(new_iface); + return NULL; + } + isw->iface_entry = new_iface; + INIT_WORK(&isw->iface_work, iface_create_proc_worker); + schedule_work(&isw->iface_work); + list_add(&new_iface->list, &iface_stat_list); + return new_iface; +} + +static void iface_check_stats_reset_and_adjust(struct net_device *net_dev, + struct iface_stat *iface) +{ + struct rtnl_link_stats64 dev_stats, *stats; + bool stats_rewound; + + stats = dev_get_stats(net_dev, &dev_stats); + /* No empty packets */ + stats_rewound = + (stats->rx_bytes < iface->last_known[IFS_RX].bytes) + || (stats->tx_bytes < iface->last_known[IFS_TX].bytes); + + IF_DEBUG("qtaguid: %s(%s): iface=%p netdev=%p " + "bytes rx/tx=%llu/%llu " + "active=%d last_known=%d " + "stats_rewound=%d\n", __func__, + net_dev ? net_dev->name : "?", + iface, net_dev, + stats->rx_bytes, stats->tx_bytes, + iface->active, iface->last_known_valid, stats_rewound); + + if (iface->active && iface->last_known_valid && stats_rewound) { + pr_warn_once("qtaguid: iface_stat: %s(%s): " + "iface reset its stats unexpectedly\n", __func__, + net_dev->name); + + iface->totals_via_dev[IFS_TX].bytes += + iface->last_known[IFS_TX].bytes; + iface->totals_via_dev[IFS_TX].packets += + iface->last_known[IFS_TX].packets; + iface->totals_via_dev[IFS_RX].bytes += + iface->last_known[IFS_RX].bytes; + iface->totals_via_dev[IFS_RX].packets += + iface->last_known[IFS_RX].packets; + iface->last_known_valid = false; + IF_DEBUG("qtaguid: %s(%s): iface=%p " + "used last known bytes rx/tx=%llu/%llu\n", __func__, + iface->ifname, iface, iface->last_known[IFS_RX].bytes, + iface->last_known[IFS_TX].bytes); + } +} + +/* + * Create a new entry for tracking the specified interface. + * Do nothing if the entry already exists. + * Called when an interface is configured with a valid IP address. + */ +static void iface_stat_create(struct net_device *net_dev, + struct in_ifaddr *ifa) +{ + struct in_device *in_dev = NULL; + const char *ifname; + struct iface_stat *entry; + __be32 ipaddr = 0; + struct iface_stat *new_iface; + + IF_DEBUG("qtaguid: iface_stat: create(%s): ifa=%p netdev=%p\n", + net_dev ? net_dev->name : "?", + ifa, net_dev); + if (!net_dev) { + pr_err("qtaguid: iface_stat: create(): no net dev\n"); + return; + } + + ifname = net_dev->name; + if (!ifa) { + in_dev = in_dev_get(net_dev); + if (!in_dev) { + pr_err("qtaguid: iface_stat: create(%s): no inet dev\n", + ifname); + return; + } + IF_DEBUG("qtaguid: iface_stat: create(%s): in_dev=%p\n", + ifname, in_dev); + for (ifa = in_dev->ifa_list; ifa; ifa = ifa->ifa_next) { + IF_DEBUG("qtaguid: iface_stat: create(%s): " + "ifa=%p ifa_label=%s\n", + ifname, ifa, + ifa->ifa_label ? ifa->ifa_label : "(null)"); + if (ifa->ifa_label && !strcmp(ifname, ifa->ifa_label)) + break; + } + } + + if (!ifa) { + IF_DEBUG("qtaguid: iface_stat: create(%s): no matching IP\n", + ifname); + goto done_put; + } + ipaddr = ifa->ifa_local; + + spin_lock_bh(&iface_stat_list_lock); + entry = get_iface_entry(ifname); + if (entry != NULL) { + IF_DEBUG("qtaguid: iface_stat: create(%s): entry=%p\n", + ifname, entry); + iface_check_stats_reset_and_adjust(net_dev, entry); + _iface_stat_set_active(entry, net_dev, true); + IF_DEBUG("qtaguid: %s(%s): " + "tracking now %d on ip=%pI4\n", __func__, + entry->ifname, true, &ipaddr); + goto done_unlock_put; + } + + new_iface = iface_alloc(net_dev); + IF_DEBUG("qtaguid: iface_stat: create(%s): done " + "entry=%p ip=%pI4\n", ifname, new_iface, &ipaddr); +done_unlock_put: + spin_unlock_bh(&iface_stat_list_lock); +done_put: + if (in_dev) + in_dev_put(in_dev); +} + +static void iface_stat_create_ipv6(struct net_device *net_dev, + struct inet6_ifaddr *ifa) +{ + struct in_device *in_dev; + const char *ifname; + struct iface_stat *entry; + struct iface_stat *new_iface; + int addr_type; + + IF_DEBUG("qtaguid: iface_stat: create6(): ifa=%p netdev=%p->name=%s\n", + ifa, net_dev, net_dev ? net_dev->name : ""); + if (!net_dev) { + pr_err("qtaguid: iface_stat: create6(): no net dev!\n"); + return; + } + ifname = net_dev->name; + + in_dev = in_dev_get(net_dev); + if (!in_dev) { + pr_err("qtaguid: iface_stat: create6(%s): no inet dev\n", + ifname); + return; + } + + IF_DEBUG("qtaguid: iface_stat: create6(%s): in_dev=%p\n", + ifname, in_dev); + + if (!ifa) { + IF_DEBUG("qtaguid: iface_stat: create6(%s): no matching IP\n", + ifname); + goto done_put; + } + addr_type = ipv6_addr_type(&ifa->addr); + + spin_lock_bh(&iface_stat_list_lock); + entry = get_iface_entry(ifname); + if (entry != NULL) { + IF_DEBUG("qtaguid: %s(%s): entry=%p\n", __func__, + ifname, entry); + iface_check_stats_reset_and_adjust(net_dev, entry); + _iface_stat_set_active(entry, net_dev, true); + IF_DEBUG("qtaguid: %s(%s): " + "tracking now %d on ip=%pI6c\n", __func__, + entry->ifname, true, &ifa->addr); + goto done_unlock_put; + } + + new_iface = iface_alloc(net_dev); + IF_DEBUG("qtaguid: iface_stat: create6(%s): done " + "entry=%p ip=%pI6c\n", ifname, new_iface, &ifa->addr); + +done_unlock_put: + spin_unlock_bh(&iface_stat_list_lock); +done_put: + in_dev_put(in_dev); +} + +static struct sock_tag *get_sock_stat_nl(const struct sock *sk) +{ + MT_DEBUG("qtaguid: get_sock_stat_nl(sk=%p)\n", sk); + return sock_tag_tree_search(&sock_tag_tree, sk); +} + +static struct sock_tag *get_sock_stat(const struct sock *sk) +{ + struct sock_tag *sock_tag_entry; + MT_DEBUG("qtaguid: get_sock_stat(sk=%p)\n", sk); + if (!sk) + return NULL; + spin_lock_bh(&sock_tag_list_lock); + sock_tag_entry = get_sock_stat_nl(sk); + spin_unlock_bh(&sock_tag_list_lock); + return sock_tag_entry; +} + +static int ipx_proto(const struct sk_buff *skb, + struct xt_action_param *par) +{ + int thoff = 0, tproto; + + switch (par->family) { + case NFPROTO_IPV6: + tproto = ipv6_find_hdr(skb, &thoff, -1, NULL, NULL); + if (tproto < 0) + MT_DEBUG("%s(): transport header not found in ipv6" + " skb=%p\n", __func__, skb); + break; + case NFPROTO_IPV4: + tproto = ip_hdr(skb)->protocol; + break; + default: + tproto = IPPROTO_RAW; + } + return tproto; +} + +static void +data_counters_update(struct data_counters *dc, int set, + enum ifs_tx_rx direction, int proto, int bytes) +{ + switch (proto) { + case IPPROTO_TCP: + dc_add_byte_packets(dc, set, direction, IFS_TCP, bytes, 1); + break; + case IPPROTO_UDP: + dc_add_byte_packets(dc, set, direction, IFS_UDP, bytes, 1); + break; + case IPPROTO_IP: + default: + dc_add_byte_packets(dc, set, direction, IFS_PROTO_OTHER, bytes, + 1); + break; + } +} + +/* + * Update stats for the specified interface. Do nothing if the entry + * does not exist (when a device was never configured with an IP address). + * Called when an device is being unregistered. + */ +static void iface_stat_update(struct net_device *net_dev, bool stash_only) +{ + struct rtnl_link_stats64 dev_stats, *stats; + struct iface_stat *entry; + + stats = dev_get_stats(net_dev, &dev_stats); + spin_lock_bh(&iface_stat_list_lock); + entry = get_iface_entry(net_dev->name); + if (entry == NULL) { + IF_DEBUG("qtaguid: iface_stat: update(%s): not tracked\n", + net_dev->name); + spin_unlock_bh(&iface_stat_list_lock); + return; + } + + IF_DEBUG("qtaguid: %s(%s): entry=%p\n", __func__, + net_dev->name, entry); + if (!entry->active) { + IF_DEBUG("qtaguid: %s(%s): already disabled\n", __func__, + net_dev->name); + spin_unlock_bh(&iface_stat_list_lock); + return; + } + + if (stash_only) { + entry->last_known[IFS_TX].bytes = stats->tx_bytes; + entry->last_known[IFS_TX].packets = stats->tx_packets; + entry->last_known[IFS_RX].bytes = stats->rx_bytes; + entry->last_known[IFS_RX].packets = stats->rx_packets; + entry->last_known_valid = true; + IF_DEBUG("qtaguid: %s(%s): " + "dev stats stashed rx/tx=%llu/%llu\n", __func__, + net_dev->name, stats->rx_bytes, stats->tx_bytes); + spin_unlock_bh(&iface_stat_list_lock); + return; + } + entry->totals_via_dev[IFS_TX].bytes += stats->tx_bytes; + entry->totals_via_dev[IFS_TX].packets += stats->tx_packets; + entry->totals_via_dev[IFS_RX].bytes += stats->rx_bytes; + entry->totals_via_dev[IFS_RX].packets += stats->rx_packets; + /* We don't need the last_known[] anymore */ + entry->last_known_valid = false; + _iface_stat_set_active(entry, net_dev, false); + IF_DEBUG("qtaguid: %s(%s): " + "disable tracking. rx/tx=%llu/%llu\n", __func__, + net_dev->name, stats->rx_bytes, stats->tx_bytes); + spin_unlock_bh(&iface_stat_list_lock); +} + +/* + * Update stats for the specified interface from the skb. + * Do nothing if the entry + * does not exist (when a device was never configured with an IP address). + * Called on each sk. + */ +static void iface_stat_update_from_skb(const struct sk_buff *skb, + struct xt_action_param *par) +{ + struct iface_stat *entry; + const struct net_device *el_dev; + enum ifs_tx_rx direction = par->in ? IFS_RX : IFS_TX; + int bytes = skb->len; + int proto; + + if (!skb->dev) { + MT_DEBUG("qtaguid[%d]: no skb->dev\n", par->hooknum); + el_dev = par->in ? : par->out; + } else { + const struct net_device *other_dev; + el_dev = skb->dev; + other_dev = par->in ? : par->out; + if (el_dev != other_dev) { + MT_DEBUG("qtaguid[%d]: skb->dev=%p %s vs " + "par->(in/out)=%p %s\n", + par->hooknum, el_dev, el_dev->name, other_dev, + other_dev->name); + } + } + + if (unlikely(!el_dev)) { + pr_err_ratelimited("qtaguid[%d]: %s(): no par->in/out?!!\n", + par->hooknum, __func__); + BUG(); + } else if (unlikely(!el_dev->name)) { + pr_err_ratelimited("qtaguid[%d]: %s(): no dev->name?!!\n", + par->hooknum, __func__); + BUG(); + } else { + proto = ipx_proto(skb, par); + MT_DEBUG("qtaguid[%d]: dev name=%s type=%d fam=%d proto=%d\n", + par->hooknum, el_dev->name, el_dev->type, + par->family, proto); + } + + spin_lock_bh(&iface_stat_list_lock); + entry = get_iface_entry(el_dev->name); + if (entry == NULL) { + IF_DEBUG("qtaguid: iface_stat: %s(%s): not tracked\n", + __func__, el_dev->name); + spin_unlock_bh(&iface_stat_list_lock); + return; + } + + IF_DEBUG("qtaguid: %s(%s): entry=%p\n", __func__, + el_dev->name, entry); + + data_counters_update(&entry->totals_via_skb, 0, direction, proto, + bytes); + spin_unlock_bh(&iface_stat_list_lock); +} + +static void tag_stat_update(struct tag_stat *tag_entry, + enum ifs_tx_rx direction, int proto, int bytes) +{ + int active_set; + active_set = get_active_counter_set(tag_entry->tn.tag); + MT_DEBUG("qtaguid: tag_stat_update(tag=0x%llx (uid=%u) set=%d " + "dir=%d proto=%d bytes=%d)\n", + tag_entry->tn.tag, get_uid_from_tag(tag_entry->tn.tag), + active_set, direction, proto, bytes); + data_counters_update(&tag_entry->counters, active_set, direction, + proto, bytes); + if (tag_entry->parent_counters) + data_counters_update(tag_entry->parent_counters, active_set, + direction, proto, bytes); +} + +/* + * Create a new entry for tracking the specified {acct_tag,uid_tag} within + * the interface. + * iface_entry->tag_stat_list_lock should be held. + */ +static struct tag_stat *create_if_tag_stat(struct iface_stat *iface_entry, + tag_t tag) +{ + struct tag_stat *new_tag_stat_entry = NULL; + IF_DEBUG("qtaguid: iface_stat: %s(): ife=%p tag=0x%llx" + " (uid=%u)\n", __func__, + iface_entry, tag, get_uid_from_tag(tag)); + new_tag_stat_entry = kzalloc(sizeof(*new_tag_stat_entry), GFP_ATOMIC); + if (!new_tag_stat_entry) { + pr_err("qtaguid: iface_stat: tag stat alloc failed\n"); + goto done; + } + new_tag_stat_entry->tn.tag = tag; + tag_stat_tree_insert(new_tag_stat_entry, &iface_entry->tag_stat_tree); +done: + return new_tag_stat_entry; +} + +static void if_tag_stat_update(const char *ifname, uid_t uid, + const struct sock *sk, enum ifs_tx_rx direction, + int proto, int bytes) +{ + struct tag_stat *tag_stat_entry; + tag_t tag, acct_tag; + tag_t uid_tag; + struct data_counters *uid_tag_counters; + struct sock_tag *sock_tag_entry; + struct iface_stat *iface_entry; + struct tag_stat *new_tag_stat = NULL; + MT_DEBUG("qtaguid: if_tag_stat_update(ifname=%s " + "uid=%u sk=%p dir=%d proto=%d bytes=%d)\n", + ifname, uid, sk, direction, proto, bytes); + + spin_lock_bh(&iface_stat_list_lock); + iface_entry = get_iface_entry(ifname); + if (!iface_entry) { + pr_err_ratelimited("qtaguid: iface_stat: stat_update() " + "%s not found\n", ifname); + spin_unlock_bh(&iface_stat_list_lock); + return; + } + /* It is ok to process data when an iface_entry is inactive */ + + MT_DEBUG("qtaguid: iface_stat: stat_update() dev=%s entry=%p\n", + ifname, iface_entry); + + /* + * Look for a tagged sock. + * It will have an acct_uid. + */ + sock_tag_entry = get_sock_stat(sk); + if (sock_tag_entry) { + tag = sock_tag_entry->tag; + acct_tag = get_atag_from_tag(tag); + uid_tag = get_utag_from_tag(tag); + } else { + acct_tag = make_atag_from_value(0); + tag = combine_atag_with_uid(acct_tag, uid); + uid_tag = make_tag_from_uid(uid); + } + MT_DEBUG("qtaguid: iface_stat: stat_update(): " + " looking for tag=0x%llx (uid=%u) in ife=%p\n", + tag, get_uid_from_tag(tag), iface_entry); + /* Loop over tag list under this interface for {acct_tag,uid_tag} */ + spin_lock_bh(&iface_entry->tag_stat_list_lock); + + tag_stat_entry = tag_stat_tree_search(&iface_entry->tag_stat_tree, + tag); + if (tag_stat_entry) { + /* + * Updating the {acct_tag, uid_tag} entry handles both stats: + * {0, uid_tag} will also get updated. + */ + tag_stat_update(tag_stat_entry, direction, proto, bytes); + goto unlock; + } + + /* Loop over tag list under this interface for {0,uid_tag} */ + tag_stat_entry = tag_stat_tree_search(&iface_entry->tag_stat_tree, + uid_tag); + if (!tag_stat_entry) { + /* Here: the base uid_tag did not exist */ + /* + * No parent counters. So + * - No {0, uid_tag} stats and no {acc_tag, uid_tag} stats. + */ + new_tag_stat = create_if_tag_stat(iface_entry, uid_tag); + if (!new_tag_stat) + goto unlock; + uid_tag_counters = &new_tag_stat->counters; + } else { + uid_tag_counters = &tag_stat_entry->counters; + } + + if (acct_tag) { + /* Create the child {acct_tag, uid_tag} and hook up parent. */ + new_tag_stat = create_if_tag_stat(iface_entry, tag); + if (!new_tag_stat) + goto unlock; + new_tag_stat->parent_counters = uid_tag_counters; + } else { + /* + * For new_tag_stat to be still NULL here would require: + * {0, uid_tag} exists + * and {acct_tag, uid_tag} doesn't exist + * AND acct_tag == 0. + * Impossible. This reassures us that new_tag_stat + * below will always be assigned. + */ + BUG_ON(!new_tag_stat); + } + tag_stat_update(new_tag_stat, direction, proto, bytes); +unlock: + spin_unlock_bh(&iface_entry->tag_stat_list_lock); + spin_unlock_bh(&iface_stat_list_lock); +} + +static int iface_netdev_event_handler(struct notifier_block *nb, + unsigned long event, void *ptr) { + struct net_device *dev = netdev_notifier_info_to_dev(ptr); + + if (unlikely(module_passive)) + return NOTIFY_DONE; + + IF_DEBUG("qtaguid: iface_stat: netdev_event(): " + "ev=0x%lx/%s netdev=%p->name=%s\n", + event, netdev_evt_str(event), dev, dev ? dev->name : ""); + + switch (event) { + case NETDEV_UP: + iface_stat_create(dev, NULL); + atomic64_inc(&qtu_events.iface_events); + break; + case NETDEV_DOWN: + case NETDEV_UNREGISTER: + iface_stat_update(dev, event == NETDEV_DOWN); + atomic64_inc(&qtu_events.iface_events); + break; + } + return NOTIFY_DONE; +} + +static int iface_inet6addr_event_handler(struct notifier_block *nb, + unsigned long event, void *ptr) +{ + struct inet6_ifaddr *ifa = ptr; + struct net_device *dev; + + if (unlikely(module_passive)) + return NOTIFY_DONE; + + IF_DEBUG("qtaguid: iface_stat: inet6addr_event(): " + "ev=0x%lx/%s ifa=%p\n", + event, netdev_evt_str(event), ifa); + + switch (event) { + case NETDEV_UP: + BUG_ON(!ifa || !ifa->idev); + dev = (struct net_device *)ifa->idev->dev; + iface_stat_create_ipv6(dev, ifa); + atomic64_inc(&qtu_events.iface_events); + break; + case NETDEV_DOWN: + case NETDEV_UNREGISTER: + BUG_ON(!ifa || !ifa->idev); + dev = (struct net_device *)ifa->idev->dev; + iface_stat_update(dev, event == NETDEV_DOWN); + atomic64_inc(&qtu_events.iface_events); + break; + } + return NOTIFY_DONE; +} + +static int iface_inetaddr_event_handler(struct notifier_block *nb, + unsigned long event, void *ptr) +{ + struct in_ifaddr *ifa = ptr; + struct net_device *dev; + + if (unlikely(module_passive)) + return NOTIFY_DONE; + + IF_DEBUG("qtaguid: iface_stat: inetaddr_event(): " + "ev=0x%lx/%s ifa=%p\n", + event, netdev_evt_str(event), ifa); + + switch (event) { + case NETDEV_UP: + BUG_ON(!ifa || !ifa->ifa_dev); + dev = ifa->ifa_dev->dev; + iface_stat_create(dev, ifa); + atomic64_inc(&qtu_events.iface_events); + break; + case NETDEV_DOWN: + case NETDEV_UNREGISTER: + BUG_ON(!ifa || !ifa->ifa_dev); + dev = ifa->ifa_dev->dev; + iface_stat_update(dev, event == NETDEV_DOWN); + atomic64_inc(&qtu_events.iface_events); + break; + } + return NOTIFY_DONE; +} + +static struct notifier_block iface_netdev_notifier_blk = { + .notifier_call = iface_netdev_event_handler, +}; + +static struct notifier_block iface_inetaddr_notifier_blk = { + .notifier_call = iface_inetaddr_event_handler, +}; + +static struct notifier_block iface_inet6addr_notifier_blk = { + .notifier_call = iface_inet6addr_event_handler, +}; + +static const struct seq_operations iface_stat_fmt_proc_seq_ops = { + .start = iface_stat_fmt_proc_start, + .next = iface_stat_fmt_proc_next, + .stop = iface_stat_fmt_proc_stop, + .show = iface_stat_fmt_proc_show, +}; + +static int proc_iface_stat_fmt_open(struct inode *inode, struct file *file) +{ + struct proc_iface_stat_fmt_info *s; + + s = __seq_open_private(file, &iface_stat_fmt_proc_seq_ops, + sizeof(struct proc_iface_stat_fmt_info)); + if (!s) + return -ENOMEM; + + s->fmt = (uintptr_t)PDE_DATA(inode); + return 0; +} + +static const struct file_operations proc_iface_stat_fmt_fops = { + .open = proc_iface_stat_fmt_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release_private, +}; + +static int __init iface_stat_init(struct proc_dir_entry *parent_procdir) +{ + int err; + + iface_stat_procdir = proc_mkdir(iface_stat_procdirname, parent_procdir); + if (!iface_stat_procdir) { + pr_err("qtaguid: iface_stat: init failed to create proc entry\n"); + err = -1; + goto err; + } + + iface_stat_all_procfile = proc_create_data(iface_stat_all_procfilename, + proc_iface_perms, + parent_procdir, + &proc_iface_stat_fmt_fops, + (void *)1 /* fmt1 */); + if (!iface_stat_all_procfile) { + pr_err("qtaguid: iface_stat: init " + " failed to create stat_old proc entry\n"); + err = -1; + goto err_zap_entry; + } + + iface_stat_fmt_procfile = proc_create_data(iface_stat_fmt_procfilename, + proc_iface_perms, + parent_procdir, + &proc_iface_stat_fmt_fops, + (void *)2 /* fmt2 */); + if (!iface_stat_fmt_procfile) { + pr_err("qtaguid: iface_stat: init " + " failed to create stat_all proc entry\n"); + err = -1; + goto err_zap_all_stats_entry; + } + + + err = register_netdevice_notifier(&iface_netdev_notifier_blk); + if (err) { + pr_err("qtaguid: iface_stat: init " + "failed to register dev event handler\n"); + goto err_zap_all_stats_entries; + } + err = register_inetaddr_notifier(&iface_inetaddr_notifier_blk); + if (err) { + pr_err("qtaguid: iface_stat: init " + "failed to register ipv4 dev event handler\n"); + goto err_unreg_nd; + } + + err = register_inet6addr_notifier(&iface_inet6addr_notifier_blk); + if (err) { + pr_err("qtaguid: iface_stat: init " + "failed to register ipv6 dev event handler\n"); + goto err_unreg_ip4_addr; + } + return 0; + +err_unreg_ip4_addr: + unregister_inetaddr_notifier(&iface_inetaddr_notifier_blk); +err_unreg_nd: + unregister_netdevice_notifier(&iface_netdev_notifier_blk); +err_zap_all_stats_entries: + remove_proc_entry(iface_stat_fmt_procfilename, parent_procdir); +err_zap_all_stats_entry: + remove_proc_entry(iface_stat_all_procfilename, parent_procdir); +err_zap_entry: + remove_proc_entry(iface_stat_procdirname, parent_procdir); +err: + return err; +} + +static struct sock *qtaguid_find_sk(const struct sk_buff *skb, + struct xt_action_param *par) +{ + struct sock *sk; + unsigned int hook_mask = (1 << par->hooknum); + + MT_DEBUG("qtaguid: find_sk(skb=%p) hooknum=%d family=%d\n", skb, + par->hooknum, par->family); + + /* + * Let's not abuse the the xt_socket_get*_sk(), or else it will + * return garbage SKs. + */ + if (!(hook_mask & XT_SOCKET_SUPPORTED_HOOKS)) + return NULL; + + switch (par->family) { + case NFPROTO_IPV6: + sk = xt_socket_lookup_slow_v6(dev_net(skb->dev), skb, par->in); + break; + case NFPROTO_IPV4: + sk = xt_socket_lookup_slow_v4(dev_net(skb->dev), skb, par->in); + break; + default: + return NULL; + } + + if (sk) { + MT_DEBUG("qtaguid: %p->sk_proto=%u " + "->sk_state=%d\n", sk, sk->sk_protocol, sk->sk_state); + /* + * When in TCP_TIME_WAIT the sk is not a "struct sock" but + * "struct inet_timewait_sock" which is missing fields. + */ + if (!sk_fullsock(sk) || sk->sk_state == TCP_TIME_WAIT) { + sock_gen_put(sk); + sk = NULL; + } + } + return sk; +} + +static void account_for_uid(const struct sk_buff *skb, + const struct sock *alternate_sk, uid_t uid, + struct xt_action_param *par) +{ + const struct net_device *el_dev; + + if (!skb->dev) { + MT_DEBUG("qtaguid[%d]: no skb->dev\n", par->hooknum); + el_dev = par->in ? : par->out; + } else { + const struct net_device *other_dev; + el_dev = skb->dev; + other_dev = par->in ? : par->out; + if (el_dev != other_dev) { + MT_DEBUG("qtaguid[%d]: skb->dev=%p %s vs " + "par->(in/out)=%p %s\n", + par->hooknum, el_dev, el_dev->name, other_dev, + other_dev->name); + } + } + + if (unlikely(!el_dev)) { + pr_info("qtaguid[%d]: no par->in/out?!!\n", par->hooknum); + } else if (unlikely(!el_dev->name)) { + pr_info("qtaguid[%d]: no dev->name?!!\n", par->hooknum); + } else { + int proto = ipx_proto(skb, par); + MT_DEBUG("qtaguid[%d]: dev name=%s type=%d fam=%d proto=%d\n", + par->hooknum, el_dev->name, el_dev->type, + par->family, proto); + + if_tag_stat_update(el_dev->name, uid, + skb->sk ? skb->sk : alternate_sk, + par->in ? IFS_RX : IFS_TX, + proto, skb->len); + } +} + +static bool qtaguid_mt(const struct sk_buff *skb, struct xt_action_param *par) +{ + const struct xt_qtaguid_match_info *info = par->matchinfo; + const struct file *filp; + bool got_sock = false; + struct sock *sk; + kuid_t sock_uid; + bool res; + bool set_sk_callback_lock = false; + + if (unlikely(module_passive)) + return (info->match ^ info->invert) == 0; + + MT_DEBUG("qtaguid[%d]: entered skb=%p par->in=%p/out=%p fam=%d\n", + par->hooknum, skb, par->in, par->out, par->family); + + atomic64_inc(&qtu_events.match_calls); + if (skb == NULL) { + res = (info->match ^ info->invert) == 0; + goto ret_res; + } + + switch (par->hooknum) { + case NF_INET_PRE_ROUTING: + case NF_INET_POST_ROUTING: + atomic64_inc(&qtu_events.match_calls_prepost); + iface_stat_update_from_skb(skb, par); + /* + * We are done in pre/post. The skb will get processed + * further alter. + */ + res = (info->match ^ info->invert); + goto ret_res; + break; + /* default: Fall through and do UID releated work */ + } + + sk = skb_to_full_sk(skb); + /* + * When in TCP_TIME_WAIT the sk is not a "struct sock" but + * "struct inet_timewait_sock" which is missing fields. + * So we ignore it. + */ + if (sk && sk->sk_state == TCP_TIME_WAIT) + sk = NULL; + if (sk == NULL) { + /* + * A missing sk->sk_socket happens when packets are in-flight + * and the matching socket is already closed and gone. + */ + sk = qtaguid_find_sk(skb, par); + /* + * If we got the socket from the find_sk(), we will need to put + * it back, as nf_tproxy_get_sock_v4() got it. + */ + got_sock = sk; + if (sk) + atomic64_inc(&qtu_events.match_found_sk_in_ct); + else + atomic64_inc(&qtu_events.match_found_no_sk_in_ct); + } else { + atomic64_inc(&qtu_events.match_found_sk); + } + MT_DEBUG("qtaguid[%d]: sk=%p got_sock=%d fam=%d proto=%d\n", + par->hooknum, sk, got_sock, par->family, ipx_proto(skb, par)); + if (sk != NULL) { + set_sk_callback_lock = true; + read_lock_bh(&sk->sk_callback_lock); + MT_DEBUG("qtaguid[%d]: sk=%p->sk_socket=%p->file=%p\n", + par->hooknum, sk, sk->sk_socket, + sk->sk_socket ? sk->sk_socket->file : (void *)-1LL); + filp = sk->sk_socket ? sk->sk_socket->file : NULL; + MT_DEBUG("qtaguid[%d]: filp...uid=%u\n", + par->hooknum, filp ? from_kuid(&init_user_ns, filp->f_cred->fsuid) : -1); + } + + if (sk == NULL || sk->sk_socket == NULL) { + /* + * Here, the qtaguid_find_sk() using connection tracking + * couldn't find the owner, so for now we just count them + * against the system. + */ + /* + * TODO: unhack how to force just accounting. + * For now we only do iface stats when the uid-owner is not + * requested. + */ + if (!(info->match & XT_QTAGUID_UID)) + account_for_uid(skb, sk, 0, par); + MT_DEBUG("qtaguid[%d]: leaving (sk?sk->sk_socket)=%p\n", + par->hooknum, + sk ? sk->sk_socket : NULL); + res = (info->match ^ info->invert) == 0; + atomic64_inc(&qtu_events.match_no_sk); + goto put_sock_ret_res; + } else if (info->match & info->invert & XT_QTAGUID_SOCKET) { + res = false; + goto put_sock_ret_res; + } + filp = sk->sk_socket->file; + if (filp == NULL) { + MT_DEBUG("qtaguid[%d]: leaving filp=NULL\n", par->hooknum); + account_for_uid(skb, sk, 0, par); + res = ((info->match ^ info->invert) & + (XT_QTAGUID_UID | XT_QTAGUID_GID)) == 0; + atomic64_inc(&qtu_events.match_no_sk_file); + goto put_sock_ret_res; + } + sock_uid = filp->f_cred->fsuid; + /* + * TODO: unhack how to force just accounting. + * For now we only do iface stats when the uid-owner is not requested + */ + if (!(info->match & XT_QTAGUID_UID)) + account_for_uid(skb, sk, from_kuid(&init_user_ns, sock_uid), par); + + /* + * The following two tests fail the match when: + * id not in range AND no inverted condition requested + * or id in range AND inverted condition requested + * Thus (!a && b) || (a && !b) == a ^ b + */ + if (info->match & XT_QTAGUID_UID) { + kuid_t uid_min = make_kuid(&init_user_ns, info->uid_min); + kuid_t uid_max = make_kuid(&init_user_ns, info->uid_max); + + if ((uid_gte(filp->f_cred->fsuid, uid_min) && + uid_lte(filp->f_cred->fsuid, uid_max)) ^ + !(info->invert & XT_QTAGUID_UID)) { + MT_DEBUG("qtaguid[%d]: leaving uid not matching\n", + par->hooknum); + res = false; + goto put_sock_ret_res; + } + } + if (info->match & XT_QTAGUID_GID) { + kgid_t gid_min = make_kgid(&init_user_ns, info->gid_min); + kgid_t gid_max = make_kgid(&init_user_ns, info->gid_max); + + if ((gid_gte(filp->f_cred->fsgid, gid_min) && + gid_lte(filp->f_cred->fsgid, gid_max)) ^ + !(info->invert & XT_QTAGUID_GID)) { + MT_DEBUG("qtaguid[%d]: leaving gid not matching\n", + par->hooknum); + res = false; + goto put_sock_ret_res; + } + } + MT_DEBUG("qtaguid[%d]: leaving matched\n", par->hooknum); + res = true; + +put_sock_ret_res: + if (got_sock) + sock_gen_put(sk); + if (set_sk_callback_lock) + read_unlock_bh(&sk->sk_callback_lock); +ret_res: + MT_DEBUG("qtaguid[%d]: left %d\n", par->hooknum, res); + return res; +} + +#ifdef DDEBUG +/* This function is not in xt_qtaguid_print.c because of locks visibility */ +static void prdebug_full_state(int indent_level, const char *fmt, ...) +{ + va_list args; + char *fmt_buff; + char *buff; + + if (!unlikely(qtaguid_debug_mask & DDEBUG_MASK)) + return; + + fmt_buff = kasprintf(GFP_ATOMIC, + "qtaguid: %s(): %s {\n", __func__, fmt); + BUG_ON(!fmt_buff); + va_start(args, fmt); + buff = kvasprintf(GFP_ATOMIC, + fmt_buff, args); + BUG_ON(!buff); + pr_debug("%s", buff); + kfree(fmt_buff); + kfree(buff); + va_end(args); + + spin_lock_bh(&sock_tag_list_lock); + prdebug_sock_tag_tree(indent_level, &sock_tag_tree); + spin_unlock_bh(&sock_tag_list_lock); + + spin_lock_bh(&sock_tag_list_lock); + spin_lock_bh(&uid_tag_data_tree_lock); + prdebug_uid_tag_data_tree(indent_level, &uid_tag_data_tree); + prdebug_proc_qtu_data_tree(indent_level, &proc_qtu_data_tree); + spin_unlock_bh(&uid_tag_data_tree_lock); + spin_unlock_bh(&sock_tag_list_lock); + + spin_lock_bh(&iface_stat_list_lock); + prdebug_iface_stat_list(indent_level, &iface_stat_list); + spin_unlock_bh(&iface_stat_list_lock); + + pr_debug("qtaguid: %s(): }\n", __func__); +} +#else +static void prdebug_full_state(int indent_level, const char *fmt, ...) {} +#endif + +struct proc_ctrl_print_info { + struct sock *sk; /* socket found by reading to sk_pos */ + loff_t sk_pos; +}; + +static void *qtaguid_ctrl_proc_next(struct seq_file *m, void *v, loff_t *pos) +{ + struct proc_ctrl_print_info *pcpi = m->private; + struct sock_tag *sock_tag_entry = v; + struct rb_node *node; + + (*pos)++; + + if (!v || v == SEQ_START_TOKEN) + return NULL; + + node = rb_next(&sock_tag_entry->sock_node); + if (!node) { + pcpi->sk = NULL; + sock_tag_entry = SEQ_START_TOKEN; + } else { + sock_tag_entry = rb_entry(node, struct sock_tag, sock_node); + pcpi->sk = sock_tag_entry->sk; + } + pcpi->sk_pos = *pos; + return sock_tag_entry; +} + +static void *qtaguid_ctrl_proc_start(struct seq_file *m, loff_t *pos) +{ + struct proc_ctrl_print_info *pcpi = m->private; + struct sock_tag *sock_tag_entry; + struct rb_node *node; + + spin_lock_bh(&sock_tag_list_lock); + + if (unlikely(module_passive)) + return NULL; + + if (*pos == 0) { + pcpi->sk_pos = 0; + node = rb_first(&sock_tag_tree); + if (!node) { + pcpi->sk = NULL; + return SEQ_START_TOKEN; + } + sock_tag_entry = rb_entry(node, struct sock_tag, sock_node); + pcpi->sk = sock_tag_entry->sk; + } else { + sock_tag_entry = (pcpi->sk ? get_sock_stat_nl(pcpi->sk) : + NULL) ?: SEQ_START_TOKEN; + if (*pos != pcpi->sk_pos) { + /* seq_read skipped a next call */ + *pos = pcpi->sk_pos; + return qtaguid_ctrl_proc_next(m, sock_tag_entry, pos); + } + } + return sock_tag_entry; +} + +static void qtaguid_ctrl_proc_stop(struct seq_file *m, void *v) +{ + spin_unlock_bh(&sock_tag_list_lock); +} + +/* + * Procfs reader to get all active socket tags using style "1)" as described in + * fs/proc/generic.c + */ +static int qtaguid_ctrl_proc_show(struct seq_file *m, void *v) +{ + struct sock_tag *sock_tag_entry = v; + uid_t uid; + long f_count; + + CT_DEBUG("qtaguid: proc ctrl pid=%u tgid=%u uid=%u\n", + current->pid, current->tgid, from_kuid(&init_user_ns, current_fsuid())); + + if (sock_tag_entry != SEQ_START_TOKEN) { + uid = get_uid_from_tag(sock_tag_entry->tag); + CT_DEBUG("qtaguid: proc_read(): sk=%p tag=0x%llx (uid=%u) " + "pid=%u\n", + sock_tag_entry->sk, + sock_tag_entry->tag, + uid, + sock_tag_entry->pid + ); + f_count = atomic_long_read( + &sock_tag_entry->socket->file->f_count); + seq_printf(m, "sock=%pK tag=0x%llx (uid=%u) pid=%u " + "f_count=%lu\n", + sock_tag_entry->sk, + sock_tag_entry->tag, uid, + sock_tag_entry->pid, f_count); + } else { + seq_printf(m, "events: sockets_tagged=%llu " + "sockets_untagged=%llu " + "counter_set_changes=%llu " + "delete_cmds=%llu " + "iface_events=%llu " + "match_calls=%llu " + "match_calls_prepost=%llu " + "match_found_sk=%llu " + "match_found_sk_in_ct=%llu " + "match_found_no_sk_in_ct=%llu " + "match_no_sk=%llu " + "match_no_sk_file=%llu\n", + (u64)atomic64_read(&qtu_events.sockets_tagged), + (u64)atomic64_read(&qtu_events.sockets_untagged), + (u64)atomic64_read(&qtu_events.counter_set_changes), + (u64)atomic64_read(&qtu_events.delete_cmds), + (u64)atomic64_read(&qtu_events.iface_events), + (u64)atomic64_read(&qtu_events.match_calls), + (u64)atomic64_read(&qtu_events.match_calls_prepost), + (u64)atomic64_read(&qtu_events.match_found_sk), + (u64)atomic64_read(&qtu_events.match_found_sk_in_ct), + (u64)atomic64_read(&qtu_events.match_found_no_sk_in_ct), + (u64)atomic64_read(&qtu_events.match_no_sk), + (u64)atomic64_read(&qtu_events.match_no_sk_file)); + + /* Count the following as part of the last item_index */ + prdebug_full_state(0, "proc ctrl"); + } + + return 0; +} + +/* + * Delete socket tags, and stat tags associated with a given + * accouting tag and uid. + */ +static int ctrl_cmd_delete(const char *input) +{ + char cmd; + int uid_int; + kuid_t uid; + uid_t entry_uid; + tag_t acct_tag; + tag_t tag; + int res, argc; + struct iface_stat *iface_entry; + struct rb_node *node; + struct sock_tag *st_entry; + struct rb_root st_to_free_tree = RB_ROOT; + struct tag_stat *ts_entry; + struct tag_counter_set *tcs_entry; + struct tag_ref *tr_entry; + struct uid_tag_data *utd_entry; + + argc = sscanf(input, "%c %llu %u", &cmd, &acct_tag, &uid_int); + uid = make_kuid(&init_user_ns, uid_int); + CT_DEBUG("qtaguid: ctrl_delete(%s): argc=%d cmd=%c " + "user_tag=0x%llx uid=%u\n", input, argc, cmd, + acct_tag, uid_int); + if (argc < 2) { + res = -EINVAL; + goto err; + } + if (!valid_atag(acct_tag)) { + pr_info("qtaguid: ctrl_delete(%s): invalid tag\n", input); + res = -EINVAL; + goto err; + } + if (argc < 3) { + uid = current_fsuid(); + uid_int = from_kuid(&init_user_ns, uid); + } else if (!can_impersonate_uid(uid)) { + pr_info("qtaguid: ctrl_delete(%s): " + "insufficient priv from pid=%u tgid=%u uid=%u\n", + input, current->pid, current->tgid, from_kuid(&init_user_ns, current_fsuid())); + res = -EPERM; + goto err; + } + + tag = combine_atag_with_uid(acct_tag, uid_int); + CT_DEBUG("qtaguid: ctrl_delete(%s): " + "looking for tag=0x%llx (uid=%u)\n", + input, tag, uid_int); + + /* Delete socket tags */ + spin_lock_bh(&sock_tag_list_lock); + node = rb_first(&sock_tag_tree); + while (node) { + st_entry = rb_entry(node, struct sock_tag, sock_node); + entry_uid = get_uid_from_tag(st_entry->tag); + node = rb_next(node); + if (entry_uid != uid_int) + continue; + + CT_DEBUG("qtaguid: ctrl_delete(%s): st tag=0x%llx (uid=%u)\n", + input, st_entry->tag, entry_uid); + + if (!acct_tag || st_entry->tag == tag) { + rb_erase(&st_entry->sock_node, &sock_tag_tree); + /* Can't sockfd_put() within spinlock, do it later. */ + sock_tag_tree_insert(st_entry, &st_to_free_tree); + tr_entry = lookup_tag_ref(st_entry->tag, NULL); + BUG_ON(tr_entry->num_sock_tags <= 0); + tr_entry->num_sock_tags--; + /* + * TODO: remove if, and start failing. + * This is a hack to work around the fact that in some + * places we have "if (IS_ERR_OR_NULL(pqd_entry))" + * and are trying to work around apps + * that didn't open the /dev/xt_qtaguid. + */ + if (st_entry->list.next && st_entry->list.prev) + list_del(&st_entry->list); + } + } + spin_unlock_bh(&sock_tag_list_lock); + + sock_tag_tree_erase(&st_to_free_tree); + + /* Delete tag counter-sets */ + spin_lock_bh(&tag_counter_set_list_lock); + /* Counter sets are only on the uid tag, not full tag */ + tcs_entry = tag_counter_set_tree_search(&tag_counter_set_tree, tag); + if (tcs_entry) { + CT_DEBUG("qtaguid: ctrl_delete(%s): " + "erase tcs: tag=0x%llx (uid=%u) set=%d\n", + input, + tcs_entry->tn.tag, + get_uid_from_tag(tcs_entry->tn.tag), + tcs_entry->active_set); + rb_erase(&tcs_entry->tn.node, &tag_counter_set_tree); + kfree(tcs_entry); + } + spin_unlock_bh(&tag_counter_set_list_lock); + + /* + * If acct_tag is 0, then all entries belonging to uid are + * erased. + */ + spin_lock_bh(&iface_stat_list_lock); + list_for_each_entry(iface_entry, &iface_stat_list, list) { + spin_lock_bh(&iface_entry->tag_stat_list_lock); + node = rb_first(&iface_entry->tag_stat_tree); + while (node) { + ts_entry = rb_entry(node, struct tag_stat, tn.node); + entry_uid = get_uid_from_tag(ts_entry->tn.tag); + node = rb_next(node); + + CT_DEBUG("qtaguid: ctrl_delete(%s): " + "ts tag=0x%llx (uid=%u)\n", + input, ts_entry->tn.tag, entry_uid); + + if (entry_uid != uid_int) + continue; + if (!acct_tag || ts_entry->tn.tag == tag) { + CT_DEBUG("qtaguid: ctrl_delete(%s): " + "erase ts: %s 0x%llx %u\n", + input, iface_entry->ifname, + get_atag_from_tag(ts_entry->tn.tag), + entry_uid); + rb_erase(&ts_entry->tn.node, + &iface_entry->tag_stat_tree); + kfree(ts_entry); + } + } + spin_unlock_bh(&iface_entry->tag_stat_list_lock); + } + spin_unlock_bh(&iface_stat_list_lock); + + /* Cleanup the uid_tag_data */ + spin_lock_bh(&uid_tag_data_tree_lock); + node = rb_first(&uid_tag_data_tree); + while (node) { + utd_entry = rb_entry(node, struct uid_tag_data, node); + entry_uid = utd_entry->uid; + node = rb_next(node); + + CT_DEBUG("qtaguid: ctrl_delete(%s): " + "utd uid=%u\n", + input, entry_uid); + + if (entry_uid != uid_int) + continue; + /* + * Go over the tag_refs, and those that don't have + * sock_tags using them are freed. + */ + put_tag_ref_tree(tag, utd_entry); + put_utd_entry(utd_entry); + } + spin_unlock_bh(&uid_tag_data_tree_lock); + + atomic64_inc(&qtu_events.delete_cmds); + res = 0; + +err: + return res; +} + +static int ctrl_cmd_counter_set(const char *input) +{ + char cmd; + uid_t uid = 0; + tag_t tag; + int res, argc; + struct tag_counter_set *tcs; + int counter_set; + + argc = sscanf(input, "%c %d %u", &cmd, &counter_set, &uid); + CT_DEBUG("qtaguid: ctrl_counterset(%s): argc=%d cmd=%c " + "set=%d uid=%u\n", input, argc, cmd, + counter_set, uid); + if (argc != 3) { + res = -EINVAL; + goto err; + } + if (counter_set < 0 || counter_set >= IFS_MAX_COUNTER_SETS) { + pr_info("qtaguid: ctrl_counterset(%s): invalid counter_set range\n", + input); + res = -EINVAL; + goto err; + } + if (!can_manipulate_uids()) { + pr_info("qtaguid: ctrl_counterset(%s): " + "insufficient priv from pid=%u tgid=%u uid=%u\n", + input, current->pid, current->tgid, from_kuid(&init_user_ns, current_fsuid())); + res = -EPERM; + goto err; + } + + tag = make_tag_from_uid(uid); + spin_lock_bh(&tag_counter_set_list_lock); + tcs = tag_counter_set_tree_search(&tag_counter_set_tree, tag); + if (!tcs) { + tcs = kzalloc(sizeof(*tcs), GFP_ATOMIC); + if (!tcs) { + spin_unlock_bh(&tag_counter_set_list_lock); + pr_err("qtaguid: ctrl_counterset(%s): " + "failed to alloc counter set\n", + input); + res = -ENOMEM; + goto err; + } + tcs->tn.tag = tag; + tag_counter_set_tree_insert(tcs, &tag_counter_set_tree); + CT_DEBUG("qtaguid: ctrl_counterset(%s): added tcs tag=0x%llx " + "(uid=%u) set=%d\n", + input, tag, get_uid_from_tag(tag), counter_set); + } + tcs->active_set = counter_set; + spin_unlock_bh(&tag_counter_set_list_lock); + atomic64_inc(&qtu_events.counter_set_changes); + res = 0; + +err: + return res; +} + +static int ctrl_cmd_tag(const char *input) +{ + char cmd; + int sock_fd = 0; + kuid_t uid; + unsigned int uid_int = 0; + tag_t acct_tag = make_atag_from_value(0); + tag_t full_tag; + struct socket *el_socket; + int res, argc; + struct sock_tag *sock_tag_entry; + struct tag_ref *tag_ref_entry; + struct uid_tag_data *uid_tag_data_entry; + struct proc_qtu_data *pqd_entry; + + /* Unassigned args will get defaulted later. */ + argc = sscanf(input, "%c %d %llu %u", &cmd, &sock_fd, &acct_tag, &uid_int); + uid = make_kuid(&init_user_ns, uid_int); + CT_DEBUG("qtaguid: ctrl_tag(%s): argc=%d cmd=%c sock_fd=%d " + "acct_tag=0x%llx uid=%u\n", input, argc, cmd, sock_fd, + acct_tag, uid_int); + if (argc < 2) { + res = -EINVAL; + goto err; + } + el_socket = sockfd_lookup(sock_fd, &res); /* This locks the file */ + if (!el_socket) { + pr_info("qtaguid: ctrl_tag(%s): failed to lookup" + " sock_fd=%d err=%d pid=%u tgid=%u uid=%u\n", + input, sock_fd, res, current->pid, current->tgid, + from_kuid(&init_user_ns, current_fsuid())); + goto err; + } + CT_DEBUG("qtaguid: ctrl_tag(%s): socket->...->f_count=%ld ->sk=%p\n", + input, atomic_long_read(&el_socket->file->f_count), + el_socket->sk); + if (argc < 3) { + acct_tag = make_atag_from_value(0); + } else if (!valid_atag(acct_tag)) { + pr_info("qtaguid: ctrl_tag(%s): invalid tag\n", input); + res = -EINVAL; + goto err_put; + } + CT_DEBUG("qtaguid: ctrl_tag(%s): " + "pid=%u tgid=%u uid=%u euid=%u fsuid=%u " + "ctrl.gid=%u in_group()=%d in_egroup()=%d\n", + input, current->pid, current->tgid, + from_kuid(&init_user_ns, current_uid()), + from_kuid(&init_user_ns, current_euid()), + from_kuid(&init_user_ns, current_fsuid()), + from_kgid(&init_user_ns, xt_qtaguid_ctrl_file->gid), + in_group_p(xt_qtaguid_ctrl_file->gid), + in_egroup_p(xt_qtaguid_ctrl_file->gid)); + if (argc < 4) { + uid = current_fsuid(); + uid_int = from_kuid(&init_user_ns, uid); + } else if (!can_impersonate_uid(uid)) { + pr_info("qtaguid: ctrl_tag(%s): " + "insufficient priv from pid=%u tgid=%u uid=%u\n", + input, current->pid, current->tgid, from_kuid(&init_user_ns, current_fsuid())); + res = -EPERM; + goto err_put; + } + full_tag = combine_atag_with_uid(acct_tag, uid_int); + + spin_lock_bh(&sock_tag_list_lock); + sock_tag_entry = get_sock_stat_nl(el_socket->sk); + tag_ref_entry = get_tag_ref(full_tag, &uid_tag_data_entry); + if (IS_ERR(tag_ref_entry)) { + res = PTR_ERR(tag_ref_entry); + spin_unlock_bh(&sock_tag_list_lock); + goto err_put; + } + tag_ref_entry->num_sock_tags++; + if (sock_tag_entry) { + struct tag_ref *prev_tag_ref_entry; + + CT_DEBUG("qtaguid: ctrl_tag(%s): retag for sk=%p " + "st@%p ...->f_count=%ld\n", + input, el_socket->sk, sock_tag_entry, + atomic_long_read(&el_socket->file->f_count)); + /* + * This is a re-tagging, so release the sock_fd that was + * locked at the time of the 1st tagging. + * There is still the ref from this call's sockfd_lookup() so + * it can be done within the spinlock. + */ + sockfd_put(sock_tag_entry->socket); + prev_tag_ref_entry = lookup_tag_ref(sock_tag_entry->tag, + &uid_tag_data_entry); + BUG_ON(IS_ERR_OR_NULL(prev_tag_ref_entry)); + BUG_ON(prev_tag_ref_entry->num_sock_tags <= 0); + prev_tag_ref_entry->num_sock_tags--; + sock_tag_entry->tag = full_tag; + } else { + CT_DEBUG("qtaguid: ctrl_tag(%s): newtag for sk=%p\n", + input, el_socket->sk); + sock_tag_entry = kzalloc(sizeof(*sock_tag_entry), + GFP_ATOMIC); + if (!sock_tag_entry) { + pr_err("qtaguid: ctrl_tag(%s): " + "socket tag alloc failed\n", + input); + spin_unlock_bh(&sock_tag_list_lock); + res = -ENOMEM; + goto err_tag_unref_put; + } + sock_tag_entry->sk = el_socket->sk; + sock_tag_entry->socket = el_socket; + sock_tag_entry->pid = current->tgid; + sock_tag_entry->tag = combine_atag_with_uid(acct_tag, uid_int); + spin_lock_bh(&uid_tag_data_tree_lock); + pqd_entry = proc_qtu_data_tree_search( + &proc_qtu_data_tree, current->tgid); + /* + * TODO: remove if, and start failing. + * At first, we want to catch user-space code that is not + * opening the /dev/xt_qtaguid. + */ + if (IS_ERR_OR_NULL(pqd_entry)) + pr_warn_once( + "qtaguid: %s(): " + "User space forgot to open /dev/xt_qtaguid? " + "pid=%u tgid=%u uid=%u\n", __func__, + current->pid, current->tgid, + from_kuid(&init_user_ns, current_fsuid())); + else + list_add(&sock_tag_entry->list, + &pqd_entry->sock_tag_list); + spin_unlock_bh(&uid_tag_data_tree_lock); + + sock_tag_tree_insert(sock_tag_entry, &sock_tag_tree); + atomic64_inc(&qtu_events.sockets_tagged); + } + spin_unlock_bh(&sock_tag_list_lock); + /* We keep the ref to the socket (file) until it is untagged */ + CT_DEBUG("qtaguid: ctrl_tag(%s): done st@%p ...->f_count=%ld\n", + input, sock_tag_entry, + atomic_long_read(&el_socket->file->f_count)); + return 0; + +err_tag_unref_put: + BUG_ON(tag_ref_entry->num_sock_tags <= 0); + tag_ref_entry->num_sock_tags--; + free_tag_ref_from_utd_entry(tag_ref_entry, uid_tag_data_entry); +err_put: + CT_DEBUG("qtaguid: ctrl_tag(%s): done. ...->f_count=%ld\n", + input, atomic_long_read(&el_socket->file->f_count) - 1); + /* Release the sock_fd that was grabbed by sockfd_lookup(). */ + sockfd_put(el_socket); + return res; + +err: + CT_DEBUG("qtaguid: ctrl_tag(%s): done.\n", input); + return res; +} + +static int ctrl_cmd_untag(const char *input) +{ + char cmd; + int sock_fd = 0; + struct socket *el_socket; + int res, argc; + struct sock_tag *sock_tag_entry; + struct tag_ref *tag_ref_entry; + struct uid_tag_data *utd_entry; + struct proc_qtu_data *pqd_entry; + + argc = sscanf(input, "%c %d", &cmd, &sock_fd); + CT_DEBUG("qtaguid: ctrl_untag(%s): argc=%d cmd=%c sock_fd=%d\n", + input, argc, cmd, sock_fd); + if (argc < 2) { + res = -EINVAL; + goto err; + } + el_socket = sockfd_lookup(sock_fd, &res); /* This locks the file */ + if (!el_socket) { + pr_info("qtaguid: ctrl_untag(%s): failed to lookup" + " sock_fd=%d err=%d pid=%u tgid=%u uid=%u\n", + input, sock_fd, res, current->pid, current->tgid, + from_kuid(&init_user_ns, current_fsuid())); + goto err; + } + CT_DEBUG("qtaguid: ctrl_untag(%s): socket->...->f_count=%ld ->sk=%p\n", + input, atomic_long_read(&el_socket->file->f_count), + el_socket->sk); + spin_lock_bh(&sock_tag_list_lock); + sock_tag_entry = get_sock_stat_nl(el_socket->sk); + if (!sock_tag_entry) { + spin_unlock_bh(&sock_tag_list_lock); + res = -EINVAL; + goto err_put; + } + /* + * The socket already belongs to the current process + * so it can do whatever it wants to it. + */ + rb_erase(&sock_tag_entry->sock_node, &sock_tag_tree); + + tag_ref_entry = lookup_tag_ref(sock_tag_entry->tag, &utd_entry); + BUG_ON(!tag_ref_entry); + BUG_ON(tag_ref_entry->num_sock_tags <= 0); + spin_lock_bh(&uid_tag_data_tree_lock); + pqd_entry = proc_qtu_data_tree_search( + &proc_qtu_data_tree, current->tgid); + /* + * TODO: remove if, and start failing. + * At first, we want to catch user-space code that is not + * opening the /dev/xt_qtaguid. + */ + if (IS_ERR_OR_NULL(pqd_entry)) + pr_warn_once("qtaguid: %s(): " + "User space forgot to open /dev/xt_qtaguid? " + "pid=%u tgid=%u uid=%u\n", __func__, + current->pid, current->tgid, from_kuid(&init_user_ns, current_fsuid())); + else + list_del(&sock_tag_entry->list); + spin_unlock_bh(&uid_tag_data_tree_lock); + /* + * We don't free tag_ref from the utd_entry here, + * only during a cmd_delete(). + */ + tag_ref_entry->num_sock_tags--; + spin_unlock_bh(&sock_tag_list_lock); + /* + * Release the sock_fd that was grabbed at tag time, + * and once more for the sockfd_lookup() here. + */ + sockfd_put(sock_tag_entry->socket); + CT_DEBUG("qtaguid: ctrl_untag(%s): done. st@%p ...->f_count=%ld\n", + input, sock_tag_entry, + atomic_long_read(&el_socket->file->f_count) - 1); + sockfd_put(el_socket); + + kfree(sock_tag_entry); + atomic64_inc(&qtu_events.sockets_untagged); + + return 0; + +err_put: + CT_DEBUG("qtaguid: ctrl_untag(%s): done. socket->...->f_count=%ld\n", + input, atomic_long_read(&el_socket->file->f_count) - 1); + /* Release the sock_fd that was grabbed by sockfd_lookup(). */ + sockfd_put(el_socket); + return res; + +err: + CT_DEBUG("qtaguid: ctrl_untag(%s): done.\n", input); + return res; +} + +static ssize_t qtaguid_ctrl_parse(const char *input, size_t count) +{ + char cmd; + ssize_t res; + + CT_DEBUG("qtaguid: ctrl(%s): pid=%u tgid=%u uid=%u\n", + input, current->pid, current->tgid, from_kuid(&init_user_ns, current_fsuid())); + + cmd = input[0]; + /* Collect params for commands */ + switch (cmd) { + case 'd': + res = ctrl_cmd_delete(input); + break; + + case 's': + res = ctrl_cmd_counter_set(input); + break; + + case 't': + res = ctrl_cmd_tag(input); + break; + + case 'u': + res = ctrl_cmd_untag(input); + break; + + default: + res = -EINVAL; + goto err; + } + if (!res) + res = count; +err: + CT_DEBUG("qtaguid: ctrl(%s): res=%zd\n", input, res); + return res; +} + +#define MAX_QTAGUID_CTRL_INPUT_LEN 255 +static ssize_t qtaguid_ctrl_proc_write(struct file *file, const char __user *buffer, + size_t count, loff_t *offp) +{ + char input_buf[MAX_QTAGUID_CTRL_INPUT_LEN]; + + if (unlikely(module_passive)) + return count; + + if (count >= MAX_QTAGUID_CTRL_INPUT_LEN) + return -EINVAL; + + if (copy_from_user(input_buf, buffer, count)) + return -EFAULT; + + input_buf[count] = '\0'; + return qtaguid_ctrl_parse(input_buf, count); +} + +struct proc_print_info { + struct iface_stat *iface_entry; + int item_index; + tag_t tag; /* tag found by reading to tag_pos */ + off_t tag_pos; + int tag_item_index; +}; + +static void pp_stats_header(struct seq_file *m) +{ + seq_puts(m, + "idx iface acct_tag_hex uid_tag_int cnt_set " + "rx_bytes rx_packets " + "tx_bytes tx_packets " + "rx_tcp_bytes rx_tcp_packets " + "rx_udp_bytes rx_udp_packets " + "rx_other_bytes rx_other_packets " + "tx_tcp_bytes tx_tcp_packets " + "tx_udp_bytes tx_udp_packets " + "tx_other_bytes tx_other_packets\n"); +} + +static int pp_stats_line(struct seq_file *m, struct tag_stat *ts_entry, + int cnt_set) +{ + struct data_counters *cnts; + tag_t tag = ts_entry->tn.tag; + uid_t stat_uid = get_uid_from_tag(tag); + struct proc_print_info *ppi = m->private; + /* Detailed tags are not available to everybody */ + if (!can_read_other_uid_stats(make_kuid(&init_user_ns,stat_uid))) { + CT_DEBUG("qtaguid: stats line: " + "%s 0x%llx %u: insufficient priv " + "from pid=%u tgid=%u uid=%u stats.gid=%u\n", + ppi->iface_entry->ifname, + get_atag_from_tag(tag), stat_uid, + current->pid, current->tgid, from_kuid(&init_user_ns, current_fsuid()), + from_kgid(&init_user_ns,xt_qtaguid_stats_file->gid)); + return 0; + } + ppi->item_index++; + cnts = &ts_entry->counters; + seq_printf(m, "%d %s 0x%llx %u %u " + "%llu %llu " + "%llu %llu " + "%llu %llu " + "%llu %llu " + "%llu %llu " + "%llu %llu " + "%llu %llu " + "%llu %llu\n", + ppi->item_index, + ppi->iface_entry->ifname, + get_atag_from_tag(tag), + stat_uid, + cnt_set, + dc_sum_bytes(cnts, cnt_set, IFS_RX), + dc_sum_packets(cnts, cnt_set, IFS_RX), + dc_sum_bytes(cnts, cnt_set, IFS_TX), + dc_sum_packets(cnts, cnt_set, IFS_TX), + cnts->bpc[cnt_set][IFS_RX][IFS_TCP].bytes, + cnts->bpc[cnt_set][IFS_RX][IFS_TCP].packets, + cnts->bpc[cnt_set][IFS_RX][IFS_UDP].bytes, + cnts->bpc[cnt_set][IFS_RX][IFS_UDP].packets, + cnts->bpc[cnt_set][IFS_RX][IFS_PROTO_OTHER].bytes, + cnts->bpc[cnt_set][IFS_RX][IFS_PROTO_OTHER].packets, + cnts->bpc[cnt_set][IFS_TX][IFS_TCP].bytes, + cnts->bpc[cnt_set][IFS_TX][IFS_TCP].packets, + cnts->bpc[cnt_set][IFS_TX][IFS_UDP].bytes, + cnts->bpc[cnt_set][IFS_TX][IFS_UDP].packets, + cnts->bpc[cnt_set][IFS_TX][IFS_PROTO_OTHER].bytes, + cnts->bpc[cnt_set][IFS_TX][IFS_PROTO_OTHER].packets); + return seq_has_overflowed(m) ? -ENOSPC : 1; +} + +static bool pp_sets(struct seq_file *m, struct tag_stat *ts_entry) +{ + int ret; + int counter_set; + for (counter_set = 0; counter_set < IFS_MAX_COUNTER_SETS; + counter_set++) { + ret = pp_stats_line(m, ts_entry, counter_set); + if (ret < 0) + return false; + } + return true; +} + +static int qtaguid_stats_proc_iface_stat_ptr_valid(struct iface_stat *ptr) +{ + struct iface_stat *iface_entry; + + if (!ptr) + return false; + + list_for_each_entry(iface_entry, &iface_stat_list, list) + if (iface_entry == ptr) + return true; + return false; +} + +static void qtaguid_stats_proc_next_iface_entry(struct proc_print_info *ppi) +{ + spin_unlock_bh(&ppi->iface_entry->tag_stat_list_lock); + list_for_each_entry_continue(ppi->iface_entry, &iface_stat_list, list) { + spin_lock_bh(&ppi->iface_entry->tag_stat_list_lock); + return; + } + ppi->iface_entry = NULL; +} + +static void *qtaguid_stats_proc_next(struct seq_file *m, void *v, loff_t *pos) +{ + struct proc_print_info *ppi = m->private; + struct tag_stat *ts_entry; + struct rb_node *node; + + if (!v) { + pr_err("qtaguid: %s(): unexpected v: NULL\n", __func__); + return NULL; + } + + (*pos)++; + + if (!ppi->iface_entry || unlikely(module_passive)) + return NULL; + + if (v == SEQ_START_TOKEN) + node = rb_first(&ppi->iface_entry->tag_stat_tree); + else + node = rb_next(&((struct tag_stat *)v)->tn.node); + + while (!node) { + qtaguid_stats_proc_next_iface_entry(ppi); + if (!ppi->iface_entry) + return NULL; + node = rb_first(&ppi->iface_entry->tag_stat_tree); + } + + ts_entry = rb_entry(node, struct tag_stat, tn.node); + ppi->tag = ts_entry->tn.tag; + ppi->tag_pos = *pos; + ppi->tag_item_index = ppi->item_index; + return ts_entry; +} + +static void *qtaguid_stats_proc_start(struct seq_file *m, loff_t *pos) +{ + struct proc_print_info *ppi = m->private; + struct tag_stat *ts_entry = NULL; + + spin_lock_bh(&iface_stat_list_lock); + + if (*pos == 0) { + ppi->item_index = 1; + ppi->tag_pos = 0; + if (list_empty(&iface_stat_list)) { + ppi->iface_entry = NULL; + } else { + ppi->iface_entry = list_first_entry(&iface_stat_list, + struct iface_stat, + list); + spin_lock_bh(&ppi->iface_entry->tag_stat_list_lock); + } + return SEQ_START_TOKEN; + } + if (!qtaguid_stats_proc_iface_stat_ptr_valid(ppi->iface_entry)) { + if (ppi->iface_entry) { + pr_err("qtaguid: %s(): iface_entry %p not found\n", + __func__, ppi->iface_entry); + ppi->iface_entry = NULL; + } + return NULL; + } + + spin_lock_bh(&ppi->iface_entry->tag_stat_list_lock); + + if (!ppi->tag_pos) { + /* seq_read skipped first next call */ + ts_entry = SEQ_START_TOKEN; + } else { + ts_entry = tag_stat_tree_search( + &ppi->iface_entry->tag_stat_tree, ppi->tag); + if (!ts_entry) { + pr_info("qtaguid: %s(): tag_stat.tag 0x%llx not found. Abort.\n", + __func__, ppi->tag); + return NULL; + } + } + + if (*pos == ppi->tag_pos) { /* normal resume */ + ppi->item_index = ppi->tag_item_index; + } else { + /* seq_read skipped a next call */ + *pos = ppi->tag_pos; + ts_entry = qtaguid_stats_proc_next(m, ts_entry, pos); + } + + return ts_entry; +} + +static void qtaguid_stats_proc_stop(struct seq_file *m, void *v) +{ + struct proc_print_info *ppi = m->private; + if (ppi->iface_entry) + spin_unlock_bh(&ppi->iface_entry->tag_stat_list_lock); + spin_unlock_bh(&iface_stat_list_lock); +} + +/* + * Procfs reader to get all tag stats using style "1)" as described in + * fs/proc/generic.c + * Groups all protocols tx/rx bytes. + */ +static int qtaguid_stats_proc_show(struct seq_file *m, void *v) +{ + struct tag_stat *ts_entry = v; + + if (v == SEQ_START_TOKEN) + pp_stats_header(m); + else + pp_sets(m, ts_entry); + + return 0; +} + +/*------------------------------------------*/ +static int qtudev_open(struct inode *inode, struct file *file) +{ + struct uid_tag_data *utd_entry; + struct proc_qtu_data *pqd_entry; + struct proc_qtu_data *new_pqd_entry; + int res; + bool utd_entry_found; + + if (unlikely(qtu_proc_handling_passive)) + return 0; + + DR_DEBUG("qtaguid: qtudev_open(): pid=%u tgid=%u uid=%u\n", + current->pid, current->tgid, from_kuid(&init_user_ns, current_fsuid())); + + spin_lock_bh(&uid_tag_data_tree_lock); + + /* Look for existing uid data, or alloc one. */ + utd_entry = get_uid_data(from_kuid(&init_user_ns, current_fsuid()), &utd_entry_found); + if (IS_ERR_OR_NULL(utd_entry)) { + res = PTR_ERR(utd_entry); + goto err_unlock; + } + + /* Look for existing PID based proc_data */ + pqd_entry = proc_qtu_data_tree_search(&proc_qtu_data_tree, + current->tgid); + if (pqd_entry) { + pr_err("qtaguid: qtudev_open(): %u/%u %u " + "%s already opened\n", + current->pid, current->tgid, from_kuid(&init_user_ns, current_fsuid()), + QTU_DEV_NAME); + res = -EBUSY; + goto err_unlock_free_utd; + } + + new_pqd_entry = kzalloc(sizeof(*new_pqd_entry), GFP_ATOMIC); + if (!new_pqd_entry) { + pr_err("qtaguid: qtudev_open(): %u/%u %u: " + "proc data alloc failed\n", + current->pid, current->tgid, from_kuid(&init_user_ns, current_fsuid())); + res = -ENOMEM; + goto err_unlock_free_utd; + } + new_pqd_entry->pid = current->tgid; + INIT_LIST_HEAD(&new_pqd_entry->sock_tag_list); + new_pqd_entry->parent_tag_data = utd_entry; + utd_entry->num_pqd++; + + proc_qtu_data_tree_insert(new_pqd_entry, + &proc_qtu_data_tree); + + spin_unlock_bh(&uid_tag_data_tree_lock); + DR_DEBUG("qtaguid: tracking data for uid=%u in pqd=%p\n", + from_kuid(&init_user_ns, current_fsuid()), new_pqd_entry); + file->private_data = new_pqd_entry; + return 0; + +err_unlock_free_utd: + if (!utd_entry_found) { + rb_erase(&utd_entry->node, &uid_tag_data_tree); + kfree(utd_entry); + } +err_unlock: + spin_unlock_bh(&uid_tag_data_tree_lock); + return res; +} + +static int qtudev_release(struct inode *inode, struct file *file) +{ + struct proc_qtu_data *pqd_entry = file->private_data; + struct uid_tag_data *utd_entry = pqd_entry->parent_tag_data; + struct sock_tag *st_entry; + struct rb_root st_to_free_tree = RB_ROOT; + struct list_head *entry, *next; + struct tag_ref *tr; + + if (unlikely(qtu_proc_handling_passive)) + return 0; + + /* + * Do not trust the current->pid, it might just be a kworker cleaning + * up after a dead proc. + */ + DR_DEBUG("qtaguid: qtudev_release(): " + "pid=%u tgid=%u uid=%u " + "pqd_entry=%p->pid=%u utd_entry=%p->active_tags=%d\n", + current->pid, current->tgid, pqd_entry->parent_tag_data->uid, + pqd_entry, pqd_entry->pid, utd_entry, + utd_entry->num_active_tags); + + spin_lock_bh(&sock_tag_list_lock); + spin_lock_bh(&uid_tag_data_tree_lock); + + list_for_each_safe(entry, next, &pqd_entry->sock_tag_list) { + st_entry = list_entry(entry, struct sock_tag, list); + DR_DEBUG("qtaguid: %s(): " + "erase sock_tag=%p->sk=%p pid=%u tgid=%u uid=%u\n", + __func__, + st_entry, st_entry->sk, + current->pid, current->tgid, + pqd_entry->parent_tag_data->uid); + + utd_entry = uid_tag_data_tree_search( + &uid_tag_data_tree, + get_uid_from_tag(st_entry->tag)); + BUG_ON(IS_ERR_OR_NULL(utd_entry)); + DR_DEBUG("qtaguid: %s(): " + "looking for tag=0x%llx in utd_entry=%p\n", __func__, + st_entry->tag, utd_entry); + tr = tag_ref_tree_search(&utd_entry->tag_ref_tree, + st_entry->tag); + BUG_ON(!tr); + BUG_ON(tr->num_sock_tags <= 0); + tr->num_sock_tags--; + free_tag_ref_from_utd_entry(tr, utd_entry); + + rb_erase(&st_entry->sock_node, &sock_tag_tree); + list_del(&st_entry->list); + /* Can't sockfd_put() within spinlock, do it later. */ + sock_tag_tree_insert(st_entry, &st_to_free_tree); + + /* + * Try to free the utd_entry if no other proc_qtu_data is + * using it (num_pqd is 0) and it doesn't have active tags + * (num_active_tags is 0). + */ + put_utd_entry(utd_entry); + } + + rb_erase(&pqd_entry->node, &proc_qtu_data_tree); + BUG_ON(pqd_entry->parent_tag_data->num_pqd < 1); + pqd_entry->parent_tag_data->num_pqd--; + put_utd_entry(pqd_entry->parent_tag_data); + kfree(pqd_entry); + file->private_data = NULL; + + spin_unlock_bh(&uid_tag_data_tree_lock); + spin_unlock_bh(&sock_tag_list_lock); + + + sock_tag_tree_erase(&st_to_free_tree); + + prdebug_full_state(0, "%s(): pid=%u tgid=%u", __func__, + current->pid, current->tgid); + return 0; +} + +/*------------------------------------------*/ +static const struct file_operations qtudev_fops = { + .owner = THIS_MODULE, + .open = qtudev_open, + .release = qtudev_release, +}; + +static struct miscdevice qtu_device = { + .minor = MISC_DYNAMIC_MINOR, + .name = QTU_DEV_NAME, + .fops = &qtudev_fops, + /* How sad it doesn't allow for defaults: .mode = S_IRUGO | S_IWUSR */ +}; + +static const struct seq_operations proc_qtaguid_ctrl_seqops = { + .start = qtaguid_ctrl_proc_start, + .next = qtaguid_ctrl_proc_next, + .stop = qtaguid_ctrl_proc_stop, + .show = qtaguid_ctrl_proc_show, +}; + +static int proc_qtaguid_ctrl_open(struct inode *inode, struct file *file) +{ + return seq_open_private(file, &proc_qtaguid_ctrl_seqops, + sizeof(struct proc_ctrl_print_info)); +} + +static const struct file_operations proc_qtaguid_ctrl_fops = { + .open = proc_qtaguid_ctrl_open, + .read = seq_read, + .write = qtaguid_ctrl_proc_write, + .llseek = seq_lseek, + .release = seq_release_private, +}; + +static const struct seq_operations proc_qtaguid_stats_seqops = { + .start = qtaguid_stats_proc_start, + .next = qtaguid_stats_proc_next, + .stop = qtaguid_stats_proc_stop, + .show = qtaguid_stats_proc_show, +}; + +static int proc_qtaguid_stats_open(struct inode *inode, struct file *file) +{ + return seq_open_private(file, &proc_qtaguid_stats_seqops, + sizeof(struct proc_print_info)); +} + +static const struct file_operations proc_qtaguid_stats_fops = { + .open = proc_qtaguid_stats_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release_private, +}; + +/*------------------------------------------*/ +static int __init qtaguid_proc_register(struct proc_dir_entry **res_procdir) +{ + int ret; + *res_procdir = proc_mkdir(module_procdirname, init_net.proc_net); + if (!*res_procdir) { + pr_err("qtaguid: failed to create proc/.../xt_qtaguid\n"); + ret = -ENOMEM; + goto no_dir; + } + + xt_qtaguid_ctrl_file = proc_create_data("ctrl", proc_ctrl_perms, + *res_procdir, + &proc_qtaguid_ctrl_fops, + NULL); + if (!xt_qtaguid_ctrl_file) { + pr_err("qtaguid: failed to create xt_qtaguid/ctrl " + " file\n"); + ret = -ENOMEM; + goto no_ctrl_entry; + } + + xt_qtaguid_stats_file = proc_create_data("stats", proc_stats_perms, + *res_procdir, + &proc_qtaguid_stats_fops, + NULL); + if (!xt_qtaguid_stats_file) { + pr_err("qtaguid: failed to create xt_qtaguid/stats " + "file\n"); + ret = -ENOMEM; + goto no_stats_entry; + } + /* + * TODO: add support counter hacking + * xt_qtaguid_stats_file->write_proc = qtaguid_stats_proc_write; + */ + return 0; + +no_stats_entry: + remove_proc_entry("ctrl", *res_procdir); +no_ctrl_entry: + remove_proc_entry("xt_qtaguid", NULL); +no_dir: + return ret; +} + +static struct xt_match qtaguid_mt_reg __read_mostly = { + /* + * This module masquerades as the "owner" module so that iptables + * tools can deal with it. + */ + .name = "owner", + .revision = 1, + .family = NFPROTO_UNSPEC, + .match = qtaguid_mt, + .matchsize = sizeof(struct xt_qtaguid_match_info), + .me = THIS_MODULE, +}; + +static int __init qtaguid_mt_init(void) +{ + if (qtaguid_proc_register(&xt_qtaguid_procdir) + || iface_stat_init(xt_qtaguid_procdir) + || xt_register_match(&qtaguid_mt_reg) + || misc_register(&qtu_device)) + return -1; + return 0; +} + +/* + * TODO: allow unloading of the module. + * For now stats are permanent. + * Kconfig forces'y/n' and never an 'm'. + */ + +module_init(qtaguid_mt_init); +MODULE_AUTHOR("jpa "); +MODULE_DESCRIPTION("Xtables: socket owner+tag matching and associated stats"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("ipt_owner"); +MODULE_ALIAS("ip6t_owner"); +MODULE_ALIAS("ipt_qtaguid"); +MODULE_ALIAS("ip6t_qtaguid"); diff --git a/net/netfilter/xt_qtaguid_internal.h b/net/netfilter/xt_qtaguid_internal.h new file mode 100644 index 00000000000000..6dc14a9c688966 --- /dev/null +++ b/net/netfilter/xt_qtaguid_internal.h @@ -0,0 +1,352 @@ +/* + * Kernel iptables module to track stats for packets based on user tags. + * + * (C) 2011 Google, Inc + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#ifndef __XT_QTAGUID_INTERNAL_H__ +#define __XT_QTAGUID_INTERNAL_H__ + +#include +#include +#include +#include + +/* Iface handling */ +#define IDEBUG_MASK (1<<0) +/* Iptable Matching. Per packet. */ +#define MDEBUG_MASK (1<<1) +/* Red-black tree handling. Per packet. */ +#define RDEBUG_MASK (1<<2) +/* procfs ctrl/stats handling */ +#define CDEBUG_MASK (1<<3) +/* dev and resource tracking */ +#define DDEBUG_MASK (1<<4) + +/* E.g (IDEBUG_MASK | CDEBUG_MASK | DDEBUG_MASK) */ +#define DEFAULT_DEBUG_MASK 0 + +/* + * (Un)Define these *DEBUG to compile out/in the pr_debug calls. + * All undef: text size ~ 0x3030; all def: ~ 0x4404. + */ +#define IDEBUG +#define MDEBUG +#define RDEBUG +#define CDEBUG +#define DDEBUG + +#define MSK_DEBUG(mask, ...) do { \ + if (unlikely(qtaguid_debug_mask & (mask))) \ + pr_debug(__VA_ARGS__); \ + } while (0) +#ifdef IDEBUG +#define IF_DEBUG(...) MSK_DEBUG(IDEBUG_MASK, __VA_ARGS__) +#else +#define IF_DEBUG(...) no_printk(__VA_ARGS__) +#endif +#ifdef MDEBUG +#define MT_DEBUG(...) MSK_DEBUG(MDEBUG_MASK, __VA_ARGS__) +#else +#define MT_DEBUG(...) no_printk(__VA_ARGS__) +#endif +#ifdef RDEBUG +#define RB_DEBUG(...) MSK_DEBUG(RDEBUG_MASK, __VA_ARGS__) +#else +#define RB_DEBUG(...) no_printk(__VA_ARGS__) +#endif +#ifdef CDEBUG +#define CT_DEBUG(...) MSK_DEBUG(CDEBUG_MASK, __VA_ARGS__) +#else +#define CT_DEBUG(...) no_printk(__VA_ARGS__) +#endif +#ifdef DDEBUG +#define DR_DEBUG(...) MSK_DEBUG(DDEBUG_MASK, __VA_ARGS__) +#else +#define DR_DEBUG(...) no_printk(__VA_ARGS__) +#endif + +extern uint qtaguid_debug_mask; + +/*---------------------------------------------------------------------------*/ +/* + * Tags: + * + * They represent what the data usage counters will be tracked against. + * By default a tag is just based on the UID. + * The UID is used as the base for policing, and can not be ignored. + * So a tag will always at least represent a UID (uid_tag). + * + * A tag can be augmented with an "accounting tag" which is associated + * with a UID. + * User space can set the acct_tag portion of the tag which is then used + * with sockets: all data belonging to that socket will be counted against the + * tag. The policing is then based on the tag's uid_tag portion, + * and stats are collected for the acct_tag portion separately. + * + * There could be + * a: {acct_tag=1, uid_tag=10003} + * b: {acct_tag=2, uid_tag=10003} + * c: {acct_tag=3, uid_tag=10003} + * d: {acct_tag=0, uid_tag=10003} + * a, b, and c represent tags associated with specific sockets. + * d is for the totals for that uid, including all untagged traffic. + * Typically d is used with policing/quota rules. + * + * We want tag_t big enough to distinguish uid_t and acct_tag. + * It might become a struct if needed. + * Nothing should be using it as an int. + */ +typedef uint64_t tag_t; /* Only used via accessors */ + +#define TAG_UID_MASK 0xFFFFFFFFULL +#define TAG_ACCT_MASK (~0xFFFFFFFFULL) + +static inline int tag_compare(tag_t t1, tag_t t2) +{ + return t1 < t2 ? -1 : t1 == t2 ? 0 : 1; +} + +static inline tag_t combine_atag_with_uid(tag_t acct_tag, uid_t uid) +{ + return acct_tag | uid; +} +static inline tag_t make_tag_from_uid(uid_t uid) +{ + return uid; +} +static inline uid_t get_uid_from_tag(tag_t tag) +{ + return tag & TAG_UID_MASK; +} +static inline tag_t get_utag_from_tag(tag_t tag) +{ + return tag & TAG_UID_MASK; +} +static inline tag_t get_atag_from_tag(tag_t tag) +{ + return tag & TAG_ACCT_MASK; +} + +static inline bool valid_atag(tag_t tag) +{ + return !(tag & TAG_UID_MASK); +} +static inline tag_t make_atag_from_value(uint32_t value) +{ + return (uint64_t)value << 32; +} +/*---------------------------------------------------------------------------*/ + +/* + * Maximum number of socket tags that a UID is allowed to have active. + * Multiple processes belonging to the same UID contribute towards this limit. + * Special UIDs that can impersonate a UID also contribute (e.g. download + * manager, ...) + */ +#define DEFAULT_MAX_SOCK_TAGS 1024 + +/* + * For now we only track 2 sets of counters. + * The default set is 0. + * Userspace can activate another set for a given uid being tracked. + */ +#define IFS_MAX_COUNTER_SETS 2 + +enum ifs_tx_rx { + IFS_TX, + IFS_RX, + IFS_MAX_DIRECTIONS +}; + +/* For now, TCP, UDP, the rest */ +enum ifs_proto { + IFS_TCP, + IFS_UDP, + IFS_PROTO_OTHER, + IFS_MAX_PROTOS +}; + +struct byte_packet_counters { + uint64_t bytes; + uint64_t packets; +}; + +struct data_counters { + struct byte_packet_counters bpc[IFS_MAX_COUNTER_SETS][IFS_MAX_DIRECTIONS][IFS_MAX_PROTOS]; +}; + +static inline uint64_t dc_sum_bytes(struct data_counters *counters, + int set, + enum ifs_tx_rx direction) +{ + return counters->bpc[set][direction][IFS_TCP].bytes + + counters->bpc[set][direction][IFS_UDP].bytes + + counters->bpc[set][direction][IFS_PROTO_OTHER].bytes; +} + +static inline uint64_t dc_sum_packets(struct data_counters *counters, + int set, + enum ifs_tx_rx direction) +{ + return counters->bpc[set][direction][IFS_TCP].packets + + counters->bpc[set][direction][IFS_UDP].packets + + counters->bpc[set][direction][IFS_PROTO_OTHER].packets; +} + + +/* Generic X based nodes used as a base for rb_tree ops */ +struct tag_node { + struct rb_node node; + tag_t tag; +}; + +struct tag_stat { + struct tag_node tn; + struct data_counters counters; + /* + * If this tag is acct_tag based, we need to count against the + * matching parent uid_tag. + */ + struct data_counters *parent_counters; +}; + +struct iface_stat { + struct list_head list; /* in iface_stat_list */ + char *ifname; + bool active; + /* net_dev is only valid for active iface_stat */ + struct net_device *net_dev; + + struct byte_packet_counters totals_via_dev[IFS_MAX_DIRECTIONS]; + struct data_counters totals_via_skb; + /* + * We keep the last_known, because some devices reset their counters + * just before NETDEV_UP, while some will reset just before + * NETDEV_REGISTER (which is more normal). + * So now, if the device didn't do a NETDEV_UNREGISTER and we see + * its current dev stats smaller that what was previously known, we + * assume an UNREGISTER and just use the last_known. + */ + struct byte_packet_counters last_known[IFS_MAX_DIRECTIONS]; + /* last_known is usable when last_known_valid is true */ + bool last_known_valid; + + struct proc_dir_entry *proc_ptr; + + struct rb_root tag_stat_tree; + spinlock_t tag_stat_list_lock; +}; + +/* This is needed to create proc_dir_entries from atomic context. */ +struct iface_stat_work { + struct work_struct iface_work; + struct iface_stat *iface_entry; +}; + +/* + * Track tag that this socket is transferring data for, and not necessarily + * the uid that owns the socket. + * This is the tag against which tag_stat.counters will be billed. + * These structs need to be looked up by sock and pid. + */ +struct sock_tag { + struct rb_node sock_node; + struct sock *sk; /* Only used as a number, never dereferenced */ + /* The socket is needed for sockfd_put() */ + struct socket *socket; + /* Used to associate with a given pid */ + struct list_head list; /* in proc_qtu_data.sock_tag_list */ + pid_t pid; + + tag_t tag; +}; + +struct qtaguid_event_counts { + /* Various successful events */ + atomic64_t sockets_tagged; + atomic64_t sockets_untagged; + atomic64_t counter_set_changes; + atomic64_t delete_cmds; + atomic64_t iface_events; /* Number of NETDEV_* events handled */ + + atomic64_t match_calls; /* Number of times iptables called mt */ + /* Number of times iptables called mt from pre or post routing hooks */ + atomic64_t match_calls_prepost; + /* + * match_found_sk_*: numbers related to the netfilter matching + * function finding a sock for the sk_buff. + * Total skbs processed is sum(match_found*). + */ + atomic64_t match_found_sk; /* An sk was already in the sk_buff. */ + /* The connection tracker had or didn't have the sk. */ + atomic64_t match_found_sk_in_ct; + atomic64_t match_found_no_sk_in_ct; + /* + * No sk could be found. No apparent owner. Could happen with + * unsolicited traffic. + */ + atomic64_t match_no_sk; + /* + * The file ptr in the sk_socket wasn't there. + * This might happen for traffic while the socket is being closed. + */ + atomic64_t match_no_sk_file; +}; + +/* Track the set active_set for the given tag. */ +struct tag_counter_set { + struct tag_node tn; + int active_set; +}; + +/*----------------------------------------------*/ +/* + * The qtu uid data is used to track resources that are created directly or + * indirectly by processes (uid tracked). + * It is shared by the processes with the same uid. + * Some of the resource will be counted to prevent further rogue allocations, + * some will need freeing once the owner process (uid) exits. + */ +struct uid_tag_data { + struct rb_node node; + uid_t uid; + + /* + * For the uid, how many accounting tags have been set. + */ + int num_active_tags; + /* Track the number of proc_qtu_data that reference it */ + int num_pqd; + struct rb_root tag_ref_tree; + /* No tag_node_tree_lock; use uid_tag_data_tree_lock */ +}; + +struct tag_ref { + struct tag_node tn; + + /* + * This tracks the number of active sockets that have a tag on them + * which matches this tag_ref.tn.tag. + * A tag ref can live on after the sockets are untagged. + * A tag ref can only be removed during a tag delete command. + */ + int num_sock_tags; +}; + +struct proc_qtu_data { + struct rb_node node; + pid_t pid; + + struct uid_tag_data *parent_tag_data; + + /* Tracks the sock_tags that need freeing upon this proc's death */ + struct list_head sock_tag_list; + /* No spinlock_t sock_tag_list_lock; use the global one. */ +}; + +/*----------------------------------------------*/ +#endif /* ifndef __XT_QTAGUID_INTERNAL_H__ */ diff --git a/net/netfilter/xt_qtaguid_print.c b/net/netfilter/xt_qtaguid_print.c new file mode 100644 index 00000000000000..f6a00a3520ed57 --- /dev/null +++ b/net/netfilter/xt_qtaguid_print.c @@ -0,0 +1,566 @@ +/* + * Pretty printing Support for iptables xt_qtaguid module. + * + * (C) 2011 Google, Inc + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +/* + * Most of the functions in this file just waste time if DEBUG is not defined. + * The matching xt_qtaguid_print.h will static inline empty funcs if the needed + * debug flags ore not defined. + * Those funcs that fail to allocate memory will panic as there is no need to + * hobble allong just pretending to do the requested work. + */ + +#define DEBUG + +#include +#include +#include +#include +#include +#include + + +#include "xt_qtaguid_internal.h" +#include "xt_qtaguid_print.h" + +#ifdef DDEBUG + +static void _bug_on_err_or_null(void *ptr) +{ + if (IS_ERR_OR_NULL(ptr)) { + pr_err("qtaguid: kmalloc failed\n"); + BUG(); + } +} + +char *pp_tag_t(tag_t *tag) +{ + char *res; + + if (!tag) + res = kasprintf(GFP_ATOMIC, "tag_t@null{}"); + else + res = kasprintf(GFP_ATOMIC, + "tag_t@%p{tag=0x%llx, uid=%u}", + tag, *tag, get_uid_from_tag(*tag)); + _bug_on_err_or_null(res); + return res; +} + +char *pp_data_counters(struct data_counters *dc, bool showValues) +{ + char *res; + + if (!dc) + res = kasprintf(GFP_ATOMIC, "data_counters@null{}"); + else if (showValues) + res = kasprintf( + GFP_ATOMIC, "data_counters@%p{" + "set0{" + "rx{" + "tcp{b=%llu, p=%llu}, " + "udp{b=%llu, p=%llu}," + "other{b=%llu, p=%llu}}, " + "tx{" + "tcp{b=%llu, p=%llu}, " + "udp{b=%llu, p=%llu}," + "other{b=%llu, p=%llu}}}, " + "set1{" + "rx{" + "tcp{b=%llu, p=%llu}, " + "udp{b=%llu, p=%llu}," + "other{b=%llu, p=%llu}}, " + "tx{" + "tcp{b=%llu, p=%llu}, " + "udp{b=%llu, p=%llu}," + "other{b=%llu, p=%llu}}}}", + dc, + dc->bpc[0][IFS_RX][IFS_TCP].bytes, + dc->bpc[0][IFS_RX][IFS_TCP].packets, + dc->bpc[0][IFS_RX][IFS_UDP].bytes, + dc->bpc[0][IFS_RX][IFS_UDP].packets, + dc->bpc[0][IFS_RX][IFS_PROTO_OTHER].bytes, + dc->bpc[0][IFS_RX][IFS_PROTO_OTHER].packets, + dc->bpc[0][IFS_TX][IFS_TCP].bytes, + dc->bpc[0][IFS_TX][IFS_TCP].packets, + dc->bpc[0][IFS_TX][IFS_UDP].bytes, + dc->bpc[0][IFS_TX][IFS_UDP].packets, + dc->bpc[0][IFS_TX][IFS_PROTO_OTHER].bytes, + dc->bpc[0][IFS_TX][IFS_PROTO_OTHER].packets, + dc->bpc[1][IFS_RX][IFS_TCP].bytes, + dc->bpc[1][IFS_RX][IFS_TCP].packets, + dc->bpc[1][IFS_RX][IFS_UDP].bytes, + dc->bpc[1][IFS_RX][IFS_UDP].packets, + dc->bpc[1][IFS_RX][IFS_PROTO_OTHER].bytes, + dc->bpc[1][IFS_RX][IFS_PROTO_OTHER].packets, + dc->bpc[1][IFS_TX][IFS_TCP].bytes, + dc->bpc[1][IFS_TX][IFS_TCP].packets, + dc->bpc[1][IFS_TX][IFS_UDP].bytes, + dc->bpc[1][IFS_TX][IFS_UDP].packets, + dc->bpc[1][IFS_TX][IFS_PROTO_OTHER].bytes, + dc->bpc[1][IFS_TX][IFS_PROTO_OTHER].packets); + else + res = kasprintf(GFP_ATOMIC, "data_counters@%p{...}", dc); + _bug_on_err_or_null(res); + return res; +} + +char *pp_tag_node(struct tag_node *tn) +{ + char *tag_str; + char *res; + + if (!tn) { + res = kasprintf(GFP_ATOMIC, "tag_node@null{}"); + _bug_on_err_or_null(res); + return res; + } + tag_str = pp_tag_t(&tn->tag); + res = kasprintf(GFP_ATOMIC, + "tag_node@%p{tag=%s}", + tn, tag_str); + _bug_on_err_or_null(res); + kfree(tag_str); + return res; +} + +char *pp_tag_ref(struct tag_ref *tr) +{ + char *tn_str; + char *res; + + if (!tr) { + res = kasprintf(GFP_ATOMIC, "tag_ref@null{}"); + _bug_on_err_or_null(res); + return res; + } + tn_str = pp_tag_node(&tr->tn); + res = kasprintf(GFP_ATOMIC, + "tag_ref@%p{%s, num_sock_tags=%d}", + tr, tn_str, tr->num_sock_tags); + _bug_on_err_or_null(res); + kfree(tn_str); + return res; +} + +char *pp_tag_stat(struct tag_stat *ts) +{ + char *tn_str; + char *counters_str; + char *parent_counters_str; + char *res; + + if (!ts) { + res = kasprintf(GFP_ATOMIC, "tag_stat@null{}"); + _bug_on_err_or_null(res); + return res; + } + tn_str = pp_tag_node(&ts->tn); + counters_str = pp_data_counters(&ts->counters, true); + parent_counters_str = pp_data_counters(ts->parent_counters, false); + res = kasprintf(GFP_ATOMIC, + "tag_stat@%p{%s, counters=%s, parent_counters=%s}", + ts, tn_str, counters_str, parent_counters_str); + _bug_on_err_or_null(res); + kfree(tn_str); + kfree(counters_str); + kfree(parent_counters_str); + return res; +} + +char *pp_iface_stat(struct iface_stat *is) +{ + char *res; + if (!is) { + res = kasprintf(GFP_ATOMIC, "iface_stat@null{}"); + } else { + struct data_counters *cnts = &is->totals_via_skb; + res = kasprintf(GFP_ATOMIC, "iface_stat@%p{" + "list=list_head{...}, " + "ifname=%s, " + "total_dev={rx={bytes=%llu, " + "packets=%llu}, " + "tx={bytes=%llu, " + "packets=%llu}}, " + "total_skb={rx={bytes=%llu, " + "packets=%llu}, " + "tx={bytes=%llu, " + "packets=%llu}}, " + "last_known_valid=%d, " + "last_known={rx={bytes=%llu, " + "packets=%llu}, " + "tx={bytes=%llu, " + "packets=%llu}}, " + "active=%d, " + "net_dev=%p, " + "proc_ptr=%p, " + "tag_stat_tree=rb_root{...}}", + is, + is->ifname, + is->totals_via_dev[IFS_RX].bytes, + is->totals_via_dev[IFS_RX].packets, + is->totals_via_dev[IFS_TX].bytes, + is->totals_via_dev[IFS_TX].packets, + dc_sum_bytes(cnts, 0, IFS_RX), + dc_sum_packets(cnts, 0, IFS_RX), + dc_sum_bytes(cnts, 0, IFS_TX), + dc_sum_packets(cnts, 0, IFS_TX), + is->last_known_valid, + is->last_known[IFS_RX].bytes, + is->last_known[IFS_RX].packets, + is->last_known[IFS_TX].bytes, + is->last_known[IFS_TX].packets, + is->active, + is->net_dev, + is->proc_ptr); + } + _bug_on_err_or_null(res); + return res; +} + +char *pp_sock_tag(struct sock_tag *st) +{ + char *tag_str; + char *res; + + if (!st) { + res = kasprintf(GFP_ATOMIC, "sock_tag@null{}"); + _bug_on_err_or_null(res); + return res; + } + tag_str = pp_tag_t(&st->tag); + res = kasprintf(GFP_ATOMIC, "sock_tag@%p{" + "sock_node=rb_node{...}, " + "sk=%p socket=%p (f_count=%lu), list=list_head{...}, " + "pid=%u, tag=%s}", + st, st->sk, st->socket, atomic_long_read( + &st->socket->file->f_count), + st->pid, tag_str); + _bug_on_err_or_null(res); + kfree(tag_str); + return res; +} + +char *pp_uid_tag_data(struct uid_tag_data *utd) +{ + char *res; + + if (!utd) + res = kasprintf(GFP_ATOMIC, "uid_tag_data@null{}"); + else + res = kasprintf(GFP_ATOMIC, "uid_tag_data@%p{" + "uid=%u, num_active_acct_tags=%d, " + "num_pqd=%d, " + "tag_node_tree=rb_root{...}, " + "proc_qtu_data_tree=rb_root{...}}", + utd, utd->uid, + utd->num_active_tags, utd->num_pqd); + _bug_on_err_or_null(res); + return res; +} + +char *pp_proc_qtu_data(struct proc_qtu_data *pqd) +{ + char *parent_tag_data_str; + char *res; + + if (!pqd) { + res = kasprintf(GFP_ATOMIC, "proc_qtu_data@null{}"); + _bug_on_err_or_null(res); + return res; + } + parent_tag_data_str = pp_uid_tag_data(pqd->parent_tag_data); + res = kasprintf(GFP_ATOMIC, "proc_qtu_data@%p{" + "node=rb_node{...}, pid=%u, " + "parent_tag_data=%s, " + "sock_tag_list=list_head{...}}", + pqd, pqd->pid, parent_tag_data_str + ); + _bug_on_err_or_null(res); + kfree(parent_tag_data_str); + return res; +} + +/*------------------------------------------*/ +void prdebug_sock_tag_tree(int indent_level, + struct rb_root *sock_tag_tree) +{ + struct rb_node *node; + struct sock_tag *sock_tag_entry; + char *str; + + if (!unlikely(qtaguid_debug_mask & DDEBUG_MASK)) + return; + + if (RB_EMPTY_ROOT(sock_tag_tree)) { + str = "sock_tag_tree=rb_root{}"; + pr_debug("%*d: %s\n", indent_level*2, indent_level, str); + return; + } + + str = "sock_tag_tree=rb_root{"; + pr_debug("%*d: %s\n", indent_level*2, indent_level, str); + indent_level++; + for (node = rb_first(sock_tag_tree); + node; + node = rb_next(node)) { + sock_tag_entry = rb_entry(node, struct sock_tag, sock_node); + str = pp_sock_tag(sock_tag_entry); + pr_debug("%*d: %s,\n", indent_level*2, indent_level, str); + kfree(str); + } + indent_level--; + str = "}"; + pr_debug("%*d: %s\n", indent_level*2, indent_level, str); +} + +void prdebug_sock_tag_list(int indent_level, + struct list_head *sock_tag_list) +{ + struct sock_tag *sock_tag_entry; + char *str; + + if (!unlikely(qtaguid_debug_mask & DDEBUG_MASK)) + return; + + if (list_empty(sock_tag_list)) { + str = "sock_tag_list=list_head{}"; + pr_debug("%*d: %s\n", indent_level*2, indent_level, str); + return; + } + + str = "sock_tag_list=list_head{"; + pr_debug("%*d: %s\n", indent_level*2, indent_level, str); + indent_level++; + list_for_each_entry(sock_tag_entry, sock_tag_list, list) { + str = pp_sock_tag(sock_tag_entry); + pr_debug("%*d: %s,\n", indent_level*2, indent_level, str); + kfree(str); + } + indent_level--; + str = "}"; + pr_debug("%*d: %s\n", indent_level*2, indent_level, str); +} + +void prdebug_proc_qtu_data_tree(int indent_level, + struct rb_root *proc_qtu_data_tree) +{ + char *str; + struct rb_node *node; + struct proc_qtu_data *proc_qtu_data_entry; + + if (!unlikely(qtaguid_debug_mask & DDEBUG_MASK)) + return; + + if (RB_EMPTY_ROOT(proc_qtu_data_tree)) { + str = "proc_qtu_data_tree=rb_root{}"; + pr_debug("%*d: %s\n", indent_level*2, indent_level, str); + return; + } + + str = "proc_qtu_data_tree=rb_root{"; + pr_debug("%*d: %s\n", indent_level*2, indent_level, str); + indent_level++; + for (node = rb_first(proc_qtu_data_tree); + node; + node = rb_next(node)) { + proc_qtu_data_entry = rb_entry(node, + struct proc_qtu_data, + node); + str = pp_proc_qtu_data(proc_qtu_data_entry); + pr_debug("%*d: %s,\n", indent_level*2, indent_level, + str); + kfree(str); + indent_level++; + prdebug_sock_tag_list(indent_level, + &proc_qtu_data_entry->sock_tag_list); + indent_level--; + + } + indent_level--; + str = "}"; + pr_debug("%*d: %s\n", indent_level*2, indent_level, str); +} + +void prdebug_tag_ref_tree(int indent_level, struct rb_root *tag_ref_tree) +{ + char *str; + struct rb_node *node; + struct tag_ref *tag_ref_entry; + + if (!unlikely(qtaguid_debug_mask & DDEBUG_MASK)) + return; + + if (RB_EMPTY_ROOT(tag_ref_tree)) { + str = "tag_ref_tree{}"; + pr_debug("%*d: %s\n", indent_level*2, indent_level, str); + return; + } + + str = "tag_ref_tree{"; + pr_debug("%*d: %s\n", indent_level*2, indent_level, str); + indent_level++; + for (node = rb_first(tag_ref_tree); + node; + node = rb_next(node)) { + tag_ref_entry = rb_entry(node, + struct tag_ref, + tn.node); + str = pp_tag_ref(tag_ref_entry); + pr_debug("%*d: %s,\n", indent_level*2, indent_level, + str); + kfree(str); + } + indent_level--; + str = "}"; + pr_debug("%*d: %s\n", indent_level*2, indent_level, str); +} + +void prdebug_uid_tag_data_tree(int indent_level, + struct rb_root *uid_tag_data_tree) +{ + char *str; + struct rb_node *node; + struct uid_tag_data *uid_tag_data_entry; + + if (!unlikely(qtaguid_debug_mask & DDEBUG_MASK)) + return; + + if (RB_EMPTY_ROOT(uid_tag_data_tree)) { + str = "uid_tag_data_tree=rb_root{}"; + pr_debug("%*d: %s\n", indent_level*2, indent_level, str); + return; + } + + str = "uid_tag_data_tree=rb_root{"; + pr_debug("%*d: %s\n", indent_level*2, indent_level, str); + indent_level++; + for (node = rb_first(uid_tag_data_tree); + node; + node = rb_next(node)) { + uid_tag_data_entry = rb_entry(node, struct uid_tag_data, + node); + str = pp_uid_tag_data(uid_tag_data_entry); + pr_debug("%*d: %s,\n", indent_level*2, indent_level, str); + kfree(str); + if (!RB_EMPTY_ROOT(&uid_tag_data_entry->tag_ref_tree)) { + indent_level++; + prdebug_tag_ref_tree(indent_level, + &uid_tag_data_entry->tag_ref_tree); + indent_level--; + } + } + indent_level--; + str = "}"; + pr_debug("%*d: %s\n", indent_level*2, indent_level, str); +} + +void prdebug_tag_stat_tree(int indent_level, + struct rb_root *tag_stat_tree) +{ + char *str; + struct rb_node *node; + struct tag_stat *ts_entry; + + if (!unlikely(qtaguid_debug_mask & DDEBUG_MASK)) + return; + + if (RB_EMPTY_ROOT(tag_stat_tree)) { + str = "tag_stat_tree{}"; + pr_debug("%*d: %s\n", indent_level*2, indent_level, str); + return; + } + + str = "tag_stat_tree{"; + pr_debug("%*d: %s\n", indent_level*2, indent_level, str); + indent_level++; + for (node = rb_first(tag_stat_tree); + node; + node = rb_next(node)) { + ts_entry = rb_entry(node, struct tag_stat, tn.node); + str = pp_tag_stat(ts_entry); + pr_debug("%*d: %s\n", indent_level*2, indent_level, + str); + kfree(str); + } + indent_level--; + str = "}"; + pr_debug("%*d: %s\n", indent_level*2, indent_level, str); +} + +void prdebug_iface_stat_list(int indent_level, + struct list_head *iface_stat_list) +{ + char *str; + struct iface_stat *iface_entry; + + if (!unlikely(qtaguid_debug_mask & DDEBUG_MASK)) + return; + + if (list_empty(iface_stat_list)) { + str = "iface_stat_list=list_head{}"; + pr_debug("%*d: %s\n", indent_level*2, indent_level, str); + return; + } + + str = "iface_stat_list=list_head{"; + pr_debug("%*d: %s\n", indent_level*2, indent_level, str); + indent_level++; + list_for_each_entry(iface_entry, iface_stat_list, list) { + str = pp_iface_stat(iface_entry); + pr_debug("%*d: %s\n", indent_level*2, indent_level, str); + kfree(str); + + spin_lock_bh(&iface_entry->tag_stat_list_lock); + if (!RB_EMPTY_ROOT(&iface_entry->tag_stat_tree)) { + indent_level++; + prdebug_tag_stat_tree(indent_level, + &iface_entry->tag_stat_tree); + indent_level--; + } + spin_unlock_bh(&iface_entry->tag_stat_list_lock); + } + indent_level--; + str = "}"; + pr_debug("%*d: %s\n", indent_level*2, indent_level, str); +} + +#endif /* ifdef DDEBUG */ +/*------------------------------------------*/ +static const char * const netdev_event_strings[] = { + "netdev_unknown", + "NETDEV_UP", + "NETDEV_DOWN", + "NETDEV_REBOOT", + "NETDEV_CHANGE", + "NETDEV_REGISTER", + "NETDEV_UNREGISTER", + "NETDEV_CHANGEMTU", + "NETDEV_CHANGEADDR", + "NETDEV_GOING_DOWN", + "NETDEV_CHANGENAME", + "NETDEV_FEAT_CHANGE", + "NETDEV_BONDING_FAILOVER", + "NETDEV_PRE_UP", + "NETDEV_PRE_TYPE_CHANGE", + "NETDEV_POST_TYPE_CHANGE", + "NETDEV_POST_INIT", + "NETDEV_UNREGISTER_BATCH", + "NETDEV_RELEASE", + "NETDEV_NOTIFY_PEERS", + "NETDEV_JOIN", +}; + +const char *netdev_evt_str(int netdev_event) +{ + if (netdev_event < 0 + || netdev_event >= ARRAY_SIZE(netdev_event_strings)) + return "bad event num"; + return netdev_event_strings[netdev_event]; +} diff --git a/net/netfilter/xt_qtaguid_print.h b/net/netfilter/xt_qtaguid_print.h new file mode 100644 index 00000000000000..b63871a0be5a7a --- /dev/null +++ b/net/netfilter/xt_qtaguid_print.h @@ -0,0 +1,120 @@ +/* + * Pretty printing Support for iptables xt_qtaguid module. + * + * (C) 2011 Google, Inc + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#ifndef __XT_QTAGUID_PRINT_H__ +#define __XT_QTAGUID_PRINT_H__ + +#include "xt_qtaguid_internal.h" + +#ifdef DDEBUG + +char *pp_tag_t(tag_t *tag); +char *pp_data_counters(struct data_counters *dc, bool showValues); +char *pp_tag_node(struct tag_node *tn); +char *pp_tag_ref(struct tag_ref *tr); +char *pp_tag_stat(struct tag_stat *ts); +char *pp_iface_stat(struct iface_stat *is); +char *pp_sock_tag(struct sock_tag *st); +char *pp_uid_tag_data(struct uid_tag_data *qtd); +char *pp_proc_qtu_data(struct proc_qtu_data *pqd); + +/*------------------------------------------*/ +void prdebug_sock_tag_list(int indent_level, + struct list_head *sock_tag_list); +void prdebug_sock_tag_tree(int indent_level, + struct rb_root *sock_tag_tree); +void prdebug_proc_qtu_data_tree(int indent_level, + struct rb_root *proc_qtu_data_tree); +void prdebug_tag_ref_tree(int indent_level, struct rb_root *tag_ref_tree); +void prdebug_uid_tag_data_tree(int indent_level, + struct rb_root *uid_tag_data_tree); +void prdebug_tag_stat_tree(int indent_level, + struct rb_root *tag_stat_tree); +void prdebug_iface_stat_list(int indent_level, + struct list_head *iface_stat_list); + +#else + +/*------------------------------------------*/ +static inline char *pp_tag_t(tag_t *tag) +{ + return NULL; +} +static inline char *pp_data_counters(struct data_counters *dc, bool showValues) +{ + return NULL; +} +static inline char *pp_tag_node(struct tag_node *tn) +{ + return NULL; +} +static inline char *pp_tag_ref(struct tag_ref *tr) +{ + return NULL; +} +static inline char *pp_tag_stat(struct tag_stat *ts) +{ + return NULL; +} +static inline char *pp_iface_stat(struct iface_stat *is) +{ + return NULL; +} +static inline char *pp_sock_tag(struct sock_tag *st) +{ + return NULL; +} +static inline char *pp_uid_tag_data(struct uid_tag_data *qtd) +{ + return NULL; +} +static inline char *pp_proc_qtu_data(struct proc_qtu_data *pqd) +{ + return NULL; +} + +/*------------------------------------------*/ +static inline +void prdebug_sock_tag_list(int indent_level, + struct list_head *sock_tag_list) +{ +} +static inline +void prdebug_sock_tag_tree(int indent_level, + struct rb_root *sock_tag_tree) +{ +} +static inline +void prdebug_proc_qtu_data_tree(int indent_level, + struct rb_root *proc_qtu_data_tree) +{ +} +static inline +void prdebug_tag_ref_tree(int indent_level, struct rb_root *tag_ref_tree) +{ +} +static inline +void prdebug_uid_tag_data_tree(int indent_level, + struct rb_root *uid_tag_data_tree) +{ +} +static inline +void prdebug_tag_stat_tree(int indent_level, + struct rb_root *tag_stat_tree) +{ +} +static inline +void prdebug_iface_stat_list(int indent_level, + struct list_head *iface_stat_list) +{ +} +#endif +/*------------------------------------------*/ +const char *netdev_evt_str(int netdev_event); +#endif /* ifndef __XT_QTAGUID_PRINT_H__ */ diff --git a/net/netfilter/xt_quota2.c b/net/netfilter/xt_quota2.c new file mode 100644 index 00000000000000..834594aa0085fd --- /dev/null +++ b/net/netfilter/xt_quota2.c @@ -0,0 +1,401 @@ +/* + * xt_quota2 - enhanced xt_quota that can count upwards and in packets + * as a minimal accounting match. + * by Jan Engelhardt , 2008 + * + * Originally based on xt_quota.c: + * netfilter module to enforce network quotas + * Sam Johnston + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License; either + * version 2 of the License, as published by the Free Software Foundation. + */ +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#ifdef CONFIG_NETFILTER_XT_MATCH_QUOTA2_LOG +/* For compatibility, these definitions are copied from the + * deprecated header file */ +#define ULOG_MAC_LEN 80 +#define ULOG_PREFIX_LEN 32 + +/* Format of the ULOG packets passed through netlink */ +typedef struct ulog_packet_msg { + unsigned long mark; + long timestamp_sec; + long timestamp_usec; + unsigned int hook; + char indev_name[IFNAMSIZ]; + char outdev_name[IFNAMSIZ]; + size_t data_len; + char prefix[ULOG_PREFIX_LEN]; + unsigned char mac_len; + unsigned char mac[ULOG_MAC_LEN]; + unsigned char payload[0]; +} ulog_packet_msg_t; +#endif + +/** + * @lock: lock to protect quota writers from each other + */ +struct xt_quota_counter { + u_int64_t quota; + spinlock_t lock; + struct list_head list; + atomic_t ref; + char name[sizeof(((struct xt_quota_mtinfo2 *)NULL)->name)]; + struct proc_dir_entry *procfs_entry; +}; + +#ifdef CONFIG_NETFILTER_XT_MATCH_QUOTA2_LOG +/* Harald's favorite number +1 :D From ipt_ULOG.C */ +static int qlog_nl_event = 112; +module_param_named(event_num, qlog_nl_event, uint, S_IRUGO | S_IWUSR); +MODULE_PARM_DESC(event_num, + "Event number for NETLINK_NFLOG message. 0 disables log." + "111 is what ipt_ULOG uses."); +static struct sock *nflognl; +#endif + +static LIST_HEAD(counter_list); +static DEFINE_SPINLOCK(counter_list_lock); + +static struct proc_dir_entry *proc_xt_quota; +static unsigned int quota_list_perms = S_IRUGO | S_IWUSR; +static kuid_t quota_list_uid = KUIDT_INIT(0); +static kgid_t quota_list_gid = KGIDT_INIT(0); +module_param_named(perms, quota_list_perms, uint, S_IRUGO | S_IWUSR); + +#ifdef CONFIG_NETFILTER_XT_MATCH_QUOTA2_LOG +static void quota2_log(unsigned int hooknum, + const struct sk_buff *skb, + const struct net_device *in, + const struct net_device *out, + const char *prefix) +{ + ulog_packet_msg_t *pm; + struct sk_buff *log_skb; + size_t size; + struct nlmsghdr *nlh; + + if (!qlog_nl_event) + return; + + size = NLMSG_SPACE(sizeof(*pm)); + size = max(size, (size_t)NLMSG_GOODSIZE); + log_skb = alloc_skb(size, GFP_ATOMIC); + if (!log_skb) { + pr_err("xt_quota2: cannot alloc skb for logging\n"); + return; + } + + nlh = nlmsg_put(log_skb, /*pid*/0, /*seq*/0, qlog_nl_event, + sizeof(*pm), 0); + if (!nlh) { + pr_err("xt_quota2: nlmsg_put failed\n"); + kfree_skb(log_skb); + return; + } + pm = nlmsg_data(nlh); + if (skb->tstamp.tv64 == 0) + __net_timestamp((struct sk_buff *)skb); + pm->data_len = 0; + pm->hook = hooknum; + if (prefix != NULL) + strlcpy(pm->prefix, prefix, sizeof(pm->prefix)); + else + *(pm->prefix) = '\0'; + if (in) + strlcpy(pm->indev_name, in->name, sizeof(pm->indev_name)); + else + pm->indev_name[0] = '\0'; + + if (out) + strlcpy(pm->outdev_name, out->name, sizeof(pm->outdev_name)); + else + pm->outdev_name[0] = '\0'; + + NETLINK_CB(log_skb).dst_group = 1; + pr_debug("throwing 1 packets to netlink group 1\n"); + netlink_broadcast(nflognl, log_skb, 0, 1, GFP_ATOMIC); +} +#else +static void quota2_log(unsigned int hooknum, + const struct sk_buff *skb, + const struct net_device *in, + const struct net_device *out, + const char *prefix) +{ +} +#endif /* if+else CONFIG_NETFILTER_XT_MATCH_QUOTA2_LOG */ + +static ssize_t quota_proc_read(struct file *file, char __user *buf, + size_t size, loff_t *ppos) +{ + struct xt_quota_counter *e = PDE_DATA(file_inode(file)); + char tmp[24]; + size_t tmp_size; + + spin_lock_bh(&e->lock); + tmp_size = scnprintf(tmp, sizeof(tmp), "%llu\n", e->quota); + spin_unlock_bh(&e->lock); + return simple_read_from_buffer(buf, size, ppos, tmp, tmp_size); +} + +static ssize_t quota_proc_write(struct file *file, const char __user *input, + size_t size, loff_t *ppos) +{ + struct xt_quota_counter *e = PDE_DATA(file_inode(file)); + char buf[sizeof("18446744073709551616")]; + + if (size > sizeof(buf)) + size = sizeof(buf); + if (copy_from_user(buf, input, size) != 0) + return -EFAULT; + buf[sizeof(buf)-1] = '\0'; + + spin_lock_bh(&e->lock); + e->quota = simple_strtoull(buf, NULL, 0); + spin_unlock_bh(&e->lock); + return size; +} + +static const struct file_operations q2_counter_fops = { + .read = quota_proc_read, + .write = quota_proc_write, + .llseek = default_llseek, +}; + +static struct xt_quota_counter * +q2_new_counter(const struct xt_quota_mtinfo2 *q, bool anon) +{ + struct xt_quota_counter *e; + unsigned int size; + + /* Do not need all the procfs things for anonymous counters. */ + size = anon ? offsetof(typeof(*e), list) : sizeof(*e); + e = kmalloc(size, GFP_KERNEL); + if (e == NULL) + return NULL; + + e->quota = q->quota; + spin_lock_init(&e->lock); + if (!anon) { + INIT_LIST_HEAD(&e->list); + atomic_set(&e->ref, 1); + strlcpy(e->name, q->name, sizeof(e->name)); + } + return e; +} + +/** + * q2_get_counter - get ref to counter or create new + * @name: name of counter + */ +static struct xt_quota_counter * +q2_get_counter(const struct xt_quota_mtinfo2 *q) +{ + struct proc_dir_entry *p; + struct xt_quota_counter *e = NULL; + struct xt_quota_counter *new_e; + + if (*q->name == '\0') + return q2_new_counter(q, true); + + /* No need to hold a lock while getting a new counter */ + new_e = q2_new_counter(q, false); + if (new_e == NULL) + goto out; + + spin_lock_bh(&counter_list_lock); + list_for_each_entry(e, &counter_list, list) + if (strcmp(e->name, q->name) == 0) { + atomic_inc(&e->ref); + spin_unlock_bh(&counter_list_lock); + kfree(new_e); + pr_debug("xt_quota2: old counter name=%s", e->name); + return e; + } + e = new_e; + pr_debug("xt_quota2: new_counter name=%s", e->name); + list_add_tail(&e->list, &counter_list); + /* The entry having a refcount of 1 is not directly destructible. + * This func has not yet returned the new entry, thus iptables + * has not references for destroying this entry. + * For another rule to try to destroy it, it would 1st need for this + * func* to be re-invoked, acquire a new ref for the same named quota. + * Nobody will access the e->procfs_entry either. + * So release the lock. */ + spin_unlock_bh(&counter_list_lock); + + /* create_proc_entry() is not spin_lock happy */ + p = e->procfs_entry = proc_create_data(e->name, quota_list_perms, + proc_xt_quota, &q2_counter_fops, e); + + if (IS_ERR_OR_NULL(p)) { + spin_lock_bh(&counter_list_lock); + list_del(&e->list); + spin_unlock_bh(&counter_list_lock); + goto out; + } + proc_set_user(p, quota_list_uid, quota_list_gid); + return e; + + out: + kfree(e); + return NULL; +} + +static int quota_mt2_check(const struct xt_mtchk_param *par) +{ + struct xt_quota_mtinfo2 *q = par->matchinfo; + + pr_debug("xt_quota2: check() flags=0x%04x", q->flags); + + if (q->flags & ~XT_QUOTA_MASK) + return -EINVAL; + + q->name[sizeof(q->name)-1] = '\0'; + if (*q->name == '.' || strchr(q->name, '/') != NULL) { + printk(KERN_ERR "xt_quota.3: illegal name\n"); + return -EINVAL; + } + + q->master = q2_get_counter(q); + if (q->master == NULL) { + printk(KERN_ERR "xt_quota.3: memory alloc failure\n"); + return -ENOMEM; + } + + return 0; +} + +static void quota_mt2_destroy(const struct xt_mtdtor_param *par) +{ + struct xt_quota_mtinfo2 *q = par->matchinfo; + struct xt_quota_counter *e = q->master; + + if (*q->name == '\0') { + kfree(e); + return; + } + + spin_lock_bh(&counter_list_lock); + if (!atomic_dec_and_test(&e->ref)) { + spin_unlock_bh(&counter_list_lock); + return; + } + + list_del(&e->list); + remove_proc_entry(e->name, proc_xt_quota); + spin_unlock_bh(&counter_list_lock); + kfree(e); +} + +static bool +quota_mt2(const struct sk_buff *skb, struct xt_action_param *par) +{ + struct xt_quota_mtinfo2 *q = (void *)par->matchinfo; + struct xt_quota_counter *e = q->master; + bool ret = q->flags & XT_QUOTA_INVERT; + + spin_lock_bh(&e->lock); + if (q->flags & XT_QUOTA_GROW) { + /* + * While no_change is pointless in "grow" mode, we will + * implement it here simply to have a consistent behavior. + */ + if (!(q->flags & XT_QUOTA_NO_CHANGE)) { + e->quota += (q->flags & XT_QUOTA_PACKET) ? 1 : skb->len; + } + ret = true; + } else { + if (e->quota >= skb->len) { + if (!(q->flags & XT_QUOTA_NO_CHANGE)) + e->quota -= (q->flags & XT_QUOTA_PACKET) ? 1 : skb->len; + ret = !ret; + } else { + /* We are transitioning, log that fact. */ + if (e->quota) { + quota2_log(par->hooknum, + skb, + par->in, + par->out, + q->name); + } + /* we do not allow even small packets from now on */ + e->quota = 0; + } + } + spin_unlock_bh(&e->lock); + return ret; +} + +static struct xt_match quota_mt2_reg[] __read_mostly = { + { + .name = "quota2", + .revision = 3, + .family = NFPROTO_IPV4, + .checkentry = quota_mt2_check, + .match = quota_mt2, + .destroy = quota_mt2_destroy, + .matchsize = sizeof(struct xt_quota_mtinfo2), + .me = THIS_MODULE, + }, + { + .name = "quota2", + .revision = 3, + .family = NFPROTO_IPV6, + .checkentry = quota_mt2_check, + .match = quota_mt2, + .destroy = quota_mt2_destroy, + .matchsize = sizeof(struct xt_quota_mtinfo2), + .me = THIS_MODULE, + }, +}; + +static int __init quota_mt2_init(void) +{ + int ret; + pr_debug("xt_quota2: init()"); + +#ifdef CONFIG_NETFILTER_XT_MATCH_QUOTA2_LOG + nflognl = netlink_kernel_create(&init_net, NETLINK_NFLOG, NULL); + if (!nflognl) + return -ENOMEM; +#endif + + proc_xt_quota = proc_mkdir("xt_quota", init_net.proc_net); + if (proc_xt_quota == NULL) + return -EACCES; + + ret = xt_register_matches(quota_mt2_reg, ARRAY_SIZE(quota_mt2_reg)); + if (ret < 0) + remove_proc_entry("xt_quota", init_net.proc_net); + pr_debug("xt_quota2: init() %d", ret); + return ret; +} + +static void __exit quota_mt2_exit(void) +{ + xt_unregister_matches(quota_mt2_reg, ARRAY_SIZE(quota_mt2_reg)); + remove_proc_entry("xt_quota", init_net.proc_net); +} + +module_init(quota_mt2_init); +module_exit(quota_mt2_exit); +MODULE_DESCRIPTION("Xtables: countdown quota match; up counter"); +MODULE_AUTHOR("Sam Johnston "); +MODULE_AUTHOR("Jan Engelhardt "); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("ipt_quota2"); +MODULE_ALIAS("ip6t_quota2"); diff --git a/net/netfilter/xt_socket.c b/net/netfilter/xt_socket.c index b10ade272b5094..a52fbaf526913f 100644 --- a/net/netfilter/xt_socket.c +++ b/net/netfilter/xt_socket.c @@ -144,13 +144,14 @@ static bool xt_socket_sk_is_transparent(struct sock *sk) } } -static struct sock *xt_socket_lookup_slow_v4(struct net *net, +struct sock *xt_socket_lookup_slow_v4(struct net *net, const struct sk_buff *skb, const struct net_device *indev) { const struct iphdr *iph = ip_hdr(skb); struct sk_buff *data_skb = NULL; int doff = 0; + struct sock *sk = skb->sk; __be32 uninitialized_var(daddr), uninitialized_var(saddr); __be16 uninitialized_var(dport), uninitialized_var(sport); u8 uninitialized_var(protocol); @@ -205,9 +206,16 @@ static struct sock *xt_socket_lookup_slow_v4(struct net *net, } #endif - return xt_socket_get_sock_v4(net, data_skb, doff, protocol, saddr, - daddr, sport, dport, indev); + if (sk) + atomic_inc(&sk->sk_refcnt); + else + sk = xt_socket_get_sock_v4(dev_net(skb->dev), data_skb, doff, + protocol, saddr, daddr, sport, + dport, indev); + + return sk; } +EXPORT_SYMBOL(xt_socket_lookup_slow_v4); static bool socket_match(const struct sk_buff *skb, struct xt_action_param *par, @@ -239,8 +247,7 @@ socket_match(const struct sk_buff *skb, struct xt_action_param *par, transparent) pskb->mark = sk->sk_mark; - if (sk != skb->sk) - sock_gen_put(sk); + sock_gen_put(sk); if (wildcard || !transparent) sk = NULL; @@ -344,10 +351,11 @@ xt_socket_get_sock_v6(struct net *net, struct sk_buff *skb, int doff, return NULL; } -static struct sock *xt_socket_lookup_slow_v6(struct net *net, +struct sock *xt_socket_lookup_slow_v6(struct net *net, const struct sk_buff *skb, const struct net_device *indev) { + struct sock *sk = skb->sk; __be16 uninitialized_var(dport), uninitialized_var(sport); const struct in6_addr *daddr = NULL, *saddr = NULL; struct ipv6hdr *iph = ipv6_hdr(skb); @@ -387,9 +395,16 @@ static struct sock *xt_socket_lookup_slow_v6(struct net *net, return NULL; } - return xt_socket_get_sock_v6(net, data_skb, doff, tproto, saddr, daddr, - sport, dport, indev); + if (sk) + atomic_inc(&sk->sk_refcnt); + else + sk = xt_socket_get_sock_v6(dev_net(skb->dev), data_skb, doff, + tproto, saddr, daddr, sport, dport, + indev); + + return sk; } +EXPORT_SYMBOL(xt_socket_lookup_slow_v6); static bool socket_mt6_v1_v2_v3(const struct sk_buff *skb, struct xt_action_param *par) diff --git a/net/openvswitch/conntrack.c b/net/openvswitch/conntrack.c index fecefa2dc94e12..eab210bb1ef0f5 100644 --- a/net/openvswitch/conntrack.c +++ b/net/openvswitch/conntrack.c @@ -514,7 +514,7 @@ static int ovs_ct_nat_execute(struct sk_buff *skb, struct nf_conn *ct, int hooknum, nh_off, err = NF_ACCEPT; nh_off = skb_network_offset(skb); - skb_pull(skb, nh_off); + skb_pull_rcsum(skb, nh_off); /* See HOOK2MANIP(). */ if (maniptype == NF_NAT_MANIP_SRC) @@ -579,6 +579,7 @@ static int ovs_ct_nat_execute(struct sk_buff *skb, struct nf_conn *ct, err = nf_nat_packet(ct, ctinfo, hooknum, skb); push: skb_push(skb, nh_off); + skb_postpush_rcsum(skb, skb->data, nh_off); return err; } @@ -890,7 +891,7 @@ int ovs_ct_execute(struct net *net, struct sk_buff *skb, /* The conntrack module expects to be working at L3. */ nh_ofs = skb_network_offset(skb); - skb_pull(skb, nh_ofs); + skb_pull_rcsum(skb, nh_ofs); if (key->ip.frag != OVS_FRAG_TYPE_NONE) { err = handle_fragments(net, key, info->zone.id, skb); @@ -904,6 +905,7 @@ int ovs_ct_execute(struct net *net, struct sk_buff *skb, err = ovs_ct_lookup(net, key, info, skb); skb_push(skb, nh_ofs); + skb_postpush_rcsum(skb, skb->data, nh_ofs); if (err) kfree_skb(skb); return err; diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index dd2332390c45bb..94e4a5941d89e8 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -1972,7 +1972,7 @@ static int __packet_rcv_vnet(const struct sk_buff *skb, { *vnet_hdr = (const struct virtio_net_hdr) { 0 }; - if (virtio_net_hdr_from_skb(skb, vnet_hdr, vio_le())) + if (virtio_net_hdr_from_skb(skb, vnet_hdr, vio_le(), true)) BUG(); return 0; diff --git a/net/rfkill/Kconfig b/net/rfkill/Kconfig index 868f1ad0415a4e..8463a6d4d508f2 100644 --- a/net/rfkill/Kconfig +++ b/net/rfkill/Kconfig @@ -10,6 +10,11 @@ menuconfig RFKILL To compile this driver as a module, choose M here: the module will be called rfkill. +config RFKILL_PM + bool "Power off on suspend" + depends on RFKILL && PM + default y + # LED trigger support config RFKILL_LEDS bool diff --git a/net/rfkill/core.c b/net/rfkill/core.c index 884027f62783a6..9b4260dab2a4b0 100644 --- a/net/rfkill/core.c +++ b/net/rfkill/core.c @@ -796,7 +796,7 @@ void rfkill_resume_polling(struct rfkill *rfkill) } EXPORT_SYMBOL(rfkill_resume_polling); -#ifdef CONFIG_PM_SLEEP +#ifdef CONFIG_RFKILL_PM static int rfkill_suspend(struct device *dev) { struct rfkill *rfkill = to_rfkill(dev); @@ -837,7 +837,9 @@ static struct class rfkill_class = { .dev_release = rfkill_release, .dev_groups = rfkill_dev_groups, .dev_uevent = rfkill_dev_uevent, +#ifdef CONFIG_RFKILL_PM .pm = RFKILL_PM_OPS, +#endif }; bool rfkill_blocked(struct rfkill *rfkill) diff --git a/net/sched/act_api.c b/net/sched/act_api.c index f893d180da1caa..c6c2a93cc2a23a 100644 --- a/net/sched/act_api.c +++ b/net/sched/act_api.c @@ -903,8 +903,6 @@ tca_action_gd(struct net *net, struct nlattr *nla, struct nlmsghdr *n, goto err; } act->order = i; - if (event == RTM_GETACTION) - act->tcfa_refcnt++; list_add_tail(&act->list, &actions); } @@ -917,7 +915,8 @@ tca_action_gd(struct net *net, struct nlattr *nla, struct nlmsghdr *n, return ret; } err: - tcf_action_destroy(&actions, 0); + if (event != RTM_GETACTION) + tcf_action_destroy(&actions, 0); return ret; } diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c index b05d4a2155b0b0..c1a4b5d30814af 100644 --- a/net/sched/cls_api.c +++ b/net/sched/cls_api.c @@ -148,13 +148,15 @@ static int tc_ctl_tfilter(struct sk_buff *skb, struct nlmsghdr *n) unsigned long cl; unsigned long fh; int err; - int tp_created = 0; + int tp_created; if ((n->nlmsg_type != RTM_GETTFILTER) && !netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN)) return -EPERM; replay: + tp_created = 0; + err = nlmsg_parse(n, sizeof(*t), tca, TCA_MAX, NULL); if (err < 0) return err; diff --git a/net/sched/cls_flower.c b/net/sched/cls_flower.c index 904442421db3af..eee299bb6bcffb 100644 --- a/net/sched/cls_flower.c +++ b/net/sched/cls_flower.c @@ -149,10 +149,14 @@ static int fl_classify(struct sk_buff *skb, const struct tcf_proto *tp, switch (ip_tunnel_info_af(info)) { case AF_INET: + skb_key.enc_control.addr_type = + FLOW_DISSECTOR_KEY_IPV4_ADDRS; skb_key.enc_ipv4.src = key->u.ipv4.src; skb_key.enc_ipv4.dst = key->u.ipv4.dst; break; case AF_INET6: + skb_key.enc_control.addr_type = + FLOW_DISSECTOR_KEY_IPV6_ADDRS; skb_key.enc_ipv6.src = key->u.ipv6.src; skb_key.enc_ipv6.dst = key->u.ipv6.dst; break; diff --git a/net/sctp/socket.c b/net/sctp/socket.c index f23ad913dc7a07..ca12aa346c0d05 100644 --- a/net/sctp/socket.c +++ b/net/sctp/socket.c @@ -4479,9 +4479,10 @@ int sctp_transport_lookup_process(int (*cb)(struct sctp_transport *, void *), rcu_read_lock(); transport = sctp_addrs_lookup_transport(net, laddr, paddr); - if (!transport || !sctp_transport_hold(transport)) + if (!transport || !sctp_transport_hold(transport)) { + rcu_read_unlock(); goto out; - + } rcu_read_unlock(); err = cb(transport, p); sctp_transport_put(transport); diff --git a/net/socket.c b/net/socket.c index 73dc69f9681e7b..5f4ec66570e1fb 100644 --- a/net/socket.c +++ b/net/socket.c @@ -533,8 +533,22 @@ static ssize_t sockfs_listxattr(struct dentry *dentry, char *buffer, return used; } +int sockfs_setattr(struct dentry *dentry, struct iattr *iattr) +{ + int err = simple_setattr(dentry, iattr); + + if (!err && (iattr->ia_valid & ATTR_UID)) { + struct socket *sock = SOCKET_I(d_inode(dentry)); + + sock->sk->sk_uid = iattr->ia_uid; + } + + return err; +} + static const struct inode_operations sockfs_inode_ops = { .listxattr = sockfs_listxattr, + .setattr = sockfs_setattr, }; /** diff --git a/net/sunrpc/auth_gss/auth_gss.c b/net/sunrpc/auth_gss/auth_gss.c index 3dfd769dc5b51a..16cea00c959b65 100644 --- a/net/sunrpc/auth_gss/auth_gss.c +++ b/net/sunrpc/auth_gss/auth_gss.c @@ -541,9 +541,13 @@ gss_setup_upcall(struct gss_auth *gss_auth, struct rpc_cred *cred) return gss_new; gss_msg = gss_add_msg(gss_new); if (gss_msg == gss_new) { - int res = rpc_queue_upcall(gss_new->pipe, &gss_new->msg); + int res; + atomic_inc(&gss_msg->count); + res = rpc_queue_upcall(gss_new->pipe, &gss_new->msg); if (res) { gss_unhash_msg(gss_new); + atomic_dec(&gss_msg->count); + gss_release_msg(gss_new); gss_msg = ERR_PTR(res); } } else @@ -836,6 +840,7 @@ gss_pipe_destroy_msg(struct rpc_pipe_msg *msg) warn_gssd(); gss_release_msg(gss_msg); } + gss_release_msg(gss_msg); } static void gss_pipe_dentry_destroy(struct dentry *dir, diff --git a/net/sunrpc/auth_gss/gss_rpc_xdr.c b/net/sunrpc/auth_gss/gss_rpc_xdr.c index dc6fb79a361f1c..25d9a9cf7b66b7 100644 --- a/net/sunrpc/auth_gss/gss_rpc_xdr.c +++ b/net/sunrpc/auth_gss/gss_rpc_xdr.c @@ -260,7 +260,7 @@ static int gssx_dec_option_array(struct xdr_stream *xdr, if (!oa->data) return -ENOMEM; - creds = kmalloc(sizeof(struct svc_cred), GFP_KERNEL); + creds = kzalloc(sizeof(struct svc_cred), GFP_KERNEL); if (!creds) { kfree(oa->data); return -ENOMEM; diff --git a/net/sunrpc/auth_gss/svcauth_gss.c b/net/sunrpc/auth_gss/svcauth_gss.c index 45662d7f0943c6..6fdffde28733df 100644 --- a/net/sunrpc/auth_gss/svcauth_gss.c +++ b/net/sunrpc/auth_gss/svcauth_gss.c @@ -1489,7 +1489,7 @@ svcauth_gss_accept(struct svc_rqst *rqstp, __be32 *authp) case RPC_GSS_PROC_DESTROY: if (gss_write_verf(rqstp, rsci->mechctx, gc->gc_seq)) goto auth_err; - rsci->h.expiry_time = get_seconds(); + rsci->h.expiry_time = seconds_since_boot(); set_bit(CACHE_NEGATIVE, &rsci->h.flags); if (resv->iov_len + 4 > PAGE_SIZE) goto drop; diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c index 62a482790937b5..b2ae4f150ec639 100644 --- a/net/sunrpc/clnt.c +++ b/net/sunrpc/clnt.c @@ -336,6 +336,11 @@ static int rpc_client_register(struct rpc_clnt *clnt, static DEFINE_IDA(rpc_clids); +void rpc_cleanup_clids(void) +{ + ida_destroy(&rpc_clids); +} + static int rpc_alloc_clid(struct rpc_clnt *clnt) { int clid; diff --git a/net/sunrpc/sunrpc_syms.c b/net/sunrpc/sunrpc_syms.c index ee5d3d253102bf..3142f38d110462 100644 --- a/net/sunrpc/sunrpc_syms.c +++ b/net/sunrpc/sunrpc_syms.c @@ -119,6 +119,7 @@ init_sunrpc(void) static void __exit cleanup_sunrpc(void) { + rpc_cleanup_clids(); rpcauth_remove_module(); cleanup_socket_xprt(); svc_cleanup_xprt_sock(); diff --git a/net/sunrpc/svc_xprt.c b/net/sunrpc/svc_xprt.c index 3bc1d61694cbbb..9c9db55a0c1e17 100644 --- a/net/sunrpc/svc_xprt.c +++ b/net/sunrpc/svc_xprt.c @@ -799,6 +799,8 @@ static int svc_handle_xprt(struct svc_rqst *rqstp, struct svc_xprt *xprt) if (test_bit(XPT_CLOSE, &xprt->xpt_flags)) { dprintk("svc_recv: found XPT_CLOSE\n"); + if (test_and_clear_bit(XPT_KILL_TEMP, &xprt->xpt_flags)) + xprt->xpt_ops->xpo_kill_temp_xprt(xprt); svc_delete_xprt(xprt); /* Leave XPT_BUSY set on the dead xprt: */ goto out; @@ -1020,9 +1022,11 @@ void svc_age_temp_xprts_now(struct svc_serv *serv, struct sockaddr *server_addr) le = to_be_closed.next; list_del_init(le); xprt = list_entry(le, struct svc_xprt, xpt_list); - dprintk("svc_age_temp_xprts_now: closing %p\n", xprt); - xprt->xpt_ops->xpo_kill_temp_xprt(xprt); - svc_close_xprt(xprt); + set_bit(XPT_CLOSE, &xprt->xpt_flags); + set_bit(XPT_KILL_TEMP, &xprt->xpt_flags); + dprintk("svc_age_temp_xprts_now: queuing xprt %p for closing\n", + xprt); + svc_xprt_enqueue(xprt); } } EXPORT_SYMBOL_GPL(svc_age_temp_xprts_now); diff --git a/net/sunrpc/xprtrdma/frwr_ops.c b/net/sunrpc/xprtrdma/frwr_ops.c index 26b26beef2d4a6..adbf52c6df833b 100644 --- a/net/sunrpc/xprtrdma/frwr_ops.c +++ b/net/sunrpc/xprtrdma/frwr_ops.c @@ -421,7 +421,7 @@ frwr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg, IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE : IB_ACCESS_REMOTE_READ; - DECR_CQCOUNT(&r_xprt->rx_ep); + rpcrdma_set_signaled(&r_xprt->rx_ep, ®_wr->wr); rc = ib_post_send(ia->ri_id->qp, ®_wr->wr, &bad_wr); if (rc) goto out_senderr; @@ -486,7 +486,7 @@ frwr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) struct rpcrdma_ia *ia = &r_xprt->rx_ia; struct rpcrdma_mw *mw, *tmp; struct rpcrdma_frmr *f; - int rc; + int count, rc; dprintk("RPC: %s: req %p\n", __func__, req); @@ -496,6 +496,7 @@ frwr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) * a single ib_post_send() call. */ f = NULL; + count = 0; invalidate_wrs = pos = prev = NULL; list_for_each_entry(mw, &req->rl_registered, mw_list) { if ((rep->rr_wc_flags & IB_WC_WITH_INVALIDATE) && @@ -505,6 +506,7 @@ frwr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) } pos = __frwr_prepare_linv_wr(mw); + count++; if (!invalidate_wrs) invalidate_wrs = pos; @@ -523,7 +525,12 @@ frwr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) f->fr_invwr.send_flags = IB_SEND_SIGNALED; f->fr_cqe.done = frwr_wc_localinv_wake; reinit_completion(&f->fr_linv_done); - INIT_CQCOUNT(&r_xprt->rx_ep); + + /* Initialize CQ count, since there is always a signaled + * WR being posted here. The new cqcount depends on how + * many SQEs are about to be consumed. + */ + rpcrdma_init_cqcount(&r_xprt->rx_ep, count); /* Transport disconnect drains the receive CQ before it * replaces the QP. The RPC reply handler won't call us diff --git a/net/sunrpc/xprtrdma/svc_rdma_backchannel.c b/net/sunrpc/xprtrdma/svc_rdma_backchannel.c index 20027f8de129e6..6035c5a380a6b1 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_backchannel.c +++ b/net/sunrpc/xprtrdma/svc_rdma_backchannel.c @@ -359,6 +359,7 @@ xprt_setup_rdma_bc(struct xprt_create *args) out_fail: xprt_rdma_free_addresses(xprt); args->bc_xprt->xpt_bc_xprt = NULL; + args->bc_xprt->xpt_bc_xps = NULL; xprt_put(xprt); xprt_free(xprt); return ERR_PTR(-EINVAL); diff --git a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c index ad1df979b3f072..a47c9bdef5fa95 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c +++ b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c @@ -348,8 +348,6 @@ int rdma_read_chunk_frmr(struct svcxprt_rdma *xprt, atomic_inc(&rdma_stat_read); return ret; err: - ib_dma_unmap_sg(xprt->sc_cm_id->device, - frmr->sg, frmr->sg_nents, frmr->direction); svc_rdma_put_context(ctxt, 0); svc_rdma_put_frmr(xprt, frmr); return ret; diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c index ec74289af7ec90..8da7f6a4dfc387 100644 --- a/net/sunrpc/xprtrdma/verbs.c +++ b/net/sunrpc/xprtrdma/verbs.c @@ -223,8 +223,8 @@ rpcrdma_update_connect_private(struct rpcrdma_xprt *r_xprt, cdata->inline_rsize = rsize; if (wsize < cdata->inline_wsize) cdata->inline_wsize = wsize; - pr_info("rpcrdma: max send %u, max recv %u\n", - cdata->inline_wsize, cdata->inline_rsize); + dprintk("RPC: %s: max send %u, max recv %u\n", + __func__, cdata->inline_wsize, cdata->inline_rsize); rpcrdma_set_max_header_sizes(r_xprt); } @@ -532,7 +532,7 @@ rpcrdma_ep_create(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia, ep->rep_cqinit = ep->rep_attr.cap.max_send_wr/2 - 1; if (ep->rep_cqinit <= 2) ep->rep_cqinit = 0; /* always signal? */ - INIT_CQCOUNT(ep); + rpcrdma_init_cqcount(ep, 0); init_waitqueue_head(&ep->rep_connect_wait); INIT_DELAYED_WORK(&ep->rep_connect_worker, rpcrdma_connect_worker); @@ -1311,13 +1311,7 @@ rpcrdma_ep_post(struct rpcrdma_ia *ia, dprintk("RPC: %s: posting %d s/g entries\n", __func__, send_wr->num_sge); - if (DECR_CQCOUNT(ep) > 0) - send_wr->send_flags = 0; - else { /* Provider must take a send completion every now and then */ - INIT_CQCOUNT(ep); - send_wr->send_flags = IB_SEND_SIGNALED; - } - + rpcrdma_set_signaled(ep, send_wr); rc = ib_post_send(ia->ri_id->qp, send_wr, &send_wr_fail); if (rc) goto out_postsend_err; diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h index 6e1bba35820369..f6ae1b22da476b 100644 --- a/net/sunrpc/xprtrdma/xprt_rdma.h +++ b/net/sunrpc/xprtrdma/xprt_rdma.h @@ -95,8 +95,24 @@ struct rpcrdma_ep { struct delayed_work rep_connect_worker; }; -#define INIT_CQCOUNT(ep) atomic_set(&(ep)->rep_cqcount, (ep)->rep_cqinit) -#define DECR_CQCOUNT(ep) atomic_sub_return(1, &(ep)->rep_cqcount) +static inline void +rpcrdma_init_cqcount(struct rpcrdma_ep *ep, int count) +{ + atomic_set(&ep->rep_cqcount, ep->rep_cqinit - count); +} + +/* To update send queue accounting, provider must take a + * send completion every now and then. + */ +static inline void +rpcrdma_set_signaled(struct rpcrdma_ep *ep, struct ib_send_wr *send_wr) +{ + send_wr->send_flags = 0; + if (unlikely(atomic_sub_return(1, &ep->rep_cqcount) <= 0)) { + rpcrdma_init_cqcount(ep, 0); + send_wr->send_flags = IB_SEND_SIGNALED; + } +} /* Pre-allocate extra Work Requests for handling backward receives * and sends. This is a fixed value because the Work Queues are diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c index 2358f2690ec58c..2d03d5bcb5b932 100644 --- a/net/unix/af_unix.c +++ b/net/unix/af_unix.c @@ -995,6 +995,7 @@ static int unix_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) unsigned int hash; struct unix_address *addr; struct hlist_head *list; + struct path path = { NULL, NULL }; err = -EINVAL; if (sunaddr->sun_family != AF_UNIX) @@ -1010,9 +1011,20 @@ static int unix_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) goto out; addr_len = err; + if (sun_path[0]) { + umode_t mode = S_IFSOCK | + (SOCK_INODE(sock)->i_mode & ~current_umask()); + err = unix_mknod(sun_path, mode, &path); + if (err) { + if (err == -EEXIST) + err = -EADDRINUSE; + goto out; + } + } + err = mutex_lock_interruptible(&u->bindlock); if (err) - goto out; + goto out_put; err = -EINVAL; if (u->addr) @@ -1029,16 +1041,6 @@ static int unix_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) atomic_set(&addr->refcnt, 1); if (sun_path[0]) { - struct path path; - umode_t mode = S_IFSOCK | - (SOCK_INODE(sock)->i_mode & ~current_umask()); - err = unix_mknod(sun_path, mode, &path); - if (err) { - if (err == -EEXIST) - err = -EADDRINUSE; - unix_release_addr(addr); - goto out_up; - } addr->hash = UNIX_HASH_SIZE; hash = d_real_inode(path.dentry)->i_ino & (UNIX_HASH_SIZE - 1); spin_lock(&unix_table_lock); @@ -1065,6 +1067,9 @@ static int unix_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) spin_unlock(&unix_table_lock); out_up: mutex_unlock(&u->bindlock); +out_put: + if (err) + path_put(&path); out: return err; } diff --git a/net/vmw_vsock/virtio_transport_common.c b/net/vmw_vsock/virtio_transport_common.c index a53b3a16b4f1f7..62c056ea403bc1 100644 --- a/net/vmw_vsock/virtio_transport_common.c +++ b/net/vmw_vsock/virtio_transport_common.c @@ -606,9 +606,9 @@ static int virtio_transport_reset_no_sock(struct virtio_vsock_pkt *pkt) return 0; pkt = virtio_transport_alloc_pkt(&info, 0, - le32_to_cpu(pkt->hdr.dst_cid), + le64_to_cpu(pkt->hdr.dst_cid), le32_to_cpu(pkt->hdr.dst_port), - le32_to_cpu(pkt->hdr.src_cid), + le64_to_cpu(pkt->hdr.src_cid), le32_to_cpu(pkt->hdr.src_port)); if (!pkt) return -ENOMEM; @@ -823,7 +823,7 @@ virtio_transport_send_response(struct vsock_sock *vsk, struct virtio_vsock_pkt_info info = { .op = VIRTIO_VSOCK_OP_RESPONSE, .type = VIRTIO_VSOCK_TYPE_STREAM, - .remote_cid = le32_to_cpu(pkt->hdr.src_cid), + .remote_cid = le64_to_cpu(pkt->hdr.src_cid), .remote_port = le32_to_cpu(pkt->hdr.src_port), .reply = true, }; @@ -863,9 +863,9 @@ virtio_transport_recv_listen(struct sock *sk, struct virtio_vsock_pkt *pkt) child->sk_state = SS_CONNECTED; vchild = vsock_sk(child); - vsock_addr_init(&vchild->local_addr, le32_to_cpu(pkt->hdr.dst_cid), + vsock_addr_init(&vchild->local_addr, le64_to_cpu(pkt->hdr.dst_cid), le32_to_cpu(pkt->hdr.dst_port)); - vsock_addr_init(&vchild->remote_addr, le32_to_cpu(pkt->hdr.src_cid), + vsock_addr_init(&vchild->remote_addr, le64_to_cpu(pkt->hdr.src_cid), le32_to_cpu(pkt->hdr.src_port)); vsock_insert_connected(vchild); @@ -904,9 +904,9 @@ void virtio_transport_recv_pkt(struct virtio_vsock_pkt *pkt) struct sock *sk; bool space_available; - vsock_addr_init(&src, le32_to_cpu(pkt->hdr.src_cid), + vsock_addr_init(&src, le64_to_cpu(pkt->hdr.src_cid), le32_to_cpu(pkt->hdr.src_port)); - vsock_addr_init(&dst, le32_to_cpu(pkt->hdr.dst_cid), + vsock_addr_init(&dst, le64_to_cpu(pkt->hdr.dst_cid), le32_to_cpu(pkt->hdr.dst_port)); trace_virtio_transport_recv_pkt(src.svm_cid, src.svm_port, diff --git a/net/wireless/core.h b/net/wireless/core.h index f0c0c8a48c9208..5f5867f90fed5e 100644 --- a/net/wireless/core.h +++ b/net/wireless/core.h @@ -410,6 +410,7 @@ void cfg80211_sme_disassoc(struct wireless_dev *wdev); void cfg80211_sme_deauth(struct wireless_dev *wdev); void cfg80211_sme_auth_timeout(struct wireless_dev *wdev); void cfg80211_sme_assoc_timeout(struct wireless_dev *wdev); +void cfg80211_sme_abandon_assoc(struct wireless_dev *wdev); /* internal helpers */ bool cfg80211_supported_cipher_suite(struct wiphy *wiphy, u32 cipher); diff --git a/net/wireless/mlme.c b/net/wireless/mlme.c index cbb48e26a8715b..76775a2b421db3 100644 --- a/net/wireless/mlme.c +++ b/net/wireless/mlme.c @@ -149,6 +149,18 @@ void cfg80211_assoc_timeout(struct net_device *dev, struct cfg80211_bss *bss) } EXPORT_SYMBOL(cfg80211_assoc_timeout); +void cfg80211_abandon_assoc(struct net_device *dev, struct cfg80211_bss *bss) +{ + struct wireless_dev *wdev = dev->ieee80211_ptr; + struct wiphy *wiphy = wdev->wiphy; + + cfg80211_sme_abandon_assoc(wdev); + + cfg80211_unhold_bss(bss_from_pub(bss)); + cfg80211_put_bss(wiphy, bss); +} +EXPORT_SYMBOL(cfg80211_abandon_assoc); + void cfg80211_tx_mlme_mgmt(struct net_device *dev, const u8 *buf, size_t len) { struct wireless_dev *wdev = dev->ieee80211_ptr; diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c index c510810f0b7c14..cd7a419faa2105 100644 --- a/net/wireless/nl80211.c +++ b/net/wireless/nl80211.c @@ -414,6 +414,7 @@ static const struct nla_policy nl80211_policy[NUM_NL80211_ATTR] = { [NL80211_ATTR_NAN_MASTER_PREF] = { .type = NLA_U8 }, [NL80211_ATTR_NAN_DUAL] = { .type = NLA_U8 }, [NL80211_ATTR_NAN_FUNC] = { .type = NLA_NESTED }, + [NL80211_ATTR_BSSID] = { .len = ETH_ALEN }, }; /* policy for the key attributes */ @@ -5873,6 +5874,7 @@ do { \ break; } cfg->ht_opmode = ht_opmode; + mask |= (1 << (NL80211_MESHCONF_HT_OPMODE - 1)); } FILL_IN_MESH_PARAM_IF_SET(tb, cfg, dot11MeshHWMPactivePathToRootTimeout, 1, 65535, mask, @@ -6677,7 +6679,20 @@ static int nl80211_trigger_scan(struct sk_buff *skb, struct genl_info *info) request->no_cck = nla_get_flag(info->attrs[NL80211_ATTR_TX_NO_CCK_RATE]); - if (info->attrs[NL80211_ATTR_MAC]) + /* Initial implementation used NL80211_ATTR_MAC to set the specific + * BSSID to scan for. This was problematic because that same attribute + * was already used for another purpose (local random MAC address). The + * NL80211_ATTR_BSSID attribute was added to fix this. For backwards + * compatibility with older userspace components, also use the + * NL80211_ATTR_MAC value here if it can be determined to be used for + * the specific BSSID use case instead of the random MAC address + * (NL80211_ATTR_SCAN_FLAGS is used to enable random MAC address use). + */ + if (info->attrs[NL80211_ATTR_BSSID]) + memcpy(request->bssid, + nla_data(info->attrs[NL80211_ATTR_BSSID]), ETH_ALEN); + else if (!(request->flags & NL80211_SCAN_FLAG_RANDOM_ADDR) && + info->attrs[NL80211_ATTR_MAC]) memcpy(request->bssid, nla_data(info->attrs[NL80211_ATTR_MAC]), ETH_ALEN); else @@ -14388,13 +14403,17 @@ static int nl80211_netlink_notify(struct notifier_block * nb, list_for_each_entry_rcu(rdev, &cfg80211_rdev_list, list) { bool schedule_destroy_work = false; - bool schedule_scan_stop = false; struct cfg80211_sched_scan_request *sched_scan_req = rcu_dereference(rdev->sched_scan_req); if (sched_scan_req && notify->portid && - sched_scan_req->owner_nlportid == notify->portid) - schedule_scan_stop = true; + sched_scan_req->owner_nlportid == notify->portid) { + sched_scan_req->owner_nlportid = 0; + + if (rdev->ops->sched_scan_stop && + rdev->wiphy.flags & WIPHY_FLAG_SUPPORTS_SCHED_SCAN) + schedule_work(&rdev->sched_scan_stop_wk); + } list_for_each_entry_rcu(wdev, &rdev->wiphy.wdev_list, list) { cfg80211_mlme_unregister_socket(wdev, notify->portid); @@ -14425,12 +14444,6 @@ static int nl80211_netlink_notify(struct notifier_block * nb, spin_unlock(&rdev->destroy_list_lock); schedule_work(&rdev->destroy_work); } - } else if (schedule_scan_stop) { - sched_scan_req->owner_nlportid = 0; - - if (rdev->ops->sched_scan_stop && - rdev->wiphy.flags & WIPHY_FLAG_SUPPORTS_SCHED_SCAN) - schedule_work(&rdev->sched_scan_stop_wk); } } diff --git a/net/wireless/scan.c b/net/wireless/scan.c index 35ad69fd08383a..e31887810b5495 100644 --- a/net/wireless/scan.c +++ b/net/wireless/scan.c @@ -70,7 +70,7 @@ module_param(bss_entries_limit, int, 0644); MODULE_PARM_DESC(bss_entries_limit, "limit to number of scan BSS entries (per wiphy, default 1000)"); -#define IEEE80211_SCAN_RESULT_EXPIRE (30 * HZ) +#define IEEE80211_SCAN_RESULT_EXPIRE (7 * HZ) static void bss_free(struct cfg80211_internal_bss *bss) { diff --git a/net/wireless/sme.c b/net/wireless/sme.c index a77db333927ec1..35cc1de85dcc53 100644 --- a/net/wireless/sme.c +++ b/net/wireless/sme.c @@ -39,6 +39,7 @@ struct cfg80211_conn { CFG80211_CONN_ASSOCIATING, CFG80211_CONN_ASSOC_FAILED, CFG80211_CONN_DEAUTH, + CFG80211_CONN_ABANDON, CFG80211_CONN_CONNECTED, } state; u8 bssid[ETH_ALEN], prev_bssid[ETH_ALEN]; @@ -206,6 +207,8 @@ static int cfg80211_conn_do_work(struct wireless_dev *wdev) cfg80211_mlme_deauth(rdev, wdev->netdev, params->bssid, NULL, 0, WLAN_REASON_DEAUTH_LEAVING, false); + /* fall through */ + case CFG80211_CONN_ABANDON: /* free directly, disconnected event already sent */ cfg80211_sme_free(wdev); return 0; @@ -423,6 +426,17 @@ void cfg80211_sme_assoc_timeout(struct wireless_dev *wdev) schedule_work(&rdev->conn_work); } +void cfg80211_sme_abandon_assoc(struct wireless_dev *wdev) +{ + struct cfg80211_registered_device *rdev = wiphy_to_rdev(wdev->wiphy); + + if (!wdev->conn) + return; + + wdev->conn->state = CFG80211_CONN_ABANDON; + schedule_work(&rdev->conn_work); +} + static int cfg80211_sme_get_conn_ies(struct wireless_dev *wdev, const u8 *ies, size_t ies_len, const u8 **out_ies, size_t *out_ies_len) diff --git a/scripts/Makefile.clean b/scripts/Makefile.clean index 50616ea251314b..2e70c6f0635443 100644 --- a/scripts/Makefile.clean +++ b/scripts/Makefile.clean @@ -11,7 +11,7 @@ include scripts/Kbuild.include # The filename Kbuild has precedence over Makefile kbuild-dir := $(if $(filter /%,$(src)),$(src),$(srctree)/$(src)) -include $(if $(wildcard $(kbuild-dir)/Kbuild), $(kbuild-dir)/Kbuild, $(kbuild-dir)/Makefile) +-include $(if $(wildcard $(kbuild-dir)/Kbuild), $(kbuild-dir)/Kbuild, $(kbuild-dir)/Makefile) # Figure out what we need to build from the various variables # ========================================================================== diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib index 0a07f9014944ed..d3d3320722cbd1 100644 --- a/scripts/Makefile.lib +++ b/scripts/Makefile.lib @@ -314,6 +314,12 @@ $(obj)/%.dtb: $(src)/%.dts FORCE dtc-tmp = $(subst $(comma),_,$(dot-target).dts.tmp) +# cat +# --------------------------------------------------------------------------- +# Concatentate multiple files together +quiet_cmd_cat = CAT $@ +cmd_cat = (cat $(filter-out FORCE,$^) > $@) || (rm -f $@; false) + # Bzip2 # --------------------------------------------------------------------------- diff --git a/scripts/Makefile.modinst b/scripts/Makefile.modinst index 07650eeaaf06dd..6f4c3f5a7ae39b 100644 --- a/scripts/Makefile.modinst +++ b/scripts/Makefile.modinst @@ -29,7 +29,7 @@ quiet_cmd_modules_install = INSTALL $@ INSTALL_MOD_DIR ?= extra ext-mod-dir = $(INSTALL_MOD_DIR)$(subst $(patsubst %/,%,$(KBUILD_EXTMOD)),,$(@D)) -modinst_dir = $(if $(KBUILD_EXTMOD),$(ext-mod-dir),kernel/$(@D)) +modinst_dir ?= $(if $(KBUILD_EXTMOD),$(ext-mod-dir),kernel/$(@D)) $(modules): $(call cmd,modules_install,$(MODLIB)/$(modinst_dir)) diff --git a/scripts/gcc-plugins/gcc-common.h b/scripts/gcc-plugins/gcc-common.h index 950fd2e64bb73b..12262c0cc6914e 100644 --- a/scripts/gcc-plugins/gcc-common.h +++ b/scripts/gcc-plugins/gcc-common.h @@ -39,6 +39,9 @@ #include "hash-map.h" #endif +#if BUILDING_GCC_VERSION >= 7000 +#include "memmodel.h" +#endif #include "emit-rtl.h" #include "debug.h" #include "target.h" @@ -91,6 +94,9 @@ #include "tree-ssa-alias.h" #include "tree-ssa.h" #include "stringpool.h" +#if BUILDING_GCC_VERSION >= 7000 +#include "tree-vrp.h" +#endif #include "tree-ssanames.h" #include "print-tree.h" #include "tree-eh.h" @@ -287,6 +293,22 @@ static inline struct cgraph_node *cgraph_next_function_with_gimple_body(struct c return NULL; } +static inline bool cgraph_for_node_and_aliases(cgraph_node_ptr node, bool (*callback)(cgraph_node_ptr, void *), void *data, bool include_overwritable) +{ + cgraph_node_ptr alias; + + if (callback(node, data)) + return true; + + for (alias = node->same_body; alias; alias = alias->next) { + if (include_overwritable || cgraph_function_body_availability(alias) > AVAIL_OVERWRITABLE) + if (cgraph_for_node_and_aliases(alias, callback, data, include_overwritable)) + return true; + } + + return false; +} + #define FOR_EACH_FUNCTION_WITH_GIMPLE_BODY(node) \ for ((node) = cgraph_first_function_with_gimple_body(); (node); \ (node) = cgraph_next_function_with_gimple_body(node)) @@ -399,6 +421,7 @@ typedef union gimple_statement_d gassign; typedef union gimple_statement_d gcall; typedef union gimple_statement_d gcond; typedef union gimple_statement_d gdebug; +typedef union gimple_statement_d ggoto; typedef union gimple_statement_d gphi; typedef union gimple_statement_d greturn; @@ -452,6 +475,16 @@ static inline const gdebug *as_a_const_gdebug(const_gimple stmt) return stmt; } +static inline ggoto *as_a_ggoto(gimple stmt) +{ + return stmt; +} + +static inline const ggoto *as_a_const_ggoto(const_gimple stmt) +{ + return stmt; +} + static inline gphi *as_a_gphi(gimple stmt) { return stmt; @@ -496,6 +529,14 @@ static inline const greturn *as_a_const_greturn(const_gimple stmt) typedef struct rtx_def rtx_insn; +static inline const char *get_decl_section_name(const_tree decl) +{ + if (DECL_SECTION_NAME(decl) == NULL_TREE) + return NULL; + + return TREE_STRING_POINTER(DECL_SECTION_NAME(decl)); +} + static inline void set_decl_section_name(tree node, const char *value) { if (value) @@ -511,6 +552,7 @@ typedef struct gimple_statement_base gassign; typedef struct gimple_statement_call gcall; typedef struct gimple_statement_base gcond; typedef struct gimple_statement_base gdebug; +typedef struct gimple_statement_base ggoto; typedef struct gimple_statement_phi gphi; typedef struct gimple_statement_base greturn; @@ -564,6 +606,16 @@ static inline const gdebug *as_a_const_gdebug(const_gimple stmt) return stmt; } +static inline ggoto *as_a_ggoto(gimple stmt) +{ + return stmt; +} + +static inline const ggoto *as_a_const_ggoto(const_gimple stmt) +{ + return stmt; +} + static inline gphi *as_a_gphi(gimple stmt) { return as_a(stmt); @@ -611,6 +663,11 @@ inline bool is_a_helper::test(const_gimple gs) #define INSN_DELETED_P(insn) (insn)->deleted() +static inline const char *get_decl_section_name(const_tree decl) +{ + return DECL_SECTION_NAME(decl); +} + /* symtab/cgraph related */ #define debug_cgraph_node(node) (node)->debug() #define cgraph_get_node(decl) cgraph_node::get(decl) @@ -619,6 +676,7 @@ inline bool is_a_helper::test(const_gimple gs) #define cgraph_n_nodes symtab->cgraph_count #define cgraph_max_uid symtab->cgraph_max_uid #define varpool_get_node(decl) varpool_node::get(decl) +#define dump_varpool_node(file, node) (node)->dump(file) #define cgraph_create_edge(caller, callee, call_stmt, count, freq, nest) \ (caller)->create_edge((callee), (call_stmt), (count), (freq)) @@ -674,6 +732,11 @@ static inline cgraph_node_ptr cgraph_alias_target(cgraph_node_ptr node) return node->get_alias_target(); } +static inline bool cgraph_for_node_and_aliases(cgraph_node_ptr node, bool (*callback)(cgraph_node_ptr, void *), void *data, bool include_overwritable) +{ + return node->call_for_symbol_thunks_and_aliases(callback, data, include_overwritable); +} + static inline struct cgraph_node_hook_list *cgraph_add_function_insertion_hook(cgraph_node_hook hook, void *data) { return symtab->add_cgraph_insertion_hook(hook, data); @@ -729,6 +792,13 @@ static inline gimple gimple_build_assign_with_ops(enum tree_code subcode, tree l return gimple_build_assign(lhs, subcode, op1, op2 PASS_MEM_STAT); } +template <> +template <> +inline bool is_a_helper::test(const_gimple gs) +{ + return gs->code == GIMPLE_GOTO; +} + template <> template <> inline bool is_a_helper::test(const_gimple gs) @@ -766,6 +836,16 @@ static inline const gcall *as_a_const_gcall(const_gimple stmt) return as_a(stmt); } +static inline ggoto *as_a_ggoto(gimple stmt) +{ + return as_a(stmt); +} + +static inline const ggoto *as_a_const_ggoto(const_gimple stmt) +{ + return as_a(stmt); +} + static inline gphi *as_a_gphi(gimple stmt) { return as_a(stmt); @@ -828,4 +908,9 @@ static inline void debug_gimple_stmt(const_gimple s) #define debug_gimple_stmt(s) debug_gimple_stmt(CONST_CAST_GIMPLE(s)) #endif +#if BUILDING_GCC_VERSION >= 7000 +#define get_inner_reference(exp, pbitsize, pbitpos, poffset, pmode, punsignedp, preversep, pvolatilep, keep_aligning) \ + get_inner_reference(exp, pbitsize, pbitpos, poffset, pmode, punsignedp, preversep, pvolatilep) +#endif + #endif diff --git a/scripts/gcc-plugins/latent_entropy_plugin.c b/scripts/gcc-plugins/latent_entropy_plugin.c index 8160f1c1b56ed9..dff390f692a2c6 100644 --- a/scripts/gcc-plugins/latent_entropy_plugin.c +++ b/scripts/gcc-plugins/latent_entropy_plugin.c @@ -328,9 +328,9 @@ static enum tree_code get_op(tree *rhs) op = LROTATE_EXPR; /* * This code limits the value of random_const to - * the size of a wide int for the rotation + * the size of a long for the rotation */ - random_const &= HOST_BITS_PER_WIDE_INT - 1; + random_const %= TYPE_PRECISION(long_unsigned_type_node); break; } diff --git a/scripts/kconfig/nconf.gui.c b/scripts/kconfig/nconf.gui.c index 8275f0e55106bc..4b2f44c20caf89 100644 --- a/scripts/kconfig/nconf.gui.c +++ b/scripts/kconfig/nconf.gui.c @@ -364,12 +364,14 @@ int dialog_inputbox(WINDOW *main_window, WINDOW *prompt_win; WINDOW *form_win; PANEL *panel; - int i, x, y; + int i, x, y, lines, columns, win_lines, win_cols; int res = -1; int cursor_position = strlen(init); int cursor_form_win; char *result = *resultp; + getmaxyx(stdscr, lines, columns); + if (strlen(init)+1 > *result_len) { *result_len = strlen(init)+1; *resultp = result = realloc(result, *result_len); @@ -386,14 +388,19 @@ int dialog_inputbox(WINDOW *main_window, if (title) prompt_width = max(prompt_width, strlen(title)); + win_lines = min(prompt_lines+6, lines-2); + win_cols = min(prompt_width+7, columns-2); + prompt_lines = max(win_lines-6, 0); + prompt_width = max(win_cols-7, 0); + /* place dialog in middle of screen */ - y = (getmaxy(stdscr)-(prompt_lines+4))/2; - x = (getmaxx(stdscr)-(prompt_width+4))/2; + y = (lines-win_lines)/2; + x = (columns-win_cols)/2; strncpy(result, init, *result_len); /* create the windows */ - win = newwin(prompt_lines+6, prompt_width+7, y, x); + win = newwin(win_lines, win_cols, y, x); prompt_win = derwin(win, prompt_lines+1, prompt_width, 2, 2); form_win = derwin(win, 1, prompt_width, prompt_lines+3, 2); keypad(form_win, TRUE); diff --git a/scripts/package/builddeb b/scripts/package/builddeb index 8ea9fd2b65736c..3c575cd0788880 100755 --- a/scripts/package/builddeb +++ b/scripts/package/builddeb @@ -51,7 +51,7 @@ set_debarch() { debarch=hppa ;; mips*) debarch=mips$(grep -q CPU_LITTLE_ENDIAN=y $KCONFIG_CONFIG && echo el || true) ;; - arm64) + aarch64|arm64) debarch=arm64 ;; arm*) if grep -q CONFIG_AEABI=y $KCONFIG_CONFIG; then diff --git a/security/Kconfig b/security/Kconfig index 118f4549404ef2..59aea7df12b04d 100644 --- a/security/Kconfig +++ b/security/Kconfig @@ -18,6 +18,15 @@ config SECURITY_DMESG_RESTRICT If you are unsure how to answer this question, answer N. +config SECURITY_PERF_EVENTS_RESTRICT + bool "Restrict unprivileged use of performance events" + depends on PERF_EVENTS + help + If you say Y here, the kernel.perf_event_paranoid sysctl + will be set to 3 by default, and no unprivileged use of the + perf_event_open syscall will be permitted unless it is + changed. + config SECURITY bool "Enable different security models" depends on SYSFS diff --git a/security/commoncap.c b/security/commoncap.c index 8df676fbd39366..a8e4aacf0b0c0e 100644 --- a/security/commoncap.c +++ b/security/commoncap.c @@ -31,6 +31,10 @@ #include #include +#ifdef CONFIG_ANDROID_PARANOID_NETWORK +#include +#endif + /* * If a non-root user executes a setuid-root binary in * !secure(SECURE_NOROOT) mode, then we raise capabilities. @@ -73,6 +77,13 @@ int cap_capable(const struct cred *cred, struct user_namespace *targ_ns, { struct user_namespace *ns = targ_ns; +#ifdef CONFIG_ANDROID_PARANOID_NETWORK + if (cap == CAP_NET_RAW && in_egroup_p(AID_NET_RAW)) + return 0; + if (cap == CAP_NET_ADMIN && in_egroup_p(AID_NET_ADMIN)) + return 0; +#endif + /* See if cred has the capability in the target user namespace * by examining the target user namespace and all of the target * user namespace's parents. diff --git a/security/inode.c b/security/inode.c index c83db05c15aba9..b4531f2be0f1bb 100644 --- a/security/inode.c +++ b/security/inode.c @@ -100,7 +100,7 @@ struct dentry *securityfs_create_file(const char *name, umode_t mode, dir = d_inode(parent); inode_lock(dir); - dentry = lookup_one_len(name, parent, strlen(name)); + dentry = lookup_one_len2(name, mount, parent, strlen(name)); if (IS_ERR(dentry)) goto out; diff --git a/security/integrity/ima/ima_fs.c b/security/integrity/ima/ima_fs.c index c07a3844ea0a29..3df46906492dec 100644 --- a/security/integrity/ima/ima_fs.c +++ b/security/integrity/ima/ima_fs.c @@ -401,7 +401,7 @@ static int ima_release_policy(struct inode *inode, struct file *file) const char *cause = valid_policy ? "completed" : "failed"; if ((file->f_flags & O_ACCMODE) == O_RDONLY) - return 0; + return seq_release(inode, file); if (valid_policy && ima_check_policy() < 0) { cause = "failed"; diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 09fd6108e42134..8b918f8d099b73 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -491,6 +491,7 @@ static int selinux_is_sblabel_mnt(struct super_block *sb) !strcmp(sb->s_type->name, "sysfs") || !strcmp(sb->s_type->name, "pstore") || !strcmp(sb->s_type->name, "debugfs") || + !strcmp(sb->s_type->name, "tracefs") || !strcmp(sb->s_type->name, "rootfs"); } @@ -5858,7 +5859,7 @@ static int selinux_setprocattr(struct task_struct *p, return error; /* Obtain a SID for the context, if one was specified. */ - if (size && str[1] && str[1] != '\n') { + if (size && str[0] && str[0] != '\n') { if (str[size-1] == '\n') { str[size-1] = 0; size--; diff --git a/sound/core/seq/seq_memory.c b/sound/core/seq/seq_memory.c index c850345c43b53d..dfa5156f358563 100644 --- a/sound/core/seq/seq_memory.c +++ b/sound/core/seq/seq_memory.c @@ -419,7 +419,6 @@ int snd_seq_pool_done(struct snd_seq_pool *pool) { unsigned long flags; struct snd_seq_event_cell *ptr; - int max_count = 5 * HZ; if (snd_BUG_ON(!pool)) return -EINVAL; @@ -432,14 +431,8 @@ int snd_seq_pool_done(struct snd_seq_pool *pool) if (waitqueue_active(&pool->output_sleep)) wake_up(&pool->output_sleep); - while (atomic_read(&pool->counter) > 0) { - if (max_count == 0) { - pr_warn("ALSA: snd_seq_pool_done timeout: %d cells remain\n", atomic_read(&pool->counter)); - break; - } + while (atomic_read(&pool->counter) > 0) schedule_timeout_uninterruptible(1); - max_count--; - } /* release all resources */ spin_lock_irqsave(&pool->lock, flags); diff --git a/sound/core/seq/seq_queue.c b/sound/core/seq/seq_queue.c index 0bec02e89d5118..450c5187eecb6b 100644 --- a/sound/core/seq/seq_queue.c +++ b/sound/core/seq/seq_queue.c @@ -181,6 +181,8 @@ void __exit snd_seq_queues_delete(void) } } +static void queue_use(struct snd_seq_queue *queue, int client, int use); + /* allocate a new queue - * return queue index value or negative value for error */ @@ -192,11 +194,11 @@ int snd_seq_queue_alloc(int client, int locked, unsigned int info_flags) if (q == NULL) return -ENOMEM; q->info_flags = info_flags; + queue_use(q, client, 1); if (queue_list_add(q) < 0) { queue_delete(q); return -ENOMEM; } - snd_seq_queue_use(q->queue, client, 1); /* use this queue */ return q->queue; } @@ -502,19 +504,9 @@ int snd_seq_queue_timer_set_tempo(int queueid, int client, return result; } - -/* use or unuse this queue - - * if it is the first client, starts the timer. - * if it is not longer used by any clients, stop the timer. - */ -int snd_seq_queue_use(int queueid, int client, int use) +/* use or unuse this queue */ +static void queue_use(struct snd_seq_queue *queue, int client, int use) { - struct snd_seq_queue *queue; - - queue = queueptr(queueid); - if (queue == NULL) - return -EINVAL; - mutex_lock(&queue->timer_mutex); if (use) { if (!test_and_set_bit(client, queue->clients_bitmap)) queue->clients++; @@ -529,6 +521,21 @@ int snd_seq_queue_use(int queueid, int client, int use) } else { snd_seq_timer_close(queue); } +} + +/* use or unuse this queue - + * if it is the first client, starts the timer. + * if it is not longer used by any clients, stop the timer. + */ +int snd_seq_queue_use(int queueid, int client, int use) +{ + struct snd_seq_queue *queue; + + queue = queueptr(queueid); + if (queue == NULL) + return -EINVAL; + mutex_lock(&queue->timer_mutex); + queue_use(queue, client, use); mutex_unlock(&queue->timer_mutex); queuefree(queue); return 0; diff --git a/sound/firewire/tascam/tascam-stream.c b/sound/firewire/tascam/tascam-stream.c index 4ad3bd7fd4453e..f1657a4e0621ef 100644 --- a/sound/firewire/tascam/tascam-stream.c +++ b/sound/firewire/tascam/tascam-stream.c @@ -343,7 +343,7 @@ int snd_tscm_stream_init_duplex(struct snd_tscm *tscm) if (err < 0) amdtp_stream_destroy(&tscm->rx_stream); - return 0; + return err; } /* At bus reset, streaming is stopped and some registers are clear. */ diff --git a/sound/pci/hda/hda_auto_parser.c b/sound/pci/hda/hda_auto_parser.c index 7f57a145a47e1f..a03cf68d0bcd6c 100644 --- a/sound/pci/hda/hda_auto_parser.c +++ b/sound/pci/hda/hda_auto_parser.c @@ -884,6 +884,8 @@ void snd_hda_apply_fixup(struct hda_codec *codec, int action) } EXPORT_SYMBOL_GPL(snd_hda_apply_fixup); +#define IGNORE_SEQ_ASSOC (~(AC_DEFCFG_SEQUENCE | AC_DEFCFG_DEF_ASSOC)) + static bool pin_config_match(struct hda_codec *codec, const struct hda_pintbl *pins) { @@ -901,7 +903,7 @@ static bool pin_config_match(struct hda_codec *codec, for (; t_pins->nid; t_pins++) { if (t_pins->nid == nid) { found = 1; - if (t_pins->val == cfg) + if ((t_pins->val & IGNORE_SEQ_ASSOC) == (cfg & IGNORE_SEQ_ASSOC)) break; else if ((cfg & 0xf0000000) == 0x40000000 && (t_pins->val & 0xf0000000) == 0x40000000) break; diff --git a/sound/pci/hda/patch_ca0132.c b/sound/pci/hda/patch_ca0132.c index ad06866d7c69e4..11b9b2f17a2ef6 100644 --- a/sound/pci/hda/patch_ca0132.c +++ b/sound/pci/hda/patch_ca0132.c @@ -780,6 +780,7 @@ static const struct hda_pintbl alienware_pincfgs[] = { static const struct snd_pci_quirk ca0132_quirks[] = { SND_PCI_QUIRK(0x1028, 0x0685, "Alienware 15 2015", QUIRK_ALIENWARE), SND_PCI_QUIRK(0x1028, 0x0688, "Alienware 17 2015", QUIRK_ALIENWARE), + SND_PCI_QUIRK(0x1028, 0x0708, "Alienware 15 R2 2016", QUIRK_ALIENWARE), {} }; diff --git a/sound/pci/hda/patch_conexant.c b/sound/pci/hda/patch_conexant.c index ed62748a6d55a8..c15c51bea26d0a 100644 --- a/sound/pci/hda/patch_conexant.c +++ b/sound/pci/hda/patch_conexant.c @@ -262,6 +262,7 @@ enum { CXT_FIXUP_CAP_MIX_AMP_5047, CXT_FIXUP_MUTE_LED_EAPD, CXT_FIXUP_HP_SPECTRE, + CXT_FIXUP_HP_GATE_MIC, }; /* for hda_fixup_thinkpad_acpi() */ @@ -633,6 +634,17 @@ static void cxt_fixup_cap_mix_amp_5047(struct hda_codec *codec, (1 << AC_AMPCAP_MUTE_SHIFT)); } +static void cxt_fixup_hp_gate_mic_jack(struct hda_codec *codec, + const struct hda_fixup *fix, + int action) +{ + /* the mic pin (0x19) doesn't give an unsolicited event; + * probe the mic pin together with the headphone pin (0x16) + */ + if (action == HDA_FIXUP_ACT_PROBE) + snd_hda_jack_set_gating_jack(codec, 0x19, 0x16); +} + /* ThinkPad X200 & co with cxt5051 */ static const struct hda_pintbl cxt_pincfg_lenovo_x200[] = { { 0x16, 0x042140ff }, /* HP (seq# overridden) */ @@ -774,6 +786,10 @@ static const struct hda_fixup cxt_fixups[] = { { } } }, + [CXT_FIXUP_HP_GATE_MIC] = { + .type = HDA_FIXUP_FUNC, + .v.func = cxt_fixup_hp_gate_mic_jack, + }, }; static const struct snd_pci_quirk cxt5045_fixups[] = { @@ -824,6 +840,7 @@ static const struct snd_pci_quirk cxt5066_fixups[] = { SND_PCI_QUIRK(0x1025, 0x054c, "Acer Aspire 3830TG", CXT_FIXUP_ASPIRE_DMIC), SND_PCI_QUIRK(0x1025, 0x054f, "Acer Aspire 4830T", CXT_FIXUP_ASPIRE_DMIC), SND_PCI_QUIRK(0x103c, 0x8174, "HP Spectre x360", CXT_FIXUP_HP_SPECTRE), + SND_PCI_QUIRK(0x103c, 0x8115, "HP Z1 Gen3", CXT_FIXUP_HP_GATE_MIC), SND_PCI_QUIRK(0x1043, 0x138d, "Asus", CXT_FIXUP_HEADPHONE_MIC_PIN), SND_PCI_QUIRK(0x152d, 0x0833, "OLPC XO-1.5", CXT_FIXUP_OLPC_XO), SND_PCI_QUIRK(0x17aa, 0x20f2, "Lenovo T400", CXT_PINCFG_LENOVO_TP410), diff --git a/sound/pci/hda/patch_hdmi.c b/sound/pci/hda/patch_hdmi.c index 56e5204ac9c1ee..4bf48336b0fc91 100644 --- a/sound/pci/hda/patch_hdmi.c +++ b/sound/pci/hda/patch_hdmi.c @@ -3638,6 +3638,7 @@ HDA_CODEC_ENTRY(0x10de0070, "GPU 70 HDMI/DP", patch_nvhdmi), HDA_CODEC_ENTRY(0x10de0071, "GPU 71 HDMI/DP", patch_nvhdmi), HDA_CODEC_ENTRY(0x10de0072, "GPU 72 HDMI/DP", patch_nvhdmi), HDA_CODEC_ENTRY(0x10de007d, "GPU 7d HDMI/DP", patch_nvhdmi), +HDA_CODEC_ENTRY(0x10de0080, "GPU 80 HDMI/DP", patch_nvhdmi), HDA_CODEC_ENTRY(0x10de0082, "GPU 82 HDMI/DP", patch_nvhdmi), HDA_CODEC_ENTRY(0x10de0083, "GPU 83 HDMI/DP", patch_nvhdmi), HDA_CODEC_ENTRY(0x10de8001, "MCP73 HDMI", patch_nvhdmi_2ch), diff --git a/sound/pci/hda/patch_realtek.c b/sound/pci/hda/patch_realtek.c index ea81c08ddc7acf..758ac86a1d3a21 100644 --- a/sound/pci/hda/patch_realtek.c +++ b/sound/pci/hda/patch_realtek.c @@ -2230,6 +2230,7 @@ static const struct snd_pci_quirk alc882_fixup_tbl[] = { SND_PCI_QUIRK(0x1043, 0x1971, "Asus W2JC", ALC882_FIXUP_ASUS_W2JC), SND_PCI_QUIRK(0x1043, 0x835f, "Asus Eee 1601", ALC888_FIXUP_EEE1601), SND_PCI_QUIRK(0x1043, 0x84bc, "ASUS ET2700", ALC887_FIXUP_ASUS_BASS), + SND_PCI_QUIRK(0x1043, 0x8691, "ASUS ROG Ranger VIII", ALC882_FIXUP_GPIO3), SND_PCI_QUIRK(0x104d, 0x9047, "Sony Vaio TT", ALC889_FIXUP_VAIO_TT), SND_PCI_QUIRK(0x104d, 0x905a, "Sony Vaio Z", ALC882_FIXUP_NO_PRIMARY_HP), SND_PCI_QUIRK(0x104d, 0x9043, "Sony Vaio VGC-LN51JGB", ALC882_FIXUP_NO_PRIMARY_HP), @@ -5917,6 +5918,9 @@ static const struct snd_hda_pin_quirk alc269_pin_fixup_tbl[] = { {0x12, 0x90a60180}, {0x14, 0x90170120}, {0x21, 0x02211030}), + SND_HDA_PIN_QUIRK(0x10ec0255, 0x1028, "Dell", ALC255_FIXUP_DELL1_MIC_NO_PRESENCE, + {0x1b, 0x01011020}, + {0x21, 0x02211010}), SND_HDA_PIN_QUIRK(0x10ec0256, 0x1028, "Dell", ALC255_FIXUP_DELL1_MIC_NO_PRESENCE, {0x12, 0x90a60160}, {0x14, 0x90170120}, @@ -6940,6 +6944,7 @@ static const struct snd_pci_quirk alc662_fixup_tbl[] = { SND_PCI_QUIRK(0x1043, 0x15a7, "ASUS UX51VZH", ALC662_FIXUP_BASS_16), SND_PCI_QUIRK(0x1043, 0x177d, "ASUS N551", ALC668_FIXUP_ASUS_Nx51), SND_PCI_QUIRK(0x1043, 0x17bd, "ASUS N751", ALC668_FIXUP_ASUS_Nx51), + SND_PCI_QUIRK(0x1043, 0x1963, "ASUS X71SL", ALC662_FIXUP_ASUS_MODE8), SND_PCI_QUIRK(0x1043, 0x1b73, "ASUS N55SF", ALC662_FIXUP_BASS_16), SND_PCI_QUIRK(0x1043, 0x1bf3, "ASUS N76VZ", ALC662_FIXUP_BASS_MODE4_CHMAP), SND_PCI_QUIRK(0x1043, 0x8469, "ASUS mobo", ALC662_FIXUP_NO_JACK_DETECT), diff --git a/sound/soc/Kconfig b/sound/soc/Kconfig index 182d92efc7c801..9df9658b552b8d 100644 --- a/sound/soc/Kconfig +++ b/sound/soc/Kconfig @@ -47,6 +47,7 @@ source "sound/soc/cirrus/Kconfig" source "sound/soc/davinci/Kconfig" source "sound/soc/dwc/Kconfig" source "sound/soc/fsl/Kconfig" +source "sound/soc/hisilicon/Kconfig" source "sound/soc/jz4740/Kconfig" source "sound/soc/nuc900/Kconfig" source "sound/soc/omap/Kconfig" diff --git a/sound/soc/Makefile b/sound/soc/Makefile index 9a30f21d16ee8e..2f6aabb8b4c343 100644 --- a/sound/soc/Makefile +++ b/sound/soc/Makefile @@ -27,6 +27,7 @@ obj-$(CONFIG_SND_SOC) += cirrus/ obj-$(CONFIG_SND_SOC) += davinci/ obj-$(CONFIG_SND_SOC) += dwc/ obj-$(CONFIG_SND_SOC) += fsl/ +obj-$(CONFIG_SND_SOC) += hisilicon/ obj-$(CONFIG_SND_SOC) += jz4740/ obj-$(CONFIG_SND_SOC) += img/ obj-$(CONFIG_SND_SOC) += intel/ diff --git a/sound/soc/hisilicon/Kconfig b/sound/soc/hisilicon/Kconfig new file mode 100644 index 00000000000000..4356d5a1d338f6 --- /dev/null +++ b/sound/soc/hisilicon/Kconfig @@ -0,0 +1,5 @@ +config SND_I2S_HI6210_I2S + tristate "Hisilicon I2S controller" + select SND_SOC_GENERIC_DMAENGINE_PCM + help + Hisilicon I2S diff --git a/sound/soc/hisilicon/Makefile b/sound/soc/hisilicon/Makefile new file mode 100644 index 00000000000000..e8095e2af91a50 --- /dev/null +++ b/sound/soc/hisilicon/Makefile @@ -0,0 +1 @@ +obj-$(CONFIG_SND_I2S_HI6210_I2S) += hi6210-i2s.o diff --git a/sound/soc/hisilicon/hi6210-i2s.c b/sound/soc/hisilicon/hi6210-i2s.c new file mode 100644 index 00000000000000..a05888ac6672c8 --- /dev/null +++ b/sound/soc/hisilicon/hi6210-i2s.c @@ -0,0 +1,628 @@ +/* + * linux/sound/soc/m8m/hi6210_i2s.c - I2S IP driver + * + * Copyright (C) 2015 Linaro, Ltd + * Author: Andy Green + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * This driver only deals with S2 interface (BT) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hi6210-i2s.h" + +struct hi6210_i2s { + struct device *dev; + struct reset_control *rc; + struct clk *clk[8]; + int clocks; + struct snd_soc_dai_driver dai; + void __iomem *base; + struct regmap *sysctrl; + phys_addr_t base_phys; + struct snd_dmaengine_dai_dma_data dma_data[2]; + int clk_rate; + spinlock_t lock; + int rate; + int format; + u8 bits; + u8 channels; + u8 id; + u8 channel_length; + u8 use; + u32 master:1; + u32 status:1; +}; + +#define SC_PERIPH_CLKEN1 0x210 +#define SC_PERIPH_CLKDIS1 0x214 + +#define SC_PERIPH_CLKEN3 0x230 +#define SC_PERIPH_CLKDIS3 0x234 + +#define SC_PERIPH_CLKEN12 0x270 +#define SC_PERIPH_CLKDIS12 0x274 + +#define SC_PERIPH_RSTEN1 0x310 +#define SC_PERIPH_RSTDIS1 0x314 +#define SC_PERIPH_RSTSTAT1 0x318 + +#define SC_PERIPH_RSTEN2 0x320 +#define SC_PERIPH_RSTDIS2 0x324 +#define SC_PERIPH_RSTSTAT2 0x328 + +#define SOC_PMCTRL_BBPPLLALIAS 0x48 + +enum { + CLK_DACODEC, + CLK_I2S_BASE, +}; + +static inline void hi6210_write_reg(struct hi6210_i2s *i2s, int reg, u32 val) +{ + writel(val, i2s->base + reg); +} + +static inline u32 hi6210_read_reg(struct hi6210_i2s *i2s, int reg) +{ + return readl(i2s->base + reg); +} + +int hi6210_i2s_startup(struct snd_pcm_substream *substream, + struct snd_soc_dai *cpu_dai) +{ + struct hi6210_i2s *i2s = dev_get_drvdata(cpu_dai->dev); + int ret, n; + u32 val; + + /* deassert reset on ABB */ + regmap_read(i2s->sysctrl, SC_PERIPH_RSTSTAT2, &val); + if (val & BIT(4)) + regmap_write(i2s->sysctrl, SC_PERIPH_RSTDIS2, BIT(4)); + + for (n = 0; n < i2s->clocks; n++) { + ret = clk_prepare_enable(i2s->clk[n]); + if (ret) { + while (n--) + clk_disable_unprepare(i2s->clk[n]); + return ret; + } + } + + ret = clk_set_rate(i2s->clk[CLK_I2S_BASE], 49152000); + if (ret) { + dev_err(i2s->dev, "%s: setting 49.152MHz base rate failed %d\n", + __func__, ret); + return ret; + } + + /* enable clock before frequency division */ + regmap_write(i2s->sysctrl, SC_PERIPH_CLKEN12, BIT(9)); + + /* enable codec working clock / == "codec bus clock" */ + regmap_write(i2s->sysctrl, SC_PERIPH_CLKEN1, BIT(5)); + + /* deassert reset on codec / interface clock / working clock */ + regmap_write(i2s->sysctrl, SC_PERIPH_RSTEN1, BIT(5)); + regmap_write(i2s->sysctrl, SC_PERIPH_RSTDIS1, BIT(5)); + + /* not interested in i2s irqs */ + val = hi6210_read_reg(i2s, HII2S_CODEC_IRQ_MASK); + val |= 0x3f; + hi6210_write_reg(i2s, HII2S_CODEC_IRQ_MASK, val); + + + /* reset the stereo downlink fifo */ + val = hi6210_read_reg(i2s, HII2S_APB_AFIFO_CFG_1); + val |= (BIT(5) | BIT(4)); + hi6210_write_reg(i2s, HII2S_APB_AFIFO_CFG_1, val); + + val = hi6210_read_reg(i2s, HII2S_APB_AFIFO_CFG_1); + val &= ~(BIT(5) | BIT(4)); + hi6210_write_reg(i2s, HII2S_APB_AFIFO_CFG_1, val); + + + val = hi6210_read_reg(i2s, HII2S_SW_RST_N); + val &= ~(HII2S_SW_RST_N__ST_DL_WORDLEN_MASK << + HII2S_SW_RST_N__ST_DL_WORDLEN_SHIFT); + val |= (HII2S_BITS_16 << HII2S_SW_RST_N__ST_DL_WORDLEN_SHIFT); + hi6210_write_reg(i2s, HII2S_SW_RST_N, val); + + val = hi6210_read_reg(i2s, HII2S_MISC_CFG); + /* mux 11/12 = APB not i2s */ + val &= ~HII2S_MISC_CFG__ST_DL_TEST_SEL; + /* BT R ch 0 = mixer op of DACR ch */ + val &= ~HII2S_MISC_CFG__S2_DOUT_RIGHT_SEL; + val &= ~HII2S_MISC_CFG__S2_DOUT_TEST_SEL; + + val |= HII2S_MISC_CFG__S2_DOUT_RIGHT_SEL; + /* BT L ch = 1 = mux 7 = "mixer output of DACL */ + val |= HII2S_MISC_CFG__S2_DOUT_TEST_SEL; + hi6210_write_reg(i2s, HII2S_MISC_CFG, val); + + val = hi6210_read_reg(i2s, HII2S_SW_RST_N); + val |= HII2S_SW_RST_N__SW_RST_N; + hi6210_write_reg(i2s, HII2S_SW_RST_N, val); + + return 0; +} +void hi6210_i2s_shutdown(struct snd_pcm_substream *substream, + struct snd_soc_dai *cpu_dai) +{ + struct hi6210_i2s *i2s = dev_get_drvdata(cpu_dai->dev); + int n; + + for (n = 0; n < i2s->clocks; n++) + clk_disable_unprepare(i2s->clk[n]); + + regmap_write(i2s->sysctrl, SC_PERIPH_RSTEN1, BIT(5)); +} + +static void hi6210_i2s_txctrl(struct snd_soc_dai *cpu_dai, int on) +{ + struct hi6210_i2s *i2s = dev_get_drvdata(cpu_dai->dev); + u32 val; + + spin_lock(&i2s->lock); + if (on) { + /* enable S2 TX */ + val = hi6210_read_reg(i2s, HII2S_I2S_CFG); + val |= HII2S_I2S_CFG__S2_IF_TX_EN; + hi6210_write_reg(i2s, HII2S_I2S_CFG, val); + } else { + /* disable S2 TX */ + val = hi6210_read_reg(i2s, HII2S_I2S_CFG); + val &= ~HII2S_I2S_CFG__S2_IF_TX_EN; + hi6210_write_reg(i2s, HII2S_I2S_CFG, val); + } + spin_unlock(&i2s->lock); +} + +static void hi6210_i2s_rxctrl(struct snd_soc_dai *cpu_dai, int on) +{ + struct hi6210_i2s *i2s = dev_get_drvdata(cpu_dai->dev); + u32 val; + + spin_lock(&i2s->lock); + if (on) { + val = hi6210_read_reg(i2s, HII2S_I2S_CFG); + val |= HII2S_I2S_CFG__S2_IF_RX_EN; + hi6210_write_reg(i2s, HII2S_I2S_CFG, val); + } else { + val = hi6210_read_reg(i2s, HII2S_I2S_CFG); + val &= ~HII2S_I2S_CFG__S2_IF_RX_EN; + hi6210_write_reg(i2s, HII2S_I2S_CFG, val); + } + spin_unlock(&i2s->lock); +} + +static int hi6210_i2s_set_fmt(struct snd_soc_dai *cpu_dai, unsigned int fmt) +{ + struct hi6210_i2s *i2s = dev_get_drvdata(cpu_dai->dev); + + /* + * We don't actually set the hardware until the hw_params + * call, but we need to validate the user input here. + */ + switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) { + case SND_SOC_DAIFMT_CBM_CFM: + case SND_SOC_DAIFMT_CBS_CFS: + break; + default: + return -EINVAL; + } + + switch (fmt & SND_SOC_DAIFMT_FORMAT_MASK) { + case SND_SOC_DAIFMT_I2S: + case SND_SOC_DAIFMT_LEFT_J: + case SND_SOC_DAIFMT_RIGHT_J: + break; + default: + return -EINVAL; + } + + i2s->format = fmt; + i2s->master = (i2s->format & SND_SOC_DAIFMT_MASTER_MASK) == + SND_SOC_DAIFMT_CBS_CFS; + + return 0; +} + +static int hi6210_i2s_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params, + struct snd_soc_dai *cpu_dai) +{ + struct hi6210_i2s *i2s = dev_get_drvdata(cpu_dai->dev); + u32 bits = 0, rate = 0, signed_data = 0, fmt = 0; + u32 val; + struct snd_dmaengine_dai_dma_data *dma_data; + + switch (params_format(params)) { + case SNDRV_PCM_FORMAT_U16_LE: + signed_data = HII2S_I2S_CFG__S2_CODEC_DATA_FORMAT; + /* fallthru */ + case SNDRV_PCM_FORMAT_S16_LE: + bits = HII2S_BITS_16; + break; + case SNDRV_PCM_FORMAT_U24_LE: + signed_data = HII2S_I2S_CFG__S2_CODEC_DATA_FORMAT; + /* fallthru */ + case SNDRV_PCM_FORMAT_S24_LE: + bits = HII2S_BITS_24; + break; + default: + dev_err(cpu_dai->dev, "Bad format\n"); + return -EINVAL; + } + + + switch (params_rate(params)) { + case 8000: + rate = HII2S_FS_RATE_8KHZ; + break; + case 16000: + rate = HII2S_FS_RATE_16KHZ; + break; + case 32000: + rate = HII2S_FS_RATE_32KHZ; + break; + case 48000: + rate = HII2S_FS_RATE_48KHZ; + break; + case 96000: + rate = HII2S_FS_RATE_96KHZ; + break; + case 192000: + rate = HII2S_FS_RATE_192KHZ; + break; + default: + dev_err(cpu_dai->dev, "Bad rate: %d\n", params_rate(params)); + return -EINVAL; + } + + if (!(params_channels(params))) { + dev_err(cpu_dai->dev, "Bad channels\n"); + return -EINVAL; + } + + dma_data = snd_soc_dai_get_dma_data(cpu_dai, substream); + + switch (bits) { + case HII2S_BITS_24: + i2s->bits = 32; + dma_data->addr_width = 3; + break; + default: + i2s->bits = 16; + dma_data->addr_width = 2; + } + i2s->rate = params_rate(params); + i2s->channels = params_channels(params); + i2s->channel_length = i2s->channels * i2s->bits; + + val = hi6210_read_reg(i2s, HII2S_ST_DL_FIFO_TH_CFG); + val &= ~((HII2S_ST_DL_FIFO_TH_CFG__ST_DL_R_AEMPTY_MASK << + HII2S_ST_DL_FIFO_TH_CFG__ST_DL_R_AEMPTY_SHIFT) | + (HII2S_ST_DL_FIFO_TH_CFG__ST_DL_R_AFULL_MASK << + HII2S_ST_DL_FIFO_TH_CFG__ST_DL_R_AFULL_SHIFT) | + (HII2S_ST_DL_FIFO_TH_CFG__ST_DL_L_AEMPTY_MASK << + HII2S_ST_DL_FIFO_TH_CFG__ST_DL_L_AEMPTY_SHIFT) | + (HII2S_ST_DL_FIFO_TH_CFG__ST_DL_L_AFULL_MASK << + HII2S_ST_DL_FIFO_TH_CFG__ST_DL_L_AFULL_SHIFT)); + val |= ((16 << HII2S_ST_DL_FIFO_TH_CFG__ST_DL_R_AEMPTY_SHIFT) | + (30 << HII2S_ST_DL_FIFO_TH_CFG__ST_DL_R_AFULL_SHIFT) | + (16 << HII2S_ST_DL_FIFO_TH_CFG__ST_DL_L_AEMPTY_SHIFT) | + (30 << HII2S_ST_DL_FIFO_TH_CFG__ST_DL_L_AFULL_SHIFT)); + hi6210_write_reg(i2s, HII2S_ST_DL_FIFO_TH_CFG, val); + + + val = hi6210_read_reg(i2s, HII2S_IF_CLK_EN_CFG); + val |= (BIT(19) | BIT(18) | BIT(17) | + HII2S_IF_CLK_EN_CFG__S2_IF_CLK_EN | + HII2S_IF_CLK_EN_CFG__S2_OL_MIXER_EN | + HII2S_IF_CLK_EN_CFG__S2_OL_SRC_EN | + HII2S_IF_CLK_EN_CFG__ST_DL_R_EN | + HII2S_IF_CLK_EN_CFG__ST_DL_L_EN); + hi6210_write_reg(i2s, HII2S_IF_CLK_EN_CFG, val); + + + val = hi6210_read_reg(i2s, HII2S_DIG_FILTER_CLK_EN_CFG); + val &= ~(HII2S_DIG_FILTER_CLK_EN_CFG__DACR_SDM_EN | + HII2S_DIG_FILTER_CLK_EN_CFG__DACR_HBF2I_EN | + HII2S_DIG_FILTER_CLK_EN_CFG__DACR_AGC_EN | + HII2S_DIG_FILTER_CLK_EN_CFG__DACL_SDM_EN | + HII2S_DIG_FILTER_CLK_EN_CFG__DACL_HBF2I_EN | + HII2S_DIG_FILTER_CLK_EN_CFG__DACL_AGC_EN); + val |= (HII2S_DIG_FILTER_CLK_EN_CFG__DACR_MIXER_EN | + HII2S_DIG_FILTER_CLK_EN_CFG__DACL_MIXER_EN); + hi6210_write_reg(i2s, HII2S_DIG_FILTER_CLK_EN_CFG, val); + + + val = hi6210_read_reg(i2s, HII2S_DIG_FILTER_MODULE_CFG); + val &= ~(HII2S_DIG_FILTER_MODULE_CFG__DACR_MIXER_IN2_MUTE | + HII2S_DIG_FILTER_MODULE_CFG__DACL_MIXER_IN2_MUTE); + hi6210_write_reg(i2s, HII2S_DIG_FILTER_MODULE_CFG, val); + + val = hi6210_read_reg(i2s, HII2S_MUX_TOP_MODULE_CFG); + val &= ~(HII2S_MUX_TOP_MODULE_CFG__S2_OL_MIXER_IN1_MUTE | + HII2S_MUX_TOP_MODULE_CFG__S2_OL_MIXER_IN2_MUTE | + HII2S_MUX_TOP_MODULE_CFG__VOICE_DLINK_MIXER_IN1_MUTE | + HII2S_MUX_TOP_MODULE_CFG__VOICE_DLINK_MIXER_IN2_MUTE); + hi6210_write_reg(i2s, HII2S_MUX_TOP_MODULE_CFG, val); + + + switch (i2s->format & SND_SOC_DAIFMT_MASTER_MASK) { + case SND_SOC_DAIFMT_CBM_CFM: + i2s->master = false; + val = hi6210_read_reg(i2s, HII2S_I2S_CFG); + val |= HII2S_I2S_CFG__S2_MST_SLV; + hi6210_write_reg(i2s, HII2S_I2S_CFG, val); + break; + case SND_SOC_DAIFMT_CBS_CFS: + i2s->master = true; + val = hi6210_read_reg(i2s, HII2S_I2S_CFG); + val &= ~HII2S_I2S_CFG__S2_MST_SLV; + hi6210_write_reg(i2s, HII2S_I2S_CFG, val); + break; + default: + WARN_ONCE(1, "Invalid i2s->fmt MASTER_MASK. This shouldn't happen\n"); + } + + switch (i2s->format & SND_SOC_DAIFMT_FORMAT_MASK) { + case SND_SOC_DAIFMT_I2S: + fmt = HII2S_FORMAT_I2S; + break; + case SND_SOC_DAIFMT_LEFT_J: + fmt = HII2S_FORMAT_LEFT_JUST; + break; + case SND_SOC_DAIFMT_RIGHT_J: + fmt = HII2S_FORMAT_RIGHT_JUST; + break; + default: + WARN_ONCE(1, "Invalid i2s->fmt FORMAT_MASK. This shouldn't happen\n"); + } + + val = hi6210_read_reg(i2s, HII2S_I2S_CFG); + val &= ~(HII2S_I2S_CFG__S2_FUNC_MODE_MASK << + HII2S_I2S_CFG__S2_FUNC_MODE_SHIFT); + val |= fmt << HII2S_I2S_CFG__S2_FUNC_MODE_SHIFT; + hi6210_write_reg(i2s, HII2S_I2S_CFG, val); + + + val = hi6210_read_reg(i2s, HII2S_CLK_SEL); + val &= ~(HII2S_CLK_SEL__I2S_BT_FM_SEL | /* BT gets the I2S */ + HII2S_CLK_SEL__EXT_12_288MHZ_SEL); + hi6210_write_reg(i2s, HII2S_CLK_SEL, val); + + dma_data->maxburst = 2; + + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) + dma_data->addr = i2s->base_phys + HII2S_ST_DL_CHANNEL; + else + dma_data->addr = i2s->base_phys + HII2S_STEREO_UPLINK_CHANNEL; + + switch (i2s->channels) { + case 1: + val = hi6210_read_reg(i2s, HII2S_I2S_CFG); + val |= HII2S_I2S_CFG__S2_FRAME_MODE; + hi6210_write_reg(i2s, HII2S_I2S_CFG, val); + break; + default: + val = hi6210_read_reg(i2s, HII2S_I2S_CFG); + val &= ~HII2S_I2S_CFG__S2_FRAME_MODE; + hi6210_write_reg(i2s, HII2S_I2S_CFG, val); + } + + /* clear loopback, set signed type and word length */ + val = hi6210_read_reg(i2s, HII2S_I2S_CFG); + val &= ~HII2S_I2S_CFG__S2_CODEC_DATA_FORMAT; + val &= ~(HII2S_I2S_CFG__S2_CODEC_IO_WORDLENGTH_MASK << + HII2S_I2S_CFG__S2_CODEC_IO_WORDLENGTH_SHIFT); + val &= ~(HII2S_I2S_CFG__S2_DIRECT_LOOP_MASK << + HII2S_I2S_CFG__S2_DIRECT_LOOP_SHIFT); + val |= signed_data; + val |= (bits << HII2S_I2S_CFG__S2_CODEC_IO_WORDLENGTH_SHIFT); + hi6210_write_reg(i2s, HII2S_I2S_CFG, val); + + + if (!i2s->master) + return 0; + + /* set DAC and related units to correct rate */ + val = hi6210_read_reg(i2s, HII2S_FS_CFG); + val &= ~(HII2S_FS_CFG__FS_S2_MASK << HII2S_FS_CFG__FS_S2_SHIFT); + val &= ~(HII2S_FS_CFG__FS_DACLR_MASK << HII2S_FS_CFG__FS_DACLR_SHIFT); + val &= ~(HII2S_FS_CFG__FS_ST_DL_R_MASK << + HII2S_FS_CFG__FS_ST_DL_R_SHIFT); + val &= ~(HII2S_FS_CFG__FS_ST_DL_L_MASK << + HII2S_FS_CFG__FS_ST_DL_L_SHIFT); + val |= (rate << HII2S_FS_CFG__FS_S2_SHIFT); + val |= (rate << HII2S_FS_CFG__FS_DACLR_SHIFT); + val |= (rate << HII2S_FS_CFG__FS_ST_DL_R_SHIFT); + val |= (rate << HII2S_FS_CFG__FS_ST_DL_L_SHIFT); + hi6210_write_reg(i2s, HII2S_FS_CFG, val); + + return 0; +} + +static int hi6210_i2s_trigger(struct snd_pcm_substream *substream, int cmd, + struct snd_soc_dai *cpu_dai) +{ + pr_debug("%s\n", __func__); + switch (cmd) { + case SNDRV_PCM_TRIGGER_START: + case SNDRV_PCM_TRIGGER_PAUSE_RELEASE: + if (substream->stream == SNDRV_PCM_STREAM_CAPTURE) + hi6210_i2s_rxctrl(cpu_dai, 1); + else + hi6210_i2s_txctrl(cpu_dai, 1); + break; + case SNDRV_PCM_TRIGGER_STOP: + case SNDRV_PCM_TRIGGER_PAUSE_PUSH: + if (substream->stream == SNDRV_PCM_STREAM_CAPTURE) + hi6210_i2s_rxctrl(cpu_dai, 0); + else + hi6210_i2s_txctrl(cpu_dai, 0); + break; + default: + dev_err(cpu_dai->dev, "uknown cmd\n"); + return -EINVAL; + } + return 0; +} + +static int hi6210_i2s_dai_probe(struct snd_soc_dai *dai) +{ + struct hi6210_i2s *i2s = snd_soc_dai_get_drvdata(dai); + + snd_soc_dai_init_dma_data(dai, + &i2s->dma_data[SNDRV_PCM_STREAM_PLAYBACK], + &i2s->dma_data[SNDRV_PCM_STREAM_CAPTURE]); + + return 0; +} + + +static struct snd_soc_dai_ops hi6210_i2s_dai_ops = { + .trigger = hi6210_i2s_trigger, + .hw_params = hi6210_i2s_hw_params, + .set_fmt = hi6210_i2s_set_fmt, + .startup = hi6210_i2s_startup, + .shutdown = hi6210_i2s_shutdown, +}; + +struct snd_soc_dai_driver hi6210_i2s_dai_init = { + .name = "hi6210_i2s", + .probe = hi6210_i2s_dai_probe, + .playback = { + .channels_min = 2, + .channels_max = 2, + .formats = SNDRV_PCM_FMTBIT_S16_LE | + SNDRV_PCM_FMTBIT_U16_LE, + .rates = SNDRV_PCM_RATE_48000, + }, + .capture = { + .channels_min = 2, + .channels_max = 2, + .formats = SNDRV_PCM_FMTBIT_S16_LE | + SNDRV_PCM_FMTBIT_U16_LE, + .rates = SNDRV_PCM_RATE_48000, + }, + .ops = &hi6210_i2s_dai_ops, +}; + +static const struct snd_soc_component_driver hi6210_i2s_i2s_comp = { + .name = "hi6210_i2s-i2s", +}; + +static int hi6210_i2s_probe(struct platform_device *pdev) +{ + struct device_node *node = pdev->dev.of_node; + struct device *dev = &pdev->dev; + struct hi6210_i2s *i2s; + struct resource *res; + int ret; + + i2s = devm_kzalloc(&pdev->dev, sizeof(*i2s), GFP_KERNEL); + if (!i2s) + return -ENOMEM; + + i2s->dev = dev; + spin_lock_init(&i2s->lock); + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + i2s->base = devm_ioremap_resource(dev, res); + if (IS_ERR(i2s->base)) + return PTR_ERR(i2s->base); + + i2s->base_phys = (phys_addr_t)res->start; + i2s->dai = hi6210_i2s_dai_init; + dev_set_drvdata(&pdev->dev, i2s); + + i2s->sysctrl = syscon_regmap_lookup_by_phandle(node, + "hisilicon,sysctrl-syscon"); + if (IS_ERR(i2s->sysctrl)) + return PTR_ERR(i2s->sysctrl); + + i2s->clk[CLK_DACODEC] = devm_clk_get(&pdev->dev, "dacodec"); + if (IS_ERR_OR_NULL(i2s->clk[CLK_DACODEC])) + return PTR_ERR(i2s->clk[CLK_DACODEC]); + i2s->clocks++; + + i2s->clk[CLK_I2S_BASE] = devm_clk_get(&pdev->dev, "i2s-base"); + if (IS_ERR_OR_NULL(i2s->clk[CLK_I2S_BASE])) + return PTR_ERR(i2s->clk[CLK_I2S_BASE]); + i2s->clocks++; + + ret = devm_snd_dmaengine_pcm_register(&pdev->dev, NULL, 0); + if (ret) + return ret; + + ret = snd_soc_register_component(&pdev->dev, &hi6210_i2s_i2s_comp, + &i2s->dai, 1); + if (ret) { + dev_err(&pdev->dev, "Failed to register dai\n"); + return ret; + } + + return 0; +} + +static int hi6210_i2s_remove(struct platform_device *pdev) +{ + snd_soc_unregister_component(&pdev->dev); + dev_set_drvdata(&pdev->dev, NULL); + + return 0; +} + +static const struct of_device_id hi6210_i2s_dt_ids[] = { + { .compatible = "hisilicon,hi6210-i2s" }, + { /* sentinel */ } +}; + +MODULE_DEVICE_TABLE(of, hi6210_i2s_dt_ids); + +static struct platform_driver hi6210_i2s_driver = { + .probe = hi6210_i2s_probe, + .remove = hi6210_i2s_remove, + .driver = { + .name = "hi6210_i2s", + .of_match_table = hi6210_i2s_dt_ids, + }, +}; + +module_platform_driver(hi6210_i2s_driver); + +MODULE_DESCRIPTION("Hisilicon HI6210 I2S driver"); +MODULE_AUTHOR("Andy Green "); +MODULE_LICENSE("GPL"); diff --git a/sound/soc/hisilicon/hi6210-i2s.h b/sound/soc/hisilicon/hi6210-i2s.h new file mode 100644 index 00000000000000..85cecc4939a008 --- /dev/null +++ b/sound/soc/hisilicon/hi6210-i2s.h @@ -0,0 +1,276 @@ +/* + * linux/sound/soc/hisilicon/hi6210-i2s.h + * + * Copyright (C) 2015 Linaro, Ltd + * Author: Andy Green + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Note at least on 6220, S2 == BT, S1 == Digital FM Radio IF + */ + +#ifndef _HI6210_I2S_H +#define _HI6210_I2S_H + +#define HII2S_SW_RST_N 0 + +#define HII2S_SW_RST_N__STEREO_UPLINK_WORDLEN_SHIFT 28 +#define HII2S_SW_RST_N__STEREO_UPLINK_WORDLEN_MASK 3 +#define HII2S_SW_RST_N__THIRDMD_UPLINK_WORDLEN_SHIFT 26 +#define HII2S_SW_RST_N__THIRDMD_UPLINK_WORDLEN_MASK 3 +#define HII2S_SW_RST_N__VOICE_UPLINK_WORDLEN_SHIFT 24 +#define HII2S_SW_RST_N__VOICE_UPLINK_WORDLEN_MASK 3 +#define HII2S_SW_RST_N__ST_DL_WORDLEN_SHIFT 20 +#define HII2S_SW_RST_N__ST_DL_WORDLEN_MASK 3 +#define HII2S_SW_RST_N__THIRDMD_DLINK_WORDLEN_SHIFT 18 +#define HII2S_SW_RST_N__THIRDMD_DLINK_WORDLEN_MASK 3 +#define HII2S_SW_RST_N__VOICE_DLINK_WORDLEN_SHIFT 16 +#define HII2S_SW_RST_N__VOICE_DLINK_WORDLEN_MASK 3 + +#define HII2S_SW_RST_N__SW_RST_N BIT(0) + +enum hi6210_bits { + HII2S_BITS_16, + HII2S_BITS_18, + HII2S_BITS_20, + HII2S_BITS_24, +}; + + +#define HII2S_IF_CLK_EN_CFG 4 + +#define HII2S_IF_CLK_EN_CFG__THIRDMD_UPLINK_EN BIT(25) +#define HII2S_IF_CLK_EN_CFG__THIRDMD_DLINK_EN BIT(24) +#define HII2S_IF_CLK_EN_CFG__S3_IF_CLK_EN BIT(20) +#define HII2S_IF_CLK_EN_CFG__S2_IF_CLK_EN BIT(16) +#define HII2S_IF_CLK_EN_CFG__S2_OL_MIXER_EN BIT(15) +#define HII2S_IF_CLK_EN_CFG__S2_OL_SRC_EN BIT(14) +#define HII2S_IF_CLK_EN_CFG__S2_IR_PGA_EN BIT(13) +#define HII2S_IF_CLK_EN_CFG__S2_IL_PGA_EN BIT(12) +#define HII2S_IF_CLK_EN_CFG__S1_IR_PGA_EN BIT(10) +#define HII2S_IF_CLK_EN_CFG__S1_IL_PGA_EN BIT(9) +#define HII2S_IF_CLK_EN_CFG__S1_IF_CLK_EN BIT(8) +#define HII2S_IF_CLK_EN_CFG__VOICE_DLINK_SRC_EN BIT(7) +#define HII2S_IF_CLK_EN_CFG__VOICE_DLINK_EN BIT(6) +#define HII2S_IF_CLK_EN_CFG__ST_DL_R_EN BIT(5) +#define HII2S_IF_CLK_EN_CFG__ST_DL_L_EN BIT(4) +#define HII2S_IF_CLK_EN_CFG__VOICE_UPLINK_R_EN BIT(3) +#define HII2S_IF_CLK_EN_CFG__VOICE_UPLINK_L_EN BIT(2) +#define HII2S_IF_CLK_EN_CFG__STEREO_UPLINK_R_EN BIT(1) +#define HII2S_IF_CLK_EN_CFG__STEREO_UPLINK_L_EN BIT(0) + +#define HII2S_DIG_FILTER_CLK_EN_CFG 8 +#define HII2S_DIG_FILTER_CLK_EN_CFG__DACR_SDM_EN BIT(30) +#define HII2S_DIG_FILTER_CLK_EN_CFG__DACR_HBF2I_EN BIT(28) +#define HII2S_DIG_FILTER_CLK_EN_CFG__DACR_MIXER_EN BIT(25) +#define HII2S_DIG_FILTER_CLK_EN_CFG__DACR_AGC_EN BIT(24) +#define HII2S_DIG_FILTER_CLK_EN_CFG__DACL_SDM_EN BIT(22) +#define HII2S_DIG_FILTER_CLK_EN_CFG__DACL_HBF2I_EN BIT(20) +#define HII2S_DIG_FILTER_CLK_EN_CFG__DACL_MIXER_EN BIT(17) +#define HII2S_DIG_FILTER_CLK_EN_CFG__DACL_AGC_EN BIT(16) + +#define HII2S_FS_CFG 0xc + +#define HII2S_FS_CFG__FS_S2_SHIFT 28 +#define HII2S_FS_CFG__FS_S2_MASK 7 +#define HII2S_FS_CFG__FS_S1_SHIFT 24 +#define HII2S_FS_CFG__FS_S1_MASK 7 +#define HII2S_FS_CFG__FS_ADCLR_SHIFT 20 +#define HII2S_FS_CFG__FS_ADCLR_MASK 7 +#define HII2S_FS_CFG__FS_DACLR_SHIFT 16 +#define HII2S_FS_CFG__FS_DACLR_MASK 7 +#define HII2S_FS_CFG__FS_ST_DL_R_SHIFT 8 +#define HII2S_FS_CFG__FS_ST_DL_R_MASK 7 +#define HII2S_FS_CFG__FS_ST_DL_L_SHIFT 4 +#define HII2S_FS_CFG__FS_ST_DL_L_MASK 7 +#define HII2S_FS_CFG__FS_VOICE_DLINK_SHIFT 0 +#define HII2S_FS_CFG__FS_VOICE_DLINK_MASK 7 + +enum hi6210_i2s_rates { + HII2S_FS_RATE_8KHZ = 0, + HII2S_FS_RATE_16KHZ = 1, + HII2S_FS_RATE_32KHZ = 2, + HII2S_FS_RATE_48KHZ = 4, + HII2S_FS_RATE_96KHZ = 5, + HII2S_FS_RATE_192KHZ = 6, +}; + +#define HII2S_I2S_CFG 0x10 + +#define HII2S_I2S_CFG__S2_IF_TX_EN BIT(31) +#define HII2S_I2S_CFG__S2_IF_RX_EN BIT(30) +#define HII2S_I2S_CFG__S2_FRAME_MODE BIT(29) +#define HII2S_I2S_CFG__S2_MST_SLV BIT(28) +#define HII2S_I2S_CFG__S2_LRCK_MODE BIT(27) +#define HII2S_I2S_CFG__S2_CHNNL_MODE BIT(26) +#define HII2S_I2S_CFG__S2_CODEC_IO_WORDLENGTH_SHIFT 24 +#define HII2S_I2S_CFG__S2_CODEC_IO_WORDLENGTH_MASK 3 +#define HII2S_I2S_CFG__S2_DIRECT_LOOP_SHIFT 22 +#define HII2S_I2S_CFG__S2_DIRECT_LOOP_MASK 3 +#define HII2S_I2S_CFG__S2_TX_CLK_SEL BIT(21) +#define HII2S_I2S_CFG__S2_RX_CLK_SEL BIT(20) +#define HII2S_I2S_CFG__S2_CODEC_DATA_FORMAT BIT(19) +#define HII2S_I2S_CFG__S2_FUNC_MODE_SHIFT 16 +#define HII2S_I2S_CFG__S2_FUNC_MODE_MASK 7 +#define HII2S_I2S_CFG__S1_IF_TX_EN BIT(15) +#define HII2S_I2S_CFG__S1_IF_RX_EN BIT(14) +#define HII2S_I2S_CFG__S1_FRAME_MODE BIT(13) +#define HII2S_I2S_CFG__S1_MST_SLV BIT(12) +#define HII2S_I2S_CFG__S1_LRCK_MODE BIT(11) +#define HII2S_I2S_CFG__S1_CHNNL_MODE BIT(10) +#define HII2S_I2S_CFG__S1_CODEC_IO_WORDLENGTH_SHIFT 8 +#define HII2S_I2S_CFG__S1_CODEC_IO_WORDLENGTH_MASK 3 +#define HII2S_I2S_CFG__S1_DIRECT_LOOP_SHIFT 6 +#define HII2S_I2S_CFG__S1_DIRECT_LOOP_MASK 3 +#define HII2S_I2S_CFG__S1_TX_CLK_SEL BIT(5) +#define HII2S_I2S_CFG__S1_RX_CLK_SEL BIT(4) +#define HII2S_I2S_CFG__S1_CODEC_DATA_FORMAT BIT(3) +#define HII2S_I2S_CFG__S1_FUNC_MODE_SHIFT 0 +#define HII2S_I2S_CFG__S1_FUNC_MODE_MASK 7 + +enum hi6210_i2s_formats { + HII2S_FORMAT_I2S, + HII2S_FORMAT_PCM_STD, + HII2S_FORMAT_PCM_USER, + HII2S_FORMAT_LEFT_JUST, + HII2S_FORMAT_RIGHT_JUST, +}; + +#define HII2S_DIG_FILTER_MODULE_CFG 0x14 + +#define HII2S_DIG_FILTER_MODULE_CFG__DACR_MIXER_GAIN_SHIFT 28 +#define HII2S_DIG_FILTER_MODULE_CFG__DACR_MIXER_GAIN_MASK 3 +#define HII2S_DIG_FILTER_MODULE_CFG__DACR_MIXER_IN4_MUTE BIT(27) +#define HII2S_DIG_FILTER_MODULE_CFG__DACR_MIXER_IN3_MUTE BIT(26) +#define HII2S_DIG_FILTER_MODULE_CFG__DACR_MIXER_IN2_MUTE BIT(25) +#define HII2S_DIG_FILTER_MODULE_CFG__DACR_MIXER_IN1_MUTE BIT(24) +#define HII2S_DIG_FILTER_MODULE_CFG__DACL_MIXER_GAIN_SHIFT 20 +#define HII2S_DIG_FILTER_MODULE_CFG__DACL_MIXER_GAIN_MASK 3 +#define HII2S_DIG_FILTER_MODULE_CFG__DACL_MIXER_IN4_MUTE BIT(19) +#define HII2S_DIG_FILTER_MODULE_CFG__DACL_MIXER_IN3_MUTE BIT(18) +#define HII2S_DIG_FILTER_MODULE_CFG__DACL_MIXER_IN2_MUTE BIT(17) +#define HII2S_DIG_FILTER_MODULE_CFG__DACL_MIXER_IN1_MUTE BIT(16) +#define HII2S_DIG_FILTER_MODULE_CFG__SW_DACR_SDM_DITHER BIT(9) +#define HII2S_DIG_FILTER_MODULE_CFG__SW_DACL_SDM_DITHER BIT(8) +#define HII2S_DIG_FILTER_MODULE_CFG__LM_CODEC_DAC2ADC_SHIFT 4 +#define HII2S_DIG_FILTER_MODULE_CFG__LM_CODEC_DAC2ADC_MASK 7 +#define HII2S_DIG_FILTER_MODULE_CFG__RM_CODEC_DAC2ADC_SHIFT 0 +#define HII2S_DIG_FILTER_MODULE_CFG__RM_CODEC_DAC2ADC_MASK 7 + +enum hi6210_gains { + HII2S_GAIN_100PC, + HII2S_GAIN_50PC, + HII2S_GAIN_25PC, +}; + +#define HII2S_MUX_TOP_MODULE_CFG 0x18 + +#define HII2S_MUX_TOP_MODULE_CFG__VOICE_DLINK_MIXER_GAIN_SHIFT 14 +#define HII2S_MUX_TOP_MODULE_CFG__VOICE_DLINK_MIXER_GAIN_MASK 3 +#define HII2S_MUX_TOP_MODULE_CFG__VOICE_DLINK_MIXER_IN2_MUTE BIT(13) +#define HII2S_MUX_TOP_MODULE_CFG__VOICE_DLINK_MIXER_IN1_MUTE BIT(12) +#define HII2S_MUX_TOP_MODULE_CFG__S2_OL_MIXER_GAIN_SHIFT 10 +#define HII2S_MUX_TOP_MODULE_CFG__S2_OL_MIXER_GAIN_MASK 3 +#define HII2S_MUX_TOP_MODULE_CFG__S2_OL_MIXER_IN2_MUTE BIT(9) +#define HII2S_MUX_TOP_MODULE_CFG__S2_OL_MIXER_IN1_MUTE BIT(8) +#define HII2S_MUX_TOP_MODULE_CFG__S2_OL_SRC_RDY BIT(6) +#define HII2S_MUX_TOP_MODULE_CFG__S2_OL_SRC_MODE_SHIFT 4 +#define HII2S_MUX_TOP_MODULE_CFG__S2_OL_SRC_MODE_MASK 3 +#define HII2S_MUX_TOP_MODULE_CFG__VOICE_DLINK_SRC_RDY BIT(3) +#define HII2S_MUX_TOP_MODULE_CFG__VOICE_DLINK_SRC_MODE_SHIFT 0 +#define HII2S_MUX_TOP_MODULE_CFG__VOICE_DLINK_SRC_MODE_MASK 7 + +enum hi6210_s2_src_mode { + HII2S_S2_SRC_MODE_3, + HII2S_S2_SRC_MODE_12, + HII2S_S2_SRC_MODE_6, + HII2S_S2_SRC_MODE_2, +}; + +enum hi6210_voice_dlink_src_mode { + HII2S_VOICE_DL_SRC_MODE_12 = 1, + HII2S_VOICE_DL_SRC_MODE_6, + HII2S_VOICE_DL_SRC_MODE_2, + HII2S_VOICE_DL_SRC_MODE_3, +}; + +#define HII2S_ADC_PGA_CFG 0x1c +#define HII2S_S1_INPUT_PGA_CFG 0x20 +#define HII2S_S2_INPUT_PGA_CFG 0x24 +#define HII2S_ST_DL_PGA_CFG 0x28 +#define HII2S_VOICE_SIDETONE_DLINK_PGA_CFG 0x2c +#define HII2S_APB_AFIFO_CFG_1 0x30 +#define HII2S_APB_AFIFO_CFG_2 0x34 +#define HII2S_ST_DL_FIFO_TH_CFG 0x38 + +#define HII2S_ST_DL_FIFO_TH_CFG__ST_DL_R_AEMPTY_SHIFT 24 +#define HII2S_ST_DL_FIFO_TH_CFG__ST_DL_R_AEMPTY_MASK 0x1f +#define HII2S_ST_DL_FIFO_TH_CFG__ST_DL_R_AFULL_SHIFT 16 +#define HII2S_ST_DL_FIFO_TH_CFG__ST_DL_R_AFULL_MASK 0x1f +#define HII2S_ST_DL_FIFO_TH_CFG__ST_DL_L_AEMPTY_SHIFT 8 +#define HII2S_ST_DL_FIFO_TH_CFG__ST_DL_L_AEMPTY_MASK 0x1f +#define HII2S_ST_DL_FIFO_TH_CFG__ST_DL_L_AFULL_SHIFT 0 +#define HII2S_ST_DL_FIFO_TH_CFG__ST_DL_L_AFULL_MASK 0x1f + +#define HII2S_STEREO_UPLINK_FIFO_TH_CFG 0x3c +#define HII2S_VOICE_UPLINK_FIFO_TH_CFG 0x40 +#define HII2S_CODEC_IRQ_MASK 0x44 +#define HII2S_CODEC_IRQ 0x48 +#define HII2S_DACL_AGC_CFG_1 0x4c +#define HII2S_DACL_AGC_CFG_2 0x50 +#define HII2S_DACR_AGC_CFG_1 0x54 +#define HII2S_DACR_AGC_CFG_2 0x58 +#define HII2S_DMIC_SIF_CFG 0x5c +#define HII2S_MISC_CFG 0x60 + +#define HII2S_MISC_CFG__THIRDMD_DLINK_TEST_SEL BIT(17) +#define HII2S_MISC_CFG__THIRDMD_DLINK_DIN_SEL BIT(16) +#define HII2S_MISC_CFG__S3_DOUT_RIGHT_SEL BIT(14) +#define HII2S_MISC_CFG__S3_DOUT_LEFT_SEL BIT(13) +#define HII2S_MISC_CFG__S3_DIN_TEST_SEL BIT(12) +#define HII2S_MISC_CFG__VOICE_DLINK_SRC_UP_DOUT_VLD_SEL BIT(8) +#define HII2S_MISC_CFG__VOICE_DLINK_TEST_SEL BIT(7) +#define HII2S_MISC_CFG__VOICE_DLINK_DIN_SEL BIT(6) +#define HII2S_MISC_CFG__ST_DL_TEST_SEL BIT(4) +#define HII2S_MISC_CFG__S2_DOUT_RIGHT_SEL BIT(3) +#define HII2S_MISC_CFG__S2_DOUT_TEST_SEL BIT(2) +#define HII2S_MISC_CFG__S1_DOUT_TEST_SEL BIT(1) +#define HII2S_MISC_CFG__S2_DOUT_LEFT_SEL BIT(0) + +#define HII2S_S2_SRC_CFG 0x64 +#define HII2S_MEM_CFG 0x68 +#define HII2S_THIRDMD_PCM_PGA_CFG 0x6c +#define HII2S_THIRD_MODEM_FIFO_TH 0x70 +#define HII2S_S3_ANTI_FREQ_JITTER_TX_INC_CNT 0x74 +#define HII2S_S3_ANTI_FREQ_JITTER_TX_DEC_CNT 0x78 +#define HII2S_S3_ANTI_FREQ_JITTER_RX_INC_CNT 0x7c +#define HII2S_S3_ANTI_FREQ_JITTER_RX_DEC_CNT 0x80 +#define HII2S_ANTI_FREQ_JITTER_EN 0x84 +#define HII2S_CLK_SEL 0x88 + +/* 0 = BT owns the i2s */ +#define HII2S_CLK_SEL__I2S_BT_FM_SEL BIT(0) +/* 0 = internal source, 1 = ext */ +#define HII2S_CLK_SEL__EXT_12_288MHZ_SEL BIT(1) + + +#define HII2S_THIRDMD_DLINK_CHANNEL 0xe8 +#define HII2S_THIRDMD_ULINK_CHANNEL 0xec +#define HII2S_VOICE_DLINK_CHANNEL 0xf0 + +/* shovel data in here for playback */ +#define HII2S_ST_DL_CHANNEL 0xf4 +#define HII2S_STEREO_UPLINK_CHANNEL 0xf8 +#define HII2S_VOICE_UPLINK_CHANNEL 0xfc + +#endif/* _HI6210_I2S_H */ diff --git a/sound/soc/intel/atom/sst-mfld-platform-pcm.c b/sound/soc/intel/atom/sst-mfld-platform-pcm.c index 25c6d87c818e7c..f5a8050351b53a 100644 --- a/sound/soc/intel/atom/sst-mfld-platform-pcm.c +++ b/sound/soc/intel/atom/sst-mfld-platform-pcm.c @@ -771,6 +771,9 @@ static int sst_soc_prepare(struct device *dev) struct sst_data *drv = dev_get_drvdata(dev); struct snd_soc_pcm_runtime *rtd; + if (!drv->soc_card) + return 0; + /* suspend all pcms first */ snd_soc_suspend(drv->soc_card->dev); snd_soc_poweroff(drv->soc_card->dev); @@ -793,6 +796,9 @@ static void sst_soc_complete(struct device *dev) struct sst_data *drv = dev_get_drvdata(dev); struct snd_soc_pcm_runtime *rtd; + if (!drv->soc_card) + return; + /* restart SSPs */ list_for_each_entry(rtd, &drv->soc_card->rtd_list, list) { struct snd_soc_dai *dai = rtd->cpu_dai; diff --git a/sound/soc/intel/boards/cht_bsw_rt5645.c b/sound/soc/intel/boards/cht_bsw_rt5645.c index 56056ed7fcfd1e..16c94c45ce50a8 100644 --- a/sound/soc/intel/boards/cht_bsw_rt5645.c +++ b/sound/soc/intel/boards/cht_bsw_rt5645.c @@ -44,6 +44,7 @@ struct cht_acpi_card { struct cht_mc_private { struct snd_soc_jack jack; struct cht_acpi_card *acpi_card; + char codec_name[16]; }; static inline struct snd_soc_dai *cht_get_codec_dai(struct snd_soc_card *card) @@ -354,7 +355,6 @@ static int snd_cht_mc_probe(struct platform_device *pdev) int i; struct cht_mc_private *drv; struct snd_soc_card *card = snd_soc_cards[0].soc_card; - char codec_name[16]; struct sst_acpi_mach *mach; const char *i2c_name = NULL; int dai_index = 0; @@ -374,12 +374,12 @@ static int snd_cht_mc_probe(struct platform_device *pdev) } card->dev = &pdev->dev; mach = card->dev->platform_data; - sprintf(codec_name, "i2c-%s:00", drv->acpi_card->codec_id); + sprintf(drv->codec_name, "i2c-%s:00", drv->acpi_card->codec_id); /* set correct codec name */ for (i = 0; i < ARRAY_SIZE(cht_dailink); i++) if (!strcmp(card->dai_link[i].codec_name, "i2c-10EC5645:00")) { - card->dai_link[i].codec_name = kstrdup(codec_name, GFP_KERNEL); + card->dai_link[i].codec_name = drv->codec_name; dai_index = i; } diff --git a/sound/soc/intel/skylake/skl-sst-utils.c b/sound/soc/intel/skylake/skl-sst-utils.c index 8dc03039b3113f..ea162fbf68e5b3 100644 --- a/sound/soc/intel/skylake/skl-sst-utils.c +++ b/sound/soc/intel/skylake/skl-sst-utils.c @@ -179,7 +179,7 @@ static inline int skl_getid_32(struct uuid_module *module, u64 *val, index = ffz(mask_val); pvt_id = index + word1_mask + word2_mask; if (pvt_id <= (max_inst - 1)) { - *val |= 1 << (index + word1_mask); + *val |= 1ULL << (index + word1_mask); return pvt_id; } } diff --git a/sound/soc/qcom/lpass-platform.c b/sound/soc/qcom/lpass-platform.c index b392e51de94d17..420d200f9a0533 100644 --- a/sound/soc/qcom/lpass-platform.c +++ b/sound/soc/qcom/lpass-platform.c @@ -78,6 +78,9 @@ static int lpass_platform_pcmops_open(struct snd_pcm_substream *substream) dma_ch = 0; if (v->alloc_dma_channel) dma_ch = v->alloc_dma_channel(drvdata, dir); + else + dma_ch = 0; + if (dma_ch < 0) return dma_ch; diff --git a/sound/soc/samsung/i2s.c b/sound/soc/samsung/i2s.c index 7825bff45ae3a5..85324e61cbd56d 100644 --- a/sound/soc/samsung/i2s.c +++ b/sound/soc/samsung/i2s.c @@ -1029,12 +1029,13 @@ static int samsung_i2s_dai_probe(struct snd_soc_dai *dai) static int samsung_i2s_dai_remove(struct snd_soc_dai *dai) { struct i2s_dai *i2s = snd_soc_dai_get_drvdata(dai); + unsigned long flags; if (!is_secondary(i2s)) { if (i2s->quirks & QUIRK_NEED_RSTCLR) { - spin_lock(i2s->lock); + spin_lock_irqsave(i2s->lock, flags); writel(0, i2s->addr + I2SCON); - spin_unlock(i2s->lock); + spin_unlock_irqrestore(i2s->lock, flags); } } diff --git a/sound/usb/card.c b/sound/usb/card.c index 2ddc034673a8e9..f36cb068dad331 100644 --- a/sound/usb/card.c +++ b/sound/usb/card.c @@ -206,7 +206,6 @@ static int snd_usb_create_stream(struct snd_usb_audio *chip, int ctrlif, int int if (! snd_usb_parse_audio_interface(chip, interface)) { usb_set_interface(dev, interface, 0); /* reset the current interface */ usb_driver_claim_interface(&usb_audio_driver, iface, (void *)-1L); - return -EINVAL; } return 0; diff --git a/sound/usb/endpoint.c b/sound/usb/endpoint.c index c470251cea4b25..c5251aaad8442a 100644 --- a/sound/usb/endpoint.c +++ b/sound/usb/endpoint.c @@ -534,6 +534,11 @@ static int wait_clear_urbs(struct snd_usb_endpoint *ep) alive, ep->ep_num); clear_bit(EP_FLAG_STOPPING, &ep->flags); + ep->data_subs = NULL; + ep->sync_slave = NULL; + ep->retire_data_urb = NULL; + ep->prepare_data_urb = NULL; + return 0; } @@ -898,9 +903,7 @@ int snd_usb_endpoint_set_params(struct snd_usb_endpoint *ep, /** * snd_usb_endpoint_start: start an snd_usb_endpoint * - * @ep: the endpoint to start - * @can_sleep: flag indicating whether the operation is executed in - * non-atomic context + * @ep: the endpoint to start * * A call to this function will increment the use count of the endpoint. * In case it is not already running, the URBs for this endpoint will be @@ -910,7 +913,7 @@ int snd_usb_endpoint_set_params(struct snd_usb_endpoint *ep, * * Returns an error if the URB submission failed, 0 in all other cases. */ -int snd_usb_endpoint_start(struct snd_usb_endpoint *ep, bool can_sleep) +int snd_usb_endpoint_start(struct snd_usb_endpoint *ep) { int err; unsigned int i; @@ -924,8 +927,6 @@ int snd_usb_endpoint_start(struct snd_usb_endpoint *ep, bool can_sleep) /* just to be sure */ deactivate_urbs(ep, false); - if (can_sleep) - wait_clear_urbs(ep); ep->active_mask = 0; ep->unlink_mask = 0; @@ -1006,10 +1007,6 @@ void snd_usb_endpoint_stop(struct snd_usb_endpoint *ep) if (--ep->use_count == 0) { deactivate_urbs(ep, false); - ep->data_subs = NULL; - ep->sync_slave = NULL; - ep->retire_data_urb = NULL; - ep->prepare_data_urb = NULL; set_bit(EP_FLAG_STOPPING, &ep->flags); } } diff --git a/sound/usb/endpoint.h b/sound/usb/endpoint.h index 6428392d8f6244..584f295d7c7738 100644 --- a/sound/usb/endpoint.h +++ b/sound/usb/endpoint.h @@ -18,7 +18,7 @@ int snd_usb_endpoint_set_params(struct snd_usb_endpoint *ep, struct audioformat *fmt, struct snd_usb_endpoint *sync_ep); -int snd_usb_endpoint_start(struct snd_usb_endpoint *ep, bool can_sleep); +int snd_usb_endpoint_start(struct snd_usb_endpoint *ep); void snd_usb_endpoint_stop(struct snd_usb_endpoint *ep); void snd_usb_endpoint_sync_pending_stop(struct snd_usb_endpoint *ep); int snd_usb_endpoint_activate(struct snd_usb_endpoint *ep); diff --git a/sound/usb/hiface/pcm.c b/sound/usb/hiface/pcm.c index 2c44139b404188..33db205dd12b57 100644 --- a/sound/usb/hiface/pcm.c +++ b/sound/usb/hiface/pcm.c @@ -445,6 +445,8 @@ static int hiface_pcm_prepare(struct snd_pcm_substream *alsa_sub) mutex_lock(&rt->stream_mutex); + hiface_pcm_stream_stop(rt); + sub->dma_off = 0; sub->period_off = 0; diff --git a/sound/usb/line6/driver.c b/sound/usb/line6/driver.c index 90009c0b3a92e4..ab3c280a23d1fa 100644 --- a/sound/usb/line6/driver.c +++ b/sound/usb/line6/driver.c @@ -754,8 +754,9 @@ int line6_probe(struct usb_interface *interface, goto error; } + line6_get_interval(line6); + if (properties->capabilities & LINE6_CAP_CONTROL) { - line6_get_interval(line6); ret = line6_init_cap_control(line6); if (ret < 0) goto error; diff --git a/sound/usb/mixer.c b/sound/usb/mixer.c index 2f8c388ef84fc6..4703caea56b2f6 100644 --- a/sound/usb/mixer.c +++ b/sound/usb/mixer.c @@ -932,9 +932,10 @@ static void volume_control_quirks(struct usb_mixer_elem_info *cval, case USB_ID(0x046d, 0x0826): /* HD Webcam c525 */ case USB_ID(0x046d, 0x08ca): /* Logitech Quickcam Fusion */ case USB_ID(0x046d, 0x0991): + case USB_ID(0x046d, 0x09a2): /* QuickCam Communicate Deluxe/S7500 */ /* Most audio usb devices lie about volume resolution. * Most Logitech webcams have res = 384. - * Proboly there is some logitech magic behind this number --fishor + * Probably there is some logitech magic behind this number --fishor */ if (!strcmp(kctl->id.name, "Mic Capture Volume")) { usb_audio_info(chip, diff --git a/sound/usb/pcm.c b/sound/usb/pcm.c index 44d178ee917756..48afae053c56f9 100644 --- a/sound/usb/pcm.c +++ b/sound/usb/pcm.c @@ -218,7 +218,7 @@ int snd_usb_init_pitch(struct snd_usb_audio *chip, int iface, } } -static int start_endpoints(struct snd_usb_substream *subs, bool can_sleep) +static int start_endpoints(struct snd_usb_substream *subs) { int err; @@ -231,7 +231,7 @@ static int start_endpoints(struct snd_usb_substream *subs, bool can_sleep) dev_dbg(&subs->dev->dev, "Starting data EP @%p\n", ep); ep->data_subs = subs; - err = snd_usb_endpoint_start(ep, can_sleep); + err = snd_usb_endpoint_start(ep); if (err < 0) { clear_bit(SUBSTREAM_FLAG_DATA_EP_STARTED, &subs->flags); return err; @@ -260,7 +260,7 @@ static int start_endpoints(struct snd_usb_substream *subs, bool can_sleep) dev_dbg(&subs->dev->dev, "Starting sync EP @%p\n", ep); ep->sync_slave = subs->data_endpoint; - err = snd_usb_endpoint_start(ep, can_sleep); + err = snd_usb_endpoint_start(ep); if (err < 0) { clear_bit(SUBSTREAM_FLAG_SYNC_EP_STARTED, &subs->flags); return err; @@ -839,7 +839,7 @@ static int snd_usb_pcm_prepare(struct snd_pcm_substream *substream) /* for playback, submit the URBs now; otherwise, the first hwptr_done * updates for all URBs would happen at the same time when starting */ if (subs->direction == SNDRV_PCM_STREAM_PLAYBACK) - ret = start_endpoints(subs, true); + ret = start_endpoints(subs); unlock: snd_usb_unlock_shutdown(subs->stream->chip); @@ -1655,7 +1655,7 @@ static int snd_usb_substream_capture_trigger(struct snd_pcm_substream *substream switch (cmd) { case SNDRV_PCM_TRIGGER_START: - err = start_endpoints(subs, false); + err = start_endpoints(subs); if (err < 0) return err; diff --git a/sound/usb/quirks.c b/sound/usb/quirks.c index 2782155ae3ce02..93bb14e7e0f728 100644 --- a/sound/usb/quirks.c +++ b/sound/usb/quirks.c @@ -1135,6 +1135,7 @@ bool snd_usb_get_sample_rate_quirk(struct snd_usb_audio *chip) case USB_ID(0x045E, 0x076F): /* MS Lifecam HD-6000 */ case USB_ID(0x045E, 0x0772): /* MS Lifecam Studio */ case USB_ID(0x045E, 0x0779): /* MS Lifecam HD-3000 */ + case USB_ID(0x047F, 0x02F7): /* Plantronics BT-600 */ case USB_ID(0x047F, 0x0415): /* Plantronics BT-300 */ case USB_ID(0x047F, 0xAA05): /* Plantronics DA45 */ case USB_ID(0x04D8, 0xFEEA): /* Benchmark DAC1 Pre */ diff --git a/tools/perf/Makefile.config b/tools/perf/Makefile.config index 72edf83d76b721..cffdd9cf3ebf76 100644 --- a/tools/perf/Makefile.config +++ b/tools/perf/Makefile.config @@ -366,7 +366,7 @@ ifndef NO_SDT endif ifdef PERF_HAVE_JITDUMP - ifndef NO_DWARF + ifndef NO_LIBELF $(call detected,CONFIG_JITDUMP) CFLAGS += -DHAVE_JITDUMP endif diff --git a/tools/perf/builtin-diff.c b/tools/perf/builtin-diff.c index 9ff0db4e2d0cd1..933aeec46f4a12 100644 --- a/tools/perf/builtin-diff.c +++ b/tools/perf/builtin-diff.c @@ -1199,7 +1199,7 @@ static int ui_init(void) BUG_ON(1); } - perf_hpp__register_sort_field(fmt); + perf_hpp__prepend_sort_field(fmt); return 0; } diff --git a/tools/perf/builtin-mem.c b/tools/perf/builtin-mem.c index d1ce29be560e5e..cd7bc4d104e27e 100644 --- a/tools/perf/builtin-mem.c +++ b/tools/perf/builtin-mem.c @@ -70,8 +70,8 @@ static int __cmd_record(int argc, const char **argv, struct perf_mem *mem) OPT_UINTEGER(0, "ldlat", &perf_mem_events__loads_ldlat, "mem-loads latency"), OPT_INCR('v', "verbose", &verbose, "be more verbose (show counter open errors, etc)"), - OPT_BOOLEAN('U', "--all-user", &all_user, "collect only user level data"), - OPT_BOOLEAN('K', "--all-kernel", &all_kernel, "collect only kernel level data"), + OPT_BOOLEAN('U', "all-user", &all_user, "collect only user level data"), + OPT_BOOLEAN('K', "all-kernel", &all_kernel, "collect only kernel level data"), OPT_END() }; diff --git a/tools/perf/builtin-trace.c b/tools/perf/builtin-trace.c index c298bd3e1d909c..21f8a81797a04e 100644 --- a/tools/perf/builtin-trace.c +++ b/tools/perf/builtin-trace.c @@ -1452,7 +1452,7 @@ static int trace__printf_interrupted_entry(struct trace *trace, struct perf_samp duration = sample->time - ttrace->entry_time; - printed = trace__fprintf_entry_head(trace, trace->current, duration, sample->time, trace->output); + printed = trace__fprintf_entry_head(trace, trace->current, duration, ttrace->entry_time, trace->output); printed += fprintf(trace->output, "%-70s) ...\n", ttrace->entry_str); ttrace->entry_pending = false; @@ -1499,7 +1499,7 @@ static int trace__sys_enter(struct trace *trace, struct perf_evsel *evsel, if (sc->is_exit) { if (!(trace->duration_filter || trace->summary_only || trace->min_stack)) { - trace__fprintf_entry_head(trace, thread, 1, sample->time, trace->output); + trace__fprintf_entry_head(trace, thread, 1, ttrace->entry_time, trace->output); fprintf(trace->output, "%-70s)\n", ttrace->entry_str); } } else { @@ -1592,7 +1592,7 @@ static int trace__sys_exit(struct trace *trace, struct perf_evsel *evsel, if (trace->summary_only) goto out; - trace__fprintf_entry_head(trace, thread, duration, sample->time, trace->output); + trace__fprintf_entry_head(trace, thread, duration, ttrace->entry_time, trace->output); if (ttrace->entry_pending) { fprintf(trace->output, "%-70s", ttrace->entry_str); diff --git a/tools/perf/trace/beauty/mmap.c b/tools/perf/trace/beauty/mmap.c index fd710ab33684e6..af1cfde6b97b12 100644 --- a/tools/perf/trace/beauty/mmap.c +++ b/tools/perf/trace/beauty/mmap.c @@ -42,7 +42,9 @@ static size_t syscall_arg__scnprintf_mmap_flags(char *bf, size_t size, P_MMAP_FLAG(SHARED); P_MMAP_FLAG(PRIVATE); +#ifdef MAP_32BIT P_MMAP_FLAG(32BIT); +#endif P_MMAP_FLAG(ANONYMOUS); P_MMAP_FLAG(DENYWRITE); P_MMAP_FLAG(EXECUTABLE); diff --git a/tools/perf/ui/hist.c b/tools/perf/ui/hist.c index 37388397b5bc05..18cfcdc90356f8 100644 --- a/tools/perf/ui/hist.c +++ b/tools/perf/ui/hist.c @@ -521,6 +521,12 @@ void perf_hpp_list__register_sort_field(struct perf_hpp_list *list, list_add_tail(&format->sort_list, &list->sorts); } +void perf_hpp_list__prepend_sort_field(struct perf_hpp_list *list, + struct perf_hpp_fmt *format) +{ + list_add(&format->sort_list, &list->sorts); +} + void perf_hpp__column_unregister(struct perf_hpp_fmt *format) { list_del(&format->list); @@ -560,6 +566,10 @@ void perf_hpp__setup_output_field(struct perf_hpp_list *list) perf_hpp_list__for_each_sort_list(list, fmt) { struct perf_hpp_fmt *pos; + /* skip sort-only fields ("sort_compute" in perf diff) */ + if (!fmt->entry && !fmt->color) + continue; + perf_hpp_list__for_each_format(list, pos) { if (fmt_equal(fmt, pos)) goto next; diff --git a/tools/perf/util/Build b/tools/perf/util/Build index eb60e613d79543..1dc67efad6340f 100644 --- a/tools/perf/util/Build +++ b/tools/perf/util/Build @@ -120,7 +120,7 @@ libperf-y += demangle-rust.o ifdef CONFIG_JITDUMP libperf-$(CONFIG_LIBELF) += jitdump.o libperf-$(CONFIG_LIBELF) += genelf.o -libperf-$(CONFIG_LIBELF) += genelf_debug.o +libperf-$(CONFIG_DWARF) += genelf_debug.o endif CFLAGS_config.o += -DETC_PERFCONFIG="BUILD_STR($(ETC_PERFCONFIG_SQ))" diff --git a/tools/perf/util/annotate.c b/tools/perf/util/annotate.c index aeb5a441bd7416..430d039d00798e 100644 --- a/tools/perf/util/annotate.c +++ b/tools/perf/util/annotate.c @@ -593,7 +593,8 @@ static int __symbol__inc_addr_samples(struct symbol *sym, struct map *map, pr_debug3("%s: addr=%#" PRIx64 "\n", __func__, map->unmap_ip(map, addr)); - if (addr < sym->start || addr >= sym->end) { + if ((addr < sym->start || addr >= sym->end) && + (addr != sym->end || sym->start != sym->end)) { pr_debug("%s(%d): ERANGE! sym->name=%s, start=%#" PRIx64 ", addr=%#" PRIx64 ", end=%#" PRIx64 "\n", __func__, __LINE__, sym->name, sym->start, addr, sym->end); return -ERANGE; diff --git a/tools/perf/util/callchain.c b/tools/perf/util/callchain.c index 07fd30bc2f816f..ae58b493af4549 100644 --- a/tools/perf/util/callchain.c +++ b/tools/perf/util/callchain.c @@ -193,7 +193,6 @@ int perf_callchain_config(const char *var, const char *value) if (!strcmp(var, "record-mode")) return parse_callchain_record_opt(value, &callchain_param); -#ifdef HAVE_DWARF_UNWIND_SUPPORT if (!strcmp(var, "dump-size")) { unsigned long size = 0; int ret; @@ -203,7 +202,6 @@ int perf_callchain_config(const char *var, const char *value) return ret; } -#endif if (!strcmp(var, "print-type")) return parse_callchain_mode(value); if (!strcmp(var, "order")) diff --git a/tools/perf/util/callchain.h b/tools/perf/util/callchain.h index 13e75549c4407d..47cfd10809755f 100644 --- a/tools/perf/util/callchain.h +++ b/tools/perf/util/callchain.h @@ -11,11 +11,7 @@ #define CALLCHAIN_HELP "setup and enables call-graph (stack chain/backtrace):\n\n" -#ifdef HAVE_DWARF_UNWIND_SUPPORT # define RECORD_MODE_HELP HELP_PAD "record_mode:\tcall graph recording mode (fp|dwarf|lbr)\n" -#else -# define RECORD_MODE_HELP HELP_PAD "record_mode:\tcall graph recording mode (fp|lbr)\n" -#endif #define RECORD_SIZE_HELP \ HELP_PAD "record_size:\tif record_mode is 'dwarf', max size of stack recording ()\n" \ diff --git a/tools/perf/util/genelf.c b/tools/perf/util/genelf.c index c1ef805c6a8f4d..14a73acc549c9d 100644 --- a/tools/perf/util/genelf.c +++ b/tools/perf/util/genelf.c @@ -19,7 +19,9 @@ #include #include #include +#ifdef HAVE_DWARF_SUPPORT #include +#endif #include "perf.h" #include "genelf.h" @@ -157,7 +159,7 @@ gen_build_id(struct buildid_note *note, unsigned long load_addr, const void *cod int jit_write_elf(int fd, uint64_t load_addr, const char *sym, const void *code, int csize, - void *debug, int nr_debug_entries) + void *debug __maybe_unused, int nr_debug_entries __maybe_unused) { Elf *e; Elf_Data *d; @@ -386,11 +388,14 @@ jit_write_elf(int fd, uint64_t load_addr, const char *sym, shdr->sh_size = sizeof(bnote); shdr->sh_entsize = 0; +#ifdef HAVE_DWARF_SUPPORT if (debug && nr_debug_entries) { retval = jit_add_debug_info(e, load_addr, debug, nr_debug_entries); if (retval) goto error; - } else { + } else +#endif + { if (elf_update(e, ELF_C_WRITE) < 0) { warnx("elf_update 4 failed"); goto error; diff --git a/tools/perf/util/genelf.h b/tools/perf/util/genelf.h index 2fbeb59c4bdd91..5c933ac71451f7 100644 --- a/tools/perf/util/genelf.h +++ b/tools/perf/util/genelf.h @@ -4,8 +4,10 @@ /* genelf.c */ int jit_write_elf(int fd, uint64_t code_addr, const char *sym, const void *code, int csize, void *debug, int nr_debug_entries); +#ifdef HAVE_DWARF_SUPPORT /* genelf_debug.c */ int jit_add_debug_info(Elf *e, uint64_t code_addr, void *debug, int nr_debug_entries); +#endif #if defined(__arm__) #define GEN_ELF_ARCH EM_ARM diff --git a/tools/perf/util/hist.h b/tools/perf/util/hist.h index 9928fed8bc597c..a440a04a29ffd6 100644 --- a/tools/perf/util/hist.h +++ b/tools/perf/util/hist.h @@ -282,6 +282,8 @@ void perf_hpp_list__column_register(struct perf_hpp_list *list, struct perf_hpp_fmt *format); void perf_hpp_list__register_sort_field(struct perf_hpp_list *list, struct perf_hpp_fmt *format); +void perf_hpp_list__prepend_sort_field(struct perf_hpp_list *list, + struct perf_hpp_fmt *format); static inline void perf_hpp__column_register(struct perf_hpp_fmt *format) { @@ -293,6 +295,11 @@ static inline void perf_hpp__register_sort_field(struct perf_hpp_fmt *format) perf_hpp_list__register_sort_field(&perf_hpp_list, format); } +static inline void perf_hpp__prepend_sort_field(struct perf_hpp_fmt *format) +{ + perf_hpp_list__prepend_sort_field(&perf_hpp_list, format); +} + #define perf_hpp_list__for_each_format(_list, format) \ list_for_each_entry(format, &(_list)->fields, list) diff --git a/tools/perf/util/symbol.c b/tools/perf/util/symbol.c index aecff69a510df0..f7b35e178582b6 100644 --- a/tools/perf/util/symbol.c +++ b/tools/perf/util/symbol.c @@ -1459,7 +1459,8 @@ int dso__load(struct dso *dso, struct map *map) * Read the build id if possible. This is required for * DSO_BINARY_TYPE__BUILDID_DEBUGINFO to work */ - if (is_regular_file(dso->long_name) && + if (!dso->has_build_id && + is_regular_file(dso->long_name) && filename__read_build_id(dso->long_name, build_id, BUILD_ID_SIZE) > 0) dso__set_build_id(dso, build_id); diff --git a/tools/perf/util/trace-event-scripting.c b/tools/perf/util/trace-event-scripting.c index 9df61059a85d21..a2fd6e79d5a5c5 100644 --- a/tools/perf/util/trace-event-scripting.c +++ b/tools/perf/util/trace-event-scripting.c @@ -95,7 +95,8 @@ static void register_python_scripting(struct scripting_ops *scripting_ops) if (err) die("error registering py script extension"); - scripting_context = malloc(sizeof(struct scripting_context)); + if (scripting_context == NULL) + scripting_context = malloc(sizeof(*scripting_context)); } #ifdef NO_LIBPYTHON @@ -159,7 +160,8 @@ static void register_perl_scripting(struct scripting_ops *scripting_ops) if (err) die("error registering pl script extension"); - scripting_context = malloc(sizeof(struct scripting_context)); + if (scripting_context == NULL) + scripting_context = malloc(sizeof(*scripting_context)); } #ifdef NO_LIBPERL diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index f770dba2a6f636..a899ef81c705bc 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -87,7 +87,7 @@ ifdef INSTALL_PATH done; @# Ask all targets to emit their test scripts - echo "#!/bin/bash" > $(ALL_SCRIPT) + echo "#!/bin/sh" > $(ALL_SCRIPT) echo "cd \$$(dirname \$$0)" >> $(ALL_SCRIPT) echo "ROOT=\$$PWD" >> $(ALL_SCRIPT) diff --git a/tools/testing/selftests/net/run_netsocktests b/tools/testing/selftests/net/run_netsocktests index c09a682df56ae9..16058bbea7a850 100755 --- a/tools/testing/selftests/net/run_netsocktests +++ b/tools/testing/selftests/net/run_netsocktests @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh echo "--------------------" echo "running socket test" diff --git a/tools/testing/selftests/powerpc/pmu/ebb/pmc56_overflow_test.c b/tools/testing/selftests/powerpc/pmu/ebb/pmc56_overflow_test.c index c22860ab973378..30e1ac62e8cb42 100644 --- a/tools/testing/selftests/powerpc/pmu/ebb/pmc56_overflow_test.c +++ b/tools/testing/selftests/powerpc/pmu/ebb/pmc56_overflow_test.c @@ -66,7 +66,7 @@ int pmc56_overflow(void) FAIL_IF(ebb_event_enable(&event)); - mtspr(SPRN_PMC1, pmc_sample_period(sample_period)); + mtspr(SPRN_PMC2, pmc_sample_period(sample_period)); mtspr(SPRN_PMC5, 0); mtspr(SPRN_PMC6, 0); diff --git a/tools/virtio/linux/compiler.h b/tools/virtio/linux/compiler.h index 845960e1cbf2f4..c9ccfd42ec137d 100644 --- a/tools/virtio/linux/compiler.h +++ b/tools/virtio/linux/compiler.h @@ -4,6 +4,6 @@ #define WRITE_ONCE(var, val) \ (*((volatile typeof(val) *)(&(var))) = (val)) -#define READ_ONCE(var) (*((volatile typeof(val) *)(&(var)))) +#define READ_ONCE(var) (*((volatile typeof(var) *)(&(var)))) #endif diff --git a/tools/virtio/ringtest/run-on-all.sh b/tools/virtio/ringtest/run-on-all.sh index 2e69ca812b4cf4..29b0d3920bfc41 100755 --- a/tools/virtio/ringtest/run-on-all.sh +++ b/tools/virtio/ringtest/run-on-all.sh @@ -1,12 +1,13 @@ #!/bin/sh +CPUS_ONLINE=$(lscpu --online -p=cpu|grep -v -e '#') #use last CPU for host. Why not the first? #many devices tend to use cpu0 by default so #it tends to be busier -HOST_AFFINITY=$(lscpu -p=cpu | tail -1) +HOST_AFFINITY=$(echo "${CPUS_ONLINE}"|tail -n 1) #run command on all cpus -for cpu in $(seq 0 $HOST_AFFINITY) +for cpu in $CPUS_ONLINE do #Don't run guest and host on same CPU #It actually works ok if using signalling diff --git a/virt/kvm/arm/vgic/vgic-init.c b/virt/kvm/arm/vgic/vgic-init.c index 8cebfbc19e90ef..539d3f5cb61934 100644 --- a/virt/kvm/arm/vgic/vgic-init.c +++ b/virt/kvm/arm/vgic/vgic-init.c @@ -268,15 +268,11 @@ static void kvm_vgic_dist_destroy(struct kvm *kvm) { struct vgic_dist *dist = &kvm->arch.vgic; - mutex_lock(&kvm->lock); - dist->ready = false; dist->initialized = false; kfree(dist->spis); dist->nr_spis = 0; - - mutex_unlock(&kvm->lock); } void kvm_vgic_vcpu_destroy(struct kvm_vcpu *vcpu) @@ -286,7 +282,8 @@ void kvm_vgic_vcpu_destroy(struct kvm_vcpu *vcpu) INIT_LIST_HEAD(&vgic_cpu->ap_list_head); } -void kvm_vgic_destroy(struct kvm *kvm) +/* To be called with kvm->lock held */ +static void __kvm_vgic_destroy(struct kvm *kvm) { struct kvm_vcpu *vcpu; int i; @@ -297,6 +294,13 @@ void kvm_vgic_destroy(struct kvm *kvm) kvm_vgic_vcpu_destroy(vcpu); } +void kvm_vgic_destroy(struct kvm *kvm) +{ + mutex_lock(&kvm->lock); + __kvm_vgic_destroy(kvm); + mutex_unlock(&kvm->lock); +} + /** * vgic_lazy_init: Lazy init is only allowed if the GIC exposed to the guest * is a GICv2. A GICv3 must be explicitly initialized by the guest using the @@ -348,6 +352,10 @@ int kvm_vgic_map_resources(struct kvm *kvm) ret = vgic_v2_map_resources(kvm); else ret = vgic_v3_map_resources(kvm); + + if (ret) + __kvm_vgic_destroy(kvm); + out: mutex_unlock(&kvm->lock); return ret; diff --git a/virt/kvm/arm/vgic/vgic-v2.c b/virt/kvm/arm/vgic/vgic-v2.c index 9bab86757fa4f3..834137e7b83ff0 100644 --- a/virt/kvm/arm/vgic/vgic-v2.c +++ b/virt/kvm/arm/vgic/vgic-v2.c @@ -293,8 +293,6 @@ int vgic_v2_map_resources(struct kvm *kvm) dist->ready = true; out: - if (ret) - kvm_vgic_destroy(kvm); return ret; } diff --git a/virt/kvm/arm/vgic/vgic-v3.c b/virt/kvm/arm/vgic/vgic-v3.c index 5c9f9745e6cab8..e6b03fd8c374ca 100644 --- a/virt/kvm/arm/vgic/vgic-v3.c +++ b/virt/kvm/arm/vgic/vgic-v3.c @@ -302,8 +302,6 @@ int vgic_v3_map_resources(struct kvm *kvm) dist->ready = true; out: - if (ret) - kvm_vgic_destroy(kvm); return ret; } diff --git a/virt/lib/irqbypass.c b/virt/lib/irqbypass.c index 52abac4bb6a253..6d2fcd6fcb2509 100644 --- a/virt/lib/irqbypass.c +++ b/virt/lib/irqbypass.c @@ -195,7 +195,7 @@ int irq_bypass_register_consumer(struct irq_bypass_consumer *consumer) mutex_lock(&lock); list_for_each_entry(tmp, &consumers, node) { - if (tmp->token == consumer->token) { + if (tmp->token == consumer->token || tmp == consumer) { mutex_unlock(&lock); module_put(THIS_MODULE); return -EBUSY; @@ -245,7 +245,7 @@ void irq_bypass_unregister_consumer(struct irq_bypass_consumer *consumer) mutex_lock(&lock); list_for_each_entry(tmp, &consumers, node) { - if (tmp->token != consumer->token) + if (tmp != consumer) continue; list_for_each_entry(producer, &producers, node) {