android_kernel_xiaomi_sm8350/mm/migrate.c
Srinivasarao Pathipati b696332499 Merge android11-5.4.219+ (0ce03d1) into msm-5.4
* refs/heads/tmp-0ce03d1:
  Revert "wait: Fix __wait_event_hrtimeout for RT/DL tasks"
  Reverts below USB and netfilter patches
  BACKPORT: Kconfig.debug: provide a little extra FRAME_WARN leeway when KASAN is enabled
  UPSTREAM: bpf: Ensure correct locking around vulnerable function find_vpid()
  UPSTREAM: HID: roccat: Fix use-after-free in roccat_read()
  ANDROID: arm64: mm: perform clean & invalidation in __dma_map_area
  UPSTREAM: mmc: hsq: Fix data stomping during mmc recovery
  UPSTREAM: pinctrl: sunxi: Fix name for A100 R_PIO
  BACKPORT: mmc: core: Fix UHS-I SD 1.8V workaround branch
  UPSTREAM: Bluetooth: L2CAP: Fix l2cap_global_chan_by_psm regression
  UPSTREAM: wifi: mac80211_hwsim: set virtio device ready in probe()
  BACKPORT: f2fs: don't use casefolded comparison for "." and ".."
  UPSTREAM: Revert "mm/cma.c: remove redundant cma_mutex lock"
  UPSTREAM: usb: dwc3: Try usb-role-switch first in dwc3_drd_init
  BACKPORT: usb: typec: ucsi: Fix reuse of completion structure
  BACKPORT: tipc: fix incorrect order of state message data sanity check
  UPSTREAM: net: fix up skbs delta_truesize in UDP GRO frag_list
  UPSTREAM: cgroup-v1: Correct privileges check in release_agent writes
  UPSTREAM: mm: don't try to NUMA-migrate COW pages that have other uses
  UPSTREAM: usb: raw-gadget: fix handling of dual-direction-capable endpoints
  UPSTREAM: selinux: check return value of sel_make_avc_files
  UPSTREAM: usb: musb: select GENERIC_PHY instead of depending on it
  BACKPORT: driver core: Fix error return code in really_probe()
  UPSTREAM: fscrypt: fix derivation of SipHash keys on big endian CPUs
  BACKPORT: fscrypt: rename FS_KEY_DERIVATION_NONCE_SIZE
  UPSTREAM: socionext: account for napi_gro_receive never returning GRO_DROP
  UPSTREAM: net: socionext: netsec: fix xdp stats accounting
  BACKPORT: fs: align IOCB_* flags with RWF_* flags
  UPSTREAM: efi: capsule-loader: Fix use-after-free in efi_capsule_write
  BACKPORT: ARM: 9039/1: assembler: generalize byte swapping macro into rev_l
  BACKPORT: ARM: 9035/1: uncompress: Add be32tocpu macro
  UPSTREAM: drm/meson: Fix overflow implicit truncation warnings
  UPSTREAM: irqchip/tegra: Fix overflow implicit truncation warnings
  UPSTREAM: video: fbdev: pxa3xx-gcu: Fix integer overflow in pxa3xx_gcu_write
  UPSTREAM: mm/mremap: hold the rmap lock in write mode when moving page table entries.
  FROMLIST: binder: fix UAF of alloc->vma in race with munmap()
  UPSTREAM: mm: Fix TLB flush for not-first PFNMAP mappings in unmap_region()
  UPSTREAM: mm: Force TLB flush for PFNMAP mappings before unlink_file_vma()
  UPSTREAM: af_key: Do not call xfrm_probe_algs in parallel
  UPSTREAM: wifi: cfg80211: fix u8 overflow in cfg80211_update_notlisted_nontrans()
  UPSTREAM: wifi: cfg80211/mac80211: reject bad MBSSID elements
  UPSTREAM: wifi: cfg80211: ensure length byte is present before access
  UPSTREAM: wifi: cfg80211: fix BSS refcounting bugs
  UPSTREAM: wifi: cfg80211: avoid nontransmitted BSS list corruption
  UPSTREAM: wifi: mac80211_hwsim: avoid mac80211 warning on bad rate
  UPSTREAM: wifi: cfg80211: update hidden BSSes to avoid WARN_ON
  UPSTREAM: mac80211: mlme: find auth challenge directly
  UPSTREAM: wifi: mac80211: don't parse mbssid in assoc response
  ANDROID: GKI: db845c: Update symbols list and ABI
  UPSTREAM: wifi: mac80211: fix MBSSID parsing use-after-free
  ANDROID: Drop explicit 'CONFIG_INIT_STACK_ALL_ZERO=y' from gki_defconfig
  UPSTREAM: hardening: Remove Clang's enable flag for -ftrivial-auto-var-init=zero
  UPSTREAM: hardening: Avoid harmless Clang option under CONFIG_INIT_STACK_ALL_ZERO
  UPSTREAM: hardening: Clarify Kconfig text for auto-var-init
  ANDROID: GKI: Update FCNT KMI symbol list
  Linux 5.4.219
  wifi: mac80211: fix MBSSID parsing use-after-free
  wifi: mac80211: don't parse mbssid in assoc response
  mac80211: mlme: find auth challenge directly
  Revert "fs: check FMODE_LSEEK to control internal pipe splicing"
  Linux 5.4.218
  Input: xpad - fix wireless 360 controller breaking after suspend
  Input: xpad - add supported devices as contributed on github
  wifi: cfg80211: update hidden BSSes to avoid WARN_ON
  wifi: mac80211_hwsim: avoid mac80211 warning on bad rate
  wifi: cfg80211: avoid nontransmitted BSS list corruption
  wifi: cfg80211: fix BSS refcounting bugs
  wifi: cfg80211: ensure length byte is present before access
  wifi: cfg80211/mac80211: reject bad MBSSID elements
  wifi: cfg80211: fix u8 overflow in cfg80211_update_notlisted_nontrans()
  random: use expired timer rather than wq for mixing fast pool
  random: avoid reading two cache lines on irq randomness
  random: restore O_NONBLOCK support
  USB: serial: qcserial: add new usb-id for Dell branded EM7455
  scsi: stex: Properly zero out the passthrough command structure
  efi: Correct Macmini DMI match in uefi cert quirk
  ALSA: hda: Fix position reporting on Poulsbo
  random: clamp credited irq bits to maximum mixed
  ceph: don't truncate file in atomic_open
  nilfs2: replace WARN_ONs by nilfs_error for checkpoint acquisition failure
  nilfs2: fix leak of nilfs_root in case of writer thread creation failure
  nilfs2: fix NULL pointer dereference at nilfs_bmap_lookup_at_level()
  rpmsg: qcom: glink: replace strncpy() with strscpy_pad()
  mmc: core: Terminate infinite loop in SD-UHS voltage switch
  mmc: core: Replace with already defined values for readability
  USB: serial: ftdi_sio: fix 300 bps rate for SIO
  usb: mon: make mmapped memory read only
  arch: um: Mark the stack non-executable to fix a binutils warning
  um: Cleanup compiler warning in arch/x86/um/tls_32.c
  um: Cleanup syscall_handler_t cast in syscalls_32.h
  net/ieee802154: fix uninit value bug in dgram_sendmsg
  scsi: qedf: Fix a UAF bug in __qedf_probe()
  ARM: dts: fix Moxa SDIO 'compatible', remove 'sdhci' misnomer
  dmaengine: xilinx_dma: Report error in case of dma_set_mask_and_coherent API failure
  dmaengine: xilinx_dma: cleanup for fetching xlnx,num-fstores property
  firmware: arm_scmi: Add SCMI PM driver remove routine
  fs: fix UAF/GPF bug in nilfs_mdt_destroy
  perf tools: Fixup get_current_dir_name() compilation
  mm: pagewalk: Fix race between unmap and page walker
  ANDROID: Fix kenelci build-break for !CONFIG_PERF_EVENTS
  BACKPORT: HID: steam: Prevent NULL pointer dereference in steam_{recv,send}_report
  Linux 5.4.217
  docs: update mediator information in CoC docs
  Makefile.extrawarn: Move -Wcast-function-type-strict to W=1
  Revert "drm/amdgpu: use dirty framebuffer helper"
  xfs: remove unused variable 'done'
  xfs: fix uninitialized variable in xfs_attr3_leaf_inactive
  xfs: streamline xfs_attr3_leaf_inactive
  xfs: move incore structures out of xfs_da_format.h
  xfs: fix memory corruption during remote attr value buffer invalidation
  xfs: refactor remote attr value buffer invalidation
  xfs: fix IOCB_NOWAIT handling in xfs_file_dio_aio_read
  xfs: fix s_maxbytes computation on 32-bit kernels
  xfs: truncate should remove all blocks, not just to the end of the page cache
  xfs: introduce XFS_MAX_FILEOFF
  xfs: fix misuse of the XFS_ATTR_INCOMPLETE flag
  x86/speculation: Add RSB VM Exit protections
  x86/bugs: Warn when "ibrs" mitigation is selected on Enhanced IBRS parts
  x86/speculation: Use DECLARE_PER_CPU for x86_spec_ctrl_current
  x86/speculation: Disable RRSBA behavior
  x86/bugs: Add Cannon lake to RETBleed affected CPU list
  x86/cpu/amd: Enumerate BTC_NO
  x86/common: Stamp out the stepping madness
  x86/speculation: Fill RSB on vmexit for IBRS
  KVM: VMX: Fix IBRS handling after vmexit
  KVM: VMX: Prevent guest RSB poisoning attacks with eIBRS
  KVM: VMX: Convert launched argument to flags
  KVM: VMX: Flatten __vmx_vcpu_run()
  KVM/nVMX: Use __vmx_vcpu_run in nested_vmx_check_vmentry_hw
  KVM/VMX: Use TEST %REG,%REG instead of CMP $0,%REG in vmenter.S
  x86/speculation: Remove x86_spec_ctrl_mask
  x86/speculation: Use cached host SPEC_CTRL value for guest entry/exit
  x86/speculation: Fix SPEC_CTRL write on SMT state change
  x86/speculation: Fix firmware entry SPEC_CTRL handling
  x86/speculation: Fix RSB filling with CONFIG_RETPOLINE=n
  x86/speculation: Change FILL_RETURN_BUFFER to work with objtool
  intel_idle: Disable IBRS during long idle
  x86/bugs: Report Intel retbleed vulnerability
  x86/bugs: Split spectre_v2_select_mitigation() and spectre_v2_user_select_mitigation()
  x86/speculation: Add spectre_v2=ibrs option to support Kernel IBRS
  x86/bugs: Optimize SPEC_CTRL MSR writes
  x86/entry: Add kernel IBRS implementation
  x86/entry: Remove skip_r11rcx
  x86/bugs: Keep a per-CPU IA32_SPEC_CTRL value
  x86/bugs: Add AMD retbleed= boot parameter
  x86/bugs: Report AMD retbleed vulnerability
  x86/cpufeatures: Move RETPOLINE flags to word 11
  x86/kvm/vmx: Make noinstr clean
  x86/cpu: Add a steppings field to struct x86_cpu_id
  x86/cpu: Add consistent CPU match macros
  x86/devicetable: Move x86 specific macro out of generic code
  Revert "x86/cpu: Add a steppings field to struct x86_cpu_id"
  Revert "x86/speculation: Add RSB VM Exit protections"
  Linux 5.4.216
  clk: iproc: Do not rely on node name for correct PLL setup
  clk: imx: imx6sx: remove the SET_RATE_PARENT flag for QSPI clocks
  selftests: Fix the if conditions of in test_extra_filter()
  nvme: Fix IOC_PR_CLEAR and IOC_PR_RELEASE ioctls for nvme devices
  nvme: add new line after variable declatation
  usbnet: Fix memory leak in usbnet_disconnect()
  Input: melfas_mip4 - fix return value check in mip4_probe()
  Revert "drm: bridge: analogix/dp: add panel prepare/unprepare in suspend/resume time"
  soc: sunxi: sram: Fix debugfs info for A64 SRAM C
  soc: sunxi: sram: Fix probe function ordering issues
  soc: sunxi_sram: Make use of the helper function devm_platform_ioremap_resource()
  soc: sunxi: sram: Prevent the driver from being unbound
  soc: sunxi: sram: Actually claim SRAM regions
  ARM: dts: am33xx: Fix MMCHS0 dma properties
  ARM: dts: Move am33xx and am43xx mmc nodes to sdhci-omap driver
  media: dvb_vb2: fix possible out of bound access
  mm: fix madivse_pageout mishandling on non-LRU page
  mm/migrate_device.c: flush TLB while holding PTL
  mm: prevent page_frag_alloc() from corrupting the memory
  mm/page_alloc: fix race condition between build_all_zonelists and page allocation
  mmc: moxart: fix 4-bit bus width and remove 8-bit bus width
  libata: add ATA_HORKAGE_NOLPM for Pioneer BDR-207M and BDR-205
  Revert "net: mvpp2: debugfs: fix memory leak when using debugfs_lookup()"
  ntfs: fix BUG_ON in ntfs_lookup_inode_by_name()
  ARM: dts: integrator: Tag PCI host with device_type
  clk: ingenic-tcu: Properly enable registers before accessing timers
  net: usb: qmi_wwan: Add new usb-id for Dell branded EM7455
  uas: ignore UAS for Thinkplus chips
  usb-storage: Add Hiksemi USB3-FW to IGNORE_UAS
  uas: add no-uas quirk for Hiksemi usb_disk
  ANDROID: ABI: Update allowed list for QCOM
  UPSTREAM: wifi: mac80211_hwsim: use 32-bit skb cookie
  UPSTREAM: wifi: mac80211_hwsim: add back erroneously removed cast
  UPSTREAM: wifi: mac80211_hwsim: fix race condition in pending packet
  Linux 5.4.215
  ext4: make directory inode spreading reflect flexbg size
  xfs: fix use-after-free when aborting corrupt attr inactivation
  xfs: fix an ABBA deadlock in xfs_rename
  xfs: don't commit sunit/swidth updates to disk if that would cause repair failures
  xfs: split the sunit parameter update into two parts
  xfs: refactor agfl length computation function
  xfs: use bitops interface for buf log item AIL flag check
  xfs: stabilize insert range start boundary to avoid COW writeback race
  xfs: fix some memory leaks in log recovery
  xfs: always log corruption errors
  xfs: constify the buffer pointer arguments to error functions
  xfs: convert EIO to EFSCORRUPTED when log contents are invalid
  xfs: Fix deadlock between AGI and AGF when target_ip exists in xfs_rename()
  xfs: attach dquots and reserve quota blocks during unwritten conversion
  xfs: range check ri_cnt when recovering log items
  xfs: add missing assert in xfs_fsmap_owner_from_rmap
  xfs: slightly tweak an assert in xfs_fs_map_blocks
  xfs: replace -EIO with -EFSCORRUPTED for corrupt metadata
  ext4: fix bug in extents parsing when eh_entries == 0 and eh_depth > 0
  workqueue: don't skip lockdep work dependency in cancel_work_sync()
  drm/rockchip: Fix return type of cdn_dp_connector_mode_valid
  drm/amd/display: Limit user regamma to a valid value
  drm/amdgpu: use dirty framebuffer helper
  Drivers: hv: Never allocate anything besides framebuffer from framebuffer memory region
  cifs: always initialize struct msghdr smb_msg completely
  usb: xhci-mtk: fix issue of out-of-bounds array access
  s390/dasd: fix Oops in dasd_alias_get_start_dev due to missing pavgroup
  serial: tegra-tcu: Use uart_xmit_advance(), fixes icount.tx accounting
  serial: tegra: Use uart_xmit_advance(), fixes icount.tx accounting
  serial: Create uart_xmit_advance()
  net: sched: fix possible refcount leak in tc_new_tfilter()
  net: sunhme: Fix packet reception for len < RX_COPY_THRESHOLD
  perf kcore_copy: Do not check /proc/modules is unchanged
  perf jit: Include program header in ELF files
  can: gs_usb: gs_can_open(): fix race dev->can.state condition
  netfilter: ebtables: fix memory leak when blob is malformed
  net/sched: taprio: make qdisc_leaf() see the per-netdev-queue pfifo child qdiscs
  net/sched: taprio: avoid disabling offload when it was never enabled
  of: mdio: Add of_node_put() when breaking out of for_each_xx
  i40e: Fix set max_tx_rate when it is lower than 1 Mbps
  i40e: Fix VF set max MTU size
  iavf: Fix set max MTU size with port VLAN and jumbo frames
  iavf: Fix bad page state
  MIPS: Loongson32: Fix PHY-mode being left unspecified
  MIPS: lantiq: export clk_get_io() for lantiq_wdt.ko
  net: team: Unsync device addresses on ndo_stop
  ipvlan: Fix out-of-bound bugs caused by unset skb->mac_header
  iavf: Fix cached head and tail value for iavf_get_tx_pending
  netfilter: nfnetlink_osf: fix possible bogus match in nf_osf_find()
  netfilter: nf_conntrack_irc: Tighten matching on DCC message
  netfilter: nf_conntrack_sip: fix ct_sip_walk_headers
  arm64: dts: rockchip: Remove 'enable-active-low' from rk3399-puma
  arm64: dts: rockchip: Set RK3399-Gru PCLK_EDP to 24 MHz
  arm64: dts: rockchip: Pull up wlan wake# on Gru-Bob
  mm/slub: fix to return errno if kmalloc() fails
  efi: libstub: check Shim mode using MokSBStateRT
  ALSA: hda/realtek: Enable 4-speaker output Dell Precision 5530 laptop
  ALSA: hda/realtek: Add quirk for ASUS GA503R laptop
  ALSA: hda/realtek: Add pincfg for ASUS G533Z HP jack
  ALSA: hda/realtek: Add pincfg for ASUS G513 HP jack
  ALSA: hda/realtek: Re-arrange quirk table entries
  ALSA: hda/realtek: Add quirk for Huawei WRT-WX9
  ALSA: hda: add Intel 5 Series / 3400 PCI DID
  ALSA: hda/tegra: set depop delay for tegra
  USB: serial: option: add Quectel RM520N
  USB: serial: option: add Quectel BG95 0x0203 composition
  USB: core: Fix RST error in hub.c
  Revert "usb: gadget: udc-xilinx: replace memcpy with memcpy_toio"
  Revert "usb: add quirks for Lenovo OneLink+ Dock"
  usb: cdns3: fix issue with rearming ISO OUT endpoint
  usb: gadget: udc-xilinx: replace memcpy with memcpy_toio
  usb: add quirks for Lenovo OneLink+ Dock
  tty: serial: atmel: Preserve previous USART mode if RS485 disabled
  serial: atmel: remove redundant assignment in rs485_config
  tty/serial: atmel: RS485 & ISO7816: wait for TXRDY before sending data
  wifi: mac80211: Fix UAF in ieee80211_scan_rx()
  usb: xhci-mtk: relax TT periodic bandwidth allocation
  usb: xhci-mtk: allow multiple Start-Split in a microframe
  usb: xhci-mtk: add some schedule error number
  usb: xhci-mtk: add a function to (un)load bandwidth info
  usb: xhci-mtk: use @sch_tt to check whether need do TT schedule
  usb: xhci-mtk: add only one extra CS for FS/LS INTR
  usb: xhci-mtk: get the microframe boundary for ESIT
  usb: dwc3: gadget: Avoid duplicate requests to enable Run/Stop
  usb: dwc3: gadget: Don't modify GEVNTCOUNT in pullup()
  usb: dwc3: gadget: Refactor pullup()
  usb: dwc3: gadget: Prevent repeat pullup()
  usb: dwc3: Issue core soft reset before enabling run/stop
  usb: dwc3: gadget: Avoid starting DWC3 gadget during UDC unbind
  ALSA: hda/sigmatel: Fix unused variable warning for beep power change
  cgroup: Add missing cpus_read_lock() to cgroup_attach_task_all()
  video: fbdev: pxa3xx-gcu: Fix integer overflow in pxa3xx_gcu_write
  mksysmap: Fix the mismatch of 'L0' symbols in System.map
  MIPS: OCTEON: irq: Fix octeon_irq_force_ciu_mapping()
  afs: Return -EAGAIN, not -EREMOTEIO, when a file already locked
  net: usb: qmi_wwan: add Quectel RM520N
  ALSA: hda/tegra: Align BDL entry to 4KB boundary
  ALSA: hda/sigmatel: Keep power up while beep is enabled
  rxrpc: Fix calc of resend age
  rxrpc: Fix local destruction being repeated
  regulator: pfuze100: Fix the global-out-of-bounds access in pfuze100_regulator_probe()
  ASoC: nau8824: Fix semaphore unbalance at error paths
  iomap: iomap that extends beyond EOF should be marked dirty
  MAINTAINERS: add Chandan as xfs maintainer for 5.4.y
  cifs: don't send down the destination address to sendmsg for a SOCK_STREAM
  cifs: revalidate mapping when doing direct writes
  tracing: hold caller_addr to hardirq_{enable,disable}_ip
  task_stack, x86/cea: Force-inline stack helpers
  ALSA: pcm: oss: Fix race at SNDCTL_DSP_SYNC
  parisc: ccio-dma: Add missing iounmap in error path in ccio_probe()
  drm/meson: Fix OSD1 RGB to YCbCr coefficient
  drm/meson: Correct OSD1 global alpha value
  gpio: mpc8xxx: Fix support for IRQ_TYPE_LEVEL_LOW flow_type in mpc85xx
  NFSv4: Turn off open-by-filehandle and NFS re-export for NFSv4.0
  of: fdt: fix off-by-one error in unflatten_dt_nodes()
  Revert "USB: core: Prevent nested device-reset calls"
  Revert "io_uring: disable polling pollfree files"
  Revert "netfilter: conntrack: NF_CONNTRACK_PROCFS should no longer default to y"
  Revert "sched/deadline: Fix priority inheritance with multiple scheduling classes"
  Revert "kernel/sched: Remove dl_boosted flag comment"
  Revert "mm/rmap: Fix anon_vma->degree ambiguity leading to double-reuse"
  Revert "fs: check FMODE_LSEEK to control internal pipe splicing"
  Linux 5.4.214
  tracefs: Only clobber mode/uid/gid on remount if asked
  soc: fsl: select FSL_GUTS driver for DPIO
  net: dp83822: disable rx error interrupt
  mm: Fix TLB flush for not-first PFNMAP mappings in unmap_region()
  usb: storage: Add ASUS <0x0b05:0x1932> to IGNORE_UAS
  platform/x86: acer-wmi: Acer Aspire One AOD270/Packard Bell Dot keymap fixes
  perf/arm_pmu_platform: fix tests for platform_get_irq() failure
  nvmet-tcp: fix unhandled tcp states in nvmet_tcp_state_change()
  Input: iforce - add support for Boeder Force Feedback Wheel
  ieee802154: cc2520: add rc code in cc2520_tx()
  tg3: Disable tg3 device on system reboot to avoid triggering AER
  hid: intel-ish-hid: ishtp: Fix ishtp client sending disordered message
  HID: ishtp-hid-clientHID: ishtp-hid-client: Fix comment typo
  drm/msm/rd: Fix FIFO-full deadlock
  Linux 5.4.213
  MIPS: loongson32: ls1c: Fix hang during startup
  x86/nospec: Fix i386 RSB stuffing
  sch_sfb: Also store skb len before calling child enqueue
  tcp: fix early ETIMEDOUT after spurious non-SACK RTO
  nvme-tcp: fix UAF when detecting digest errors
  RDMA/mlx5: Set local port to one when accessing counters
  ipv6: sr: fix out-of-bounds read when setting HMAC data.
  RDMA/siw: Pass a pointer to virt_to_page()
  i40e: Fix kernel crash during module removal
  tipc: fix shift wrapping bug in map_get()
  sch_sfb: Don't assume the skb is still around after enqueueing to child
  afs: Use the operation issue time instead of the reply time for callbacks
  rxrpc: Fix an insufficiently large sglist in rxkad_verify_packet_2()
  netfilter: nf_conntrack_irc: Fix forged IP logic
  netfilter: br_netfilter: Drop dst references before setting.
  RDMA/hns: Fix supported page size
  soc: brcmstb: pm-arm: Fix refcount leak and __iomem leak bugs
  RDMA/cma: Fix arguments order in net device validation
  regulator: core: Clean up on enable failure
  ARM: dts: imx6qdl-kontron-samx6i: remove duplicated node
  smb3: missing inode locks in punch hole
  cgroup: Fix threadgroup_rwsem <-> cpus_read_lock() deadlock
  cgroup: Elide write-locking threadgroup_rwsem when updating csses on an empty subtree
  cgroup: Optimize single thread migration
  scsi: lpfc: Add missing destroy_workqueue() in error path
  scsi: mpt3sas: Fix use-after-free warning
  nvmet: fix a use-after-free
  debugfs: add debugfs_lookup_and_remove()
  kprobes: Prohibit probes in gate area
  ALSA: usb-audio: Fix an out-of-bounds bug in __snd_usb_parse_audio_interface()
  ALSA: aloop: Fix random zeros in capture data when using jiffies timer
  ALSA: emu10k1: Fix out of bounds access in snd_emu10k1_pcm_channel_alloc()
  drm/amdgpu: mmVM_L2_CNTL3 register not initialized correctly
  fbdev: chipsfb: Add missing pci_disable_device() in chipsfb_pci_init()
  arm64: cacheinfo: Fix incorrect assignment of signed error value to unsigned fw_level
  parisc: Add runtime check to prevent PA2.0 kernels on PA1.x machines
  parisc: ccio-dma: Handle kmalloc failure in ccio_init_resources()
  drm/radeon: add a force flush to delay work when radeon
  drm/amdgpu: Check num_gfx_rings for gfx v9_0 rb setup.
  drm/gem: Fix GEM handle release errors
  scsi: megaraid_sas: Fix double kfree()
  USB: serial: ch341: fix disabled rx timer on older devices
  USB: serial: ch341: fix lost character on LCR updates
  usb: dwc3: disable USB core PHY management
  usb: dwc3: fix PHY disable sequence
  btrfs: harden identification of a stale device
  drm/i915/glk: ECS Liva Q2 needs GLK HDMI port timing quirk
  ALSA: seq: Fix data-race at module auto-loading
  ALSA: seq: oss: Fix data-race for max_midi_devs access
  net: mac802154: Fix a condition in the receive path
  ip: fix triggering of 'icmp redirect'
  wifi: mac80211: Don't finalize CSA in IBSS mode if state is disconnected
  driver core: Don't probe devices after bus_type.match() probe deferral
  usb: gadget: mass_storage: Fix cdrom data transfers on MAC-OS
  USB: core: Prevent nested device-reset calls
  s390: fix nospec table alignments
  s390/hugetlb: fix prepare_hugepage_range() check for 2 GB hugepages
  usb-storage: Add ignore-residue quirk for NXP PN7462AU
  USB: cdc-acm: Add Icom PMR F3400 support (0c26:0020)
  usb: dwc2: fix wrong order of phy_power_on and phy_init
  usb: typec: altmodes/displayport: correct pin assignment for UFP receptacles
  USB: serial: option: add support for Cinterion MV32-WA/WB RmNet mode
  USB: serial: option: add Quectel EM060K modem
  USB: serial: option: add support for OPPO R11 diag port
  USB: serial: cp210x: add Decagon UCA device id
  xhci: Add grace period after xHC start to prevent premature runtime suspend.
  thunderbolt: Use the actual buffer in tb_async_error()
  gpio: pca953x: Add mutex_lock for regcache sync in PM
  hwmon: (gpio-fan) Fix array out of bounds access
  clk: bcm: rpi: Fix error handling of raspberrypi_fw_get_rate
  Input: rk805-pwrkey - fix module autoloading
  clk: core: Fix runtime PM sequence in clk_core_unprepare()
  Revert "clk: core: Honor CLK_OPS_PARENT_ENABLE for clk gate ops"
  clk: core: Honor CLK_OPS_PARENT_ENABLE for clk gate ops
  drm/i915/reg: Fix spelling mistake "Unsupport" -> "Unsupported"
  usb: dwc3: qcom: fix use-after-free on runtime-PM wakeup
  binder: fix UAF of ref->proc caused by race condition
  USB: serial: ftdi_sio: add Omron CS1W-CIF31 device id
  misc: fastrpc: fix memory corruption on open
  misc: fastrpc: fix memory corruption on probe
  iio: adc: mcp3911: use correct formula for AD conversion
  Input: iforce - wake up after clearing IFORCE_XMIT_RUNNING flag
  tty: serial: lpuart: disable flow control while waiting for the transmit engine to complete
  vt: Clear selection before changing the font
  powerpc: align syscall table for ppc32
  staging: rtl8712: fix use after free bugs
  serial: fsl_lpuart: RS485 RTS polariy is inverse
  net/smc: Remove redundant refcount increase
  Revert "sch_cake: Return __NET_XMIT_STOLEN when consuming enqueued skb"
  tcp: annotate data-race around challenge_timestamp
  sch_cake: Return __NET_XMIT_STOLEN when consuming enqueued skb
  kcm: fix strp_init() order and cleanup
  ethernet: rocker: fix sleep in atomic context bug in neigh_timer_handler
  net: sched: tbf: don't call qdisc_put() while holding tree lock
  Revert "xhci: turn off port power in shutdown"
  wifi: cfg80211: debugfs: fix return type in ht40allow_map_read()
  ieee802154/adf7242: defer destroy_workqueue call
  iio: adc: mcp3911: make use of the sign bit
  platform/x86: pmc_atom: Fix SLP_TYPx bitfield mask
  drm/msm/dsi: Fix number of regulators for msm8996_dsi_cfg
  drm/msm/dsi: fix the inconsistent indenting
  net: dp83822: disable false carrier interrupt
  Revert "mm: kmemleak: take a full lowmem check in kmemleak_*_phys()"
  fs: only do a memory barrier for the first set_buffer_uptodate()
  net: mvpp2: debugfs: fix memory leak when using debugfs_lookup()
  wifi: iwlegacy: 4965: corrected fix for potential off-by-one overflow in il4965_rs_fill_link_cmd()
  efi: capsule-loader: Fix use-after-free in efi_capsule_write
  Linux 5.4.212
  net: neigh: don't call kfree_skb() under spin_lock_irqsave()
  net/af_packet: check len when min_header_len equals to 0
  io_uring: disable polling pollfree files
  kprobes: don't call disarm_kprobe() for disabled kprobes
  lib/vdso: Mark do_hres() and do_coarse() as __always_inline
  lib/vdso: Let do_coarse() return 0 to simplify the callsite
  btrfs: tree-checker: check for overlapping extent items
  netfilter: conntrack: NF_CONNTRACK_PROCFS should no longer default to y
  drm/amd/display: Fix pixel clock programming
  s390/hypfs: avoid error message under KVM
  neigh: fix possible DoS due to net iface start/stop loop
  drm/amd/display: clear optc underflow before turn off odm clock
  drm/amd/display: Avoid MPC infinite loop
  btrfs: unify lookup return value when dir entry is missing
  btrfs: do not pin logs too early during renames
  btrfs: introduce btrfs_lookup_match_dir
  mm/rmap: Fix anon_vma->degree ambiguity leading to double-reuse
  bpf: Don't redirect packets with invalid pkt_len
  ftrace: Fix NULL pointer dereference in is_ftrace_trampoline when ftrace is dead
  fbdev: fb_pm2fb: Avoid potential divide by zero error
  HID: hidraw: fix memory leak in hidraw_release()
  media: pvrusb2: fix memory leak in pvr_probe
  udmabuf: Set the DMA mask for the udmabuf device (v2)
  HID: steam: Prevent NULL pointer dereference in steam_{recv,send}_report
  Bluetooth: L2CAP: Fix build errors in some archs
  kbuild: Fix include path in scripts/Makefile.modpost
  x86/bugs: Add "unknown" reporting for MMIO Stale Data
  s390/mm: do not trigger write fault when vma does not allow VM_WRITE
  mm: Force TLB flush for PFNMAP mappings before unlink_file_vma()
  scsi: storvsc: Remove WQ_MEM_RECLAIM from storvsc_error_wq
  perf/x86/intel/uncore: Fix broken read_counter() for SNB IMC PMU
  md: call __md_stop_writes in md_stop
  mm/hugetlb: fix hugetlb not supporting softdirty tracking
  ACPI: processor: Remove freq Qos request for all CPUs
  s390: fix double free of GS and RI CBs on fork() failure
  asm-generic: sections: refactor memory_intersects
  loop: Check for overflow while configuring loop
  x86/unwind/orc: Unwind ftrace trampolines with correct ORC entry
  btrfs: check if root is readonly while setting security xattr
  btrfs: add info when mount fails due to stale replace target
  btrfs: replace: drop assert for suspended replace
  btrfs: fix silent failure when deleting root reference
  ixgbe: stop resetting SYSTIME in ixgbe_ptp_start_cyclecounter
  net: Fix a data-race around sysctl_somaxconn.
  net: Fix a data-race around netdev_budget_usecs.
  net: Fix a data-race around netdev_budget.
  net: Fix a data-race around sysctl_net_busy_read.
  net: Fix a data-race around sysctl_net_busy_poll.
  net: Fix a data-race around sysctl_tstamp_allow_data.
  ratelimit: Fix data-races in ___ratelimit().
  net: Fix data-races around netdev_tstamp_prequeue.
  net: Fix data-races around weight_p and dev_weight_[rt]x_bias.
  netfilter: nft_tunnel: restrict it to netdev family
  netfilter: nft_osf: restrict osf to ipv4, ipv6 and inet families
  netfilter: nft_payload: do not truncate csum_offset and csum_type
  netfilter: nft_payload: report ERANGE for too long offset and length
  bnxt_en: fix NQ resource accounting during vf creation on 57500 chips
  netfilter: ebtables: reject blobs that don't provide all entry points
  net: ipvtap - add __init/__exit annotations to module init/exit funcs
  bonding: 802.3ad: fix no transmission of LACPDUs
  net: moxa: get rid of asymmetry in DMA mapping/unmapping
  net/mlx5e: Properly disable vlan strip on non-UL reps
  rose: check NULL rose_loopback_neigh->loopback
  SUNRPC: RPC level errors should set task->tk_rpc_status
  af_key: Do not call xfrm_probe_algs in parallel
  xfrm: fix refcount leak in __xfrm_policy_check()
  kernel/sched: Remove dl_boosted flag comment
  sched/deadline: Fix priority inheritance with multiple scheduling classes
  sched/deadline: Fix stale throttling on de-/boosted tasks
  sched/deadline: Unthrottle PI boosted threads while enqueuing
  pinctrl: amd: Don't save/restore interrupt status and wake status bits
  Revert "selftests/bpf: Fix test_align verifier log patterns"
  Revert "selftests/bpf: Fix "dubious pointer arithmetic" test"
  usb: cdns3: Fix issue for clear halt endpoint
  kernel/sys_ni: add compat entry for fadvise64_64
  parisc: Fix exception handler for fldw and fstw instructions
  audit: fix potential double free on error path from fsnotify_add_inode_mark
  Revert "USB: HCD: Fix URB giveback issue in tasklet function"
  Linux 5.4.211
  btrfs: raid56: don't trust any cached sector in __raid56_parity_recover()
  btrfs: only write the sectors in the vertical stripe which has data stripes
  can: j1939: j1939_session_destroy(): fix memory leak of skbs
  can: j1939: j1939_sk_queue_activate_next_locked(): replace WARN_ON_ONCE with netdev_warn_once()
  tracing/probes: Have kprobes and uprobes use $COMM too
  MIPS: tlbex: Explicitly compare _PAGE_NO_EXEC against 0
  video: fbdev: i740fb: Check the argument of i740_calc_vclk()
  powerpc/64: Init jump labels before parse_early_param()
  smb3: check xattr value length earlier
  f2fs: fix to avoid use f2fs_bug_on() in f2fs_new_node_page()
  ALSA: timer: Use deferred fasync helper
  ALSA: core: Add async signal helpers
  powerpc/32: Don't always pass -mcpu=powerpc to the compiler
  watchdog: export lockup_detector_reconfigure
  RISC-V: Add fast call path of crash_kexec()
  riscv: mmap with PROT_WRITE but no PROT_READ is invalid
  mips: cavium-octeon: Fix missing of_node_put() in octeon2_usb_clocks_start
  vfio: Clear the caps->buf to NULL after free
  tty: serial: Fix refcount leak bug in ucc_uart.c
  lib/list_debug.c: Detect uninitialized lists
  ext4: avoid resizing to a partial cluster size
  ext4: avoid remove directory when directory is corrupted
  drivers:md:fix a potential use-after-free bug
  nvmet-tcp: fix lockdep complaint on nvmet_tcp_wq flush during queue teardown
  dmaengine: sprd: Cleanup in .remove() after pm_runtime_get_sync() failed
  selftests/kprobe: Do not test for GRP/ without event failures
  um: add "noreboot" command line option for PANIC_TIMEOUT=-1 setups
  PCI/ACPI: Guard ARM64-specific mcfg_quirks
  cxl: Fix a memory leak in an error handling path
  gadgetfs: ep_io - wait until IRQ finishes
  scsi: lpfc: Prevent buffer overflow crashes in debugfs with malformed user input
  clk: qcom: ipq8074: dont disable gcc_sleep_clk_src
  vboxguest: Do not use devm for irq
  usb: renesas: Fix refcount leak bug
  usb: host: ohci-ppc-of: Fix refcount leak bug
  drm/meson: Fix overflow implicit truncation warnings
  irqchip/tegra: Fix overflow implicit truncation warnings
  usb: gadget: uvc: call uvc uvcg_warn on completed status instead of uvcg_info
  usb: cdns3 fix use-after-free at workaround 2
  PCI: Add ACS quirk for Broadcom BCM5750x NICs
  drm/meson: Fix refcount bugs in meson_vpu_has_available_connectors()
  locking/atomic: Make test_and_*_bit() ordered on failure
  gcc-plugins: Undefine LATENT_ENTROPY_PLUGIN when plugin disabled for a file
  igb: Add lock to avoid data race
  fec: Fix timer capture timing in `fec_ptp_enable_pps()`
  i40e: Fix to stop tx_timeout recovery if GLOBR fails
  ice: Ignore EEXIST when setting promisc mode
  net: dsa: microchip: ksz9477: fix fdb_dump last invalid entry
  net: moxa: pass pdev instead of ndev to DMA functions
  net: dsa: mv88e6060: prevent crash on an unused port
  powerpc/pci: Fix get_phb_number() locking
  netfilter: nf_tables: really skip inactive sets when allocating name
  clk: rockchip: add sclk_mac_lbtest to rk3188_critical_clocks
  iavf: Fix adminq error handling
  nios2: add force_successful_syscall_return()
  nios2: restarts apply only to the first sigframe we build...
  nios2: fix syscall restart checks
  nios2: traced syscall does need to check the syscall number
  nios2: don't leave NULLs in sys_call_table[]
  nios2: page fault et.al. are *not* restartable syscalls...
  tee: add overflow check in register_shm_helper()
  dpaa2-eth: trace the allocated address instead of page struct
  atm: idt77252: fix use-after-free bugs caused by tst_timer
  xen/xenbus: fix return type in xenbus_file_read()
  nfp: ethtool: fix the display error of `ethtool -m DEVNAME`
  NTB: ntb_tool: uninitialized heap data in tool_fn_write()
  tools build: Switch to new openssl API for test-libcrypto
  tools/vm/slabinfo: use alphabetic order when two values are equal
  dt-bindings: arm: qcom: fix MSM8916 MTP compatibles
  vsock: Set socket state back to SS_UNCONNECTED in vsock_connect_timeout()
  vsock: Fix memory leak in vsock_connect()
  plip: avoid rcu debug splat
  geneve: do not use RT_TOS for IPv6 flowlabel
  ACPI: property: Return type of acpi_add_nondev_subnodes() should be bool
  pinctrl: sunxi: Add I/O bias setting for H6 R-PIO
  pinctrl: qcom: msm8916: Allow CAMSS GP clocks to be muxed
  pinctrl: nomadik: Fix refcount leak in nmk_pinctrl_dt_subnode_to_map
  net: bgmac: Fix a BUG triggered by wrong bytes_compl
  devlink: Fix use-after-free after a failed reload
  SUNRPC: Reinitialise the backchannel request buffers before reuse
  sunrpc: fix expiry of auth creds
  can: mcp251x: Fix race condition on receive interrupt
  NFSv4/pnfs: Fix a use-after-free bug in open
  NFSv4.1: RECLAIM_COMPLETE must handle EACCES
  NFSv4: Fix races in the legacy idmapper upcall
  NFSv4.1: Handle NFS4ERR_DELAY replies to OP_SEQUENCE correctly
  NFSv4.1: Don't decrease the value of seq_nr_highest_sent
  Documentation: ACPI: EINJ: Fix obsolete example
  apparmor: Fix memleak in aa_simple_write_to_buffer()
  apparmor: fix reference count leak in aa_pivotroot()
  apparmor: fix overlapping attachment computation
  apparmor: fix aa_label_asxprint return check
  apparmor: Fix failed mount permission check error message
  apparmor: fix absroot causing audited secids to begin with =
  apparmor: fix quiet_denied for file rules
  can: ems_usb: fix clang's -Wunaligned-access warning
  tracing: Have filter accept "common_cpu" to be consistent
  btrfs: fix lost error handling when looking up extended ref on log replay
  mmc: pxamci: Fix an error handling path in pxamci_probe()
  mmc: pxamci: Fix another error handling path in pxamci_probe()
  ata: libata-eh: Add missing command name
  rds: add missing barrier to release_refill
  ALSA: info: Fix llseek return value when using callback
  net_sched: cls_route: disallow handle of 0
  net/9p: Initialize the iounit field during fid creation
  Bluetooth: L2CAP: Fix l2cap_global_chan_by_psm regression
  Revert "net: usb: ax88179_178a needs FLAG_SEND_ZLP"
  scsi: sg: Allow waiting for commands to complete on removed device
  tcp: fix over estimation in sk_forced_mem_schedule()
  KVM: x86: Avoid theoretical NULL pointer dereference in kvm_irq_delivery_to_apic_fast()
  KVM: x86: Check lapic_in_kernel() before attempting to set a SynIC irq
  KVM: Add infrastructure and macro to mark VM as bugged
  btrfs: reject log replay if there is unsupported RO compat flag
  net_sched: cls_route: remove from list when handle is 0
  iommu/vt-d: avoid invalid memory access via node_online(NUMA_NO_NODE)
  firmware: arm_scpi: Ensure scpi_info is not assigned if the probe fails
  timekeeping: contribute wall clock to rng on time change
  ACPI: CPPC: Do not prevent CPPC from working in the future
  dm writecache: set a default MAX_WRITEBACK_JOBS
  dm thin: fix use-after-free crash in dm_sm_register_threshold_callback
  dm raid: fix address sanitizer warning in raid_status
  dm raid: fix address sanitizer warning in raid_resume
  intel_th: pci: Add Meteor Lake-P support
  intel_th: pci: Add Raptor Lake-S PCH support
  intel_th: pci: Add Raptor Lake-S CPU support
  ext4: correct the misjudgment in ext4_iget_extra_inode
  ext4: correct max_inline_xattr_value_size computing
  ext4: fix extent status tree race in writeback error recovery path
  ext4: update s_overhead_clusters in the superblock during an on-line resize
  ext4: fix use-after-free in ext4_xattr_set_entry
  ext4: make sure ext4_append() always allocates new block
  ext4: add EXT4_INODE_HAS_XATTR_SPACE macro in xattr.h
  btrfs: reset block group chunk force if we have to wait
  tpm: eventlog: Fix section mismatch for DEBUG_SECTION_MISMATCH
  kexec, KEYS, s390: Make use of built-in and secondary keyring for signature verification
  spmi: trace: fix stack-out-of-bound access in SPMI tracing functions
  x86/olpc: fix 'logical not is only applied to the left hand side'
  scsi: qla2xxx: Fix erroneous mailbox timeout after PCI error injection
  scsi: qla2xxx: Turn off multi-queue for 8G adapters
  scsi: qla2xxx: Fix discovery issues in FC-AL topology
  scsi: zfcp: Fix missing auto port scan and thus missing target ports
  video: fbdev: s3fb: Check the size of screen before memset_io()
  video: fbdev: arkfb: Check the size of screen before memset_io()
  video: fbdev: vt8623fb: Check the size of screen before memset_io()
  tools/thermal: Fix possible path truncations
  video: fbdev: arkfb: Fix a divide-by-zero bug in ark_set_pixclock()
  x86/numa: Use cpumask_available instead of hardcoded NULL check
  scripts/faddr2line: Fix vmlinux detection on arm64
  genelf: Use HAVE_LIBCRYPTO_SUPPORT, not the never defined HAVE_LIBCRYPTO
  powerpc/pci: Fix PHB numbering when using opal-phbid
  kprobes: Forbid probing on trampoline and BPF code areas
  perf symbol: Fail to read phdr workaround
  powerpc/cell/axon_msi: Fix refcount leak in setup_msi_msg_address
  powerpc/xive: Fix refcount leak in xive_get_max_prio
  powerpc/spufs: Fix refcount leak in spufs_init_isolated_loader
  powerpc/pci: Prefer PCI domain assignment via DT 'linux,pci-domain' and alias
  powerpc/32: Do not allow selection of e5500 or e6500 CPUs on PPC32
  video: fbdev: sis: fix typos in SiS_GetModeID()
  video: fbdev: amba-clcd: Fix refcount leak bugs
  watchdog: armada_37xx_wdt: check the return value of devm_ioremap() in armada_37xx_wdt_probe()
  ASoC: audio-graph-card: Add of_node_put() in fail path
  fuse: Remove the control interface for virtio-fs
  ASoC: qcom: q6dsp: Fix an off-by-one in q6adm_alloc_copp()
  s390/zcore: fix race when reading from hardware system area
  iommu/arm-smmu: qcom_iommu: Add of_node_put() when breaking out of loop
  mfd: max77620: Fix refcount leak in max77620_initialise_fps
  mfd: t7l66xb: Drop platform disable callback
  kfifo: fix kfifo_to_user() return type
  rpmsg: qcom_smd: Fix refcount leak in qcom_smd_parse_edge
  iommu/exynos: Handle failed IOMMU device registration properly
  tty: n_gsm: fix missing corner cases in gsmld_poll()
  tty: n_gsm: fix DM command
  tty: n_gsm: fix wrong T1 retry count handling
  vfio/ccw: Do not change FSM state in subchannel event
  remoteproc: qcom: wcnss: Fix handling of IRQs
  tty: n_gsm: fix race condition in gsmld_write()
  tty: n_gsm: fix packet re-transmission without open control channel
  tty: n_gsm: fix non flow control frames during mux flow off
  profiling: fix shift too large makes kernel panic
  ASoC: codecs: wcd9335: move gains from SX_TLV to S8_TLV
  ASoC: codecs: msm8916-wcd-digital: move gains from SX_TLV to S8_TLV
  serial: 8250_dw: Store LSR into lsr_saved_flags in dw8250_tx_wait_empty()
  ASoC: mediatek: mt8173-rt5650: Fix refcount leak in mt8173_rt5650_dev_probe
  ASoC: codecs: da7210: add check for i2c_add_driver
  ASoC: mt6797-mt6351: Fix refcount leak in mt6797_mt6351_dev_probe
  ASoC: mediatek: mt8173: Fix refcount leak in mt8173_rt5650_rt5676_dev_probe
  opp: Fix error check in dev_pm_opp_attach_genpd()
  jbd2: fix assertion 'jh->b_frozen_data == NULL' failure when journal aborted
  ext4: recover csum seed of tmp_inode after migrating to extents
  jbd2: fix outstanding credits assert in jbd2_journal_commit_transaction()
  null_blk: fix ida error handling in null_add_dev()
  RDMA/rxe: Fix error unwind in rxe_create_qp()
  mm/mmap.c: fix missing call to vm_unacct_memory in mmap_region
  platform/olpc: Fix uninitialized data in debugfs write
  USB: serial: fix tty-port initialized comments
  PCI: tegra194: Fix link up retry sequence
  PCI: tegra194: Fix Root Port interrupt handling
  HID: alps: Declare U1_UNICORN_LEGACY support
  mmc: cavium-thunderx: Add of_node_put() when breaking out of loop
  mmc: cavium-octeon: Add of_node_put() when breaking out of loop
  gpio: gpiolib-of: Fix refcount bugs in of_mm_gpiochip_add_data()
  RDMA/hfi1: fix potential memory leak in setup_base_ctxt()
  RDMA/siw: Fix duplicated reported IW_CM_EVENT_CONNECT_REPLY event
  RDMA/hns: Fix incorrect clearing of interrupt status register
  usb: gadget: udc: amd5536 depends on HAS_DMA
  scsi: smartpqi: Fix DMA direction for RAID requests
  mmc: sdhci-of-at91: fix set_uhs_signaling rewriting of MC1R
  memstick/ms_block: Fix a memory leak
  memstick/ms_block: Fix some incorrect memory allocation
  mmc: sdhci-of-esdhc: Fix refcount leak in esdhc_signal_voltage_switch
  staging: rtl8192u: Fix sleep in atomic context bug in dm_fsync_timer_callback
  intel_th: msu: Fix vmalloced buffers
  intel_th: msu-sink: Potential dereference of null pointer
  intel_th: Fix a resource leak in an error handling path
  soundwire: bus_type: fix remove and shutdown support
  clk: qcom: camcc-sdm845: Fix topology around titan_top power domain
  clk: qcom: ipq8074: set BRANCH_HALT_DELAY flag for UBI clocks
  clk: qcom: ipq8074: fix NSS port frequency tables
  usb: host: xhci: use snprintf() in xhci_decode_trb()
  clk: qcom: clk-krait: unlock spin after mux completion
  driver core: fix potential deadlock in __driver_attach
  misc: rtsx: Fix an error handling path in rtsx_pci_probe()
  clk: mediatek: reset: Fix written reset bit offset
  usb: xhci: tegra: Fix error check
  usb: ohci-nxp: Fix refcount leak in ohci_hcd_nxp_probe
  usb: host: Fix refcount leak in ehci_hcd_ppc_of_probe
  fpga: altera-pr-ip: fix unsigned comparison with less than zero
  mtd: st_spi_fsm: Add a clk_disable_unprepare() in .probe()'s error path
  mtd: partitions: Fix refcount leak in parse_redboot_of
  mtd: sm_ftl: Fix deadlock caused by cancel_work_sync in sm_release
  HID: cp2112: prevent a buffer overflow in cp2112_xfer()
  mtd: rawnand: meson: Fix a potential double free issue
  mtd: maps: Fix refcount leak in ap_flash_init
  mtd: maps: Fix refcount leak in of_flash_probe_versatile
  clk: renesas: r9a06g032: Fix UART clkgrp bitsel
  dccp: put dccp_qpolicy_full() and dccp_qpolicy_push() in the same lock
  net: rose: fix netdev reference changes
  netdevsim: Avoid allocation warnings triggered from user space
  iavf: Fix max_rate limiting
  crypto: inside-secure - Add missing MODULE_DEVICE_TABLE for of
  net/mlx5e: Fix the value of MLX5E_MAX_RQ_NUM_MTTS
  wifi: libertas: Fix possible refcount leak in if_usb_probe()
  wifi: iwlwifi: mvm: fix double list_add at iwl_mvm_mac_wake_tx_queue
  wifi: wil6210: debugfs: fix uninitialized variable use in `wil_write_file_wmi()`
  i2c: mux-gpmux: Add of_node_put() when breaking out of loop
  i2c: cadence: Support PEC for SMBus block read
  Bluetooth: hci_intel: Add check for platform_driver_register
  can: pch_can: pch_can_error(): initialize errc before using it
  can: error: specify the values of data[5..7] of CAN error frames
  can: usb_8dev: do not report txerr and rxerr during bus-off
  can: kvaser_usb_leaf: do not report txerr and rxerr during bus-off
  can: kvaser_usb_hydra: do not report txerr and rxerr during bus-off
  can: sun4i_can: do not report txerr and rxerr during bus-off
  can: hi311x: do not report txerr and rxerr during bus-off
  can: sja1000: do not report txerr and rxerr during bus-off
  can: rcar_can: do not report txerr and rxerr during bus-off
  can: pch_can: do not report txerr and rxerr during bus-off
  selftests/bpf: fix a test for snprintf() overflow
  wifi: p54: add missing parentheses in p54_flush()
  wifi: p54: Fix an error handling path in p54spi_probe()
  wifi: wil6210: debugfs: fix info leak in wil_write_file_wmi()
  fs: check FMODE_LSEEK to control internal pipe splicing
  selftests: timers: clocksource-switch: fix passing errors from child
  selftests: timers: valid-adjtimex: build fix for newer toolchains
  libbpf: Fix the name of a reused map
  tcp: make retransmitted SKB fit into the send window
  drm/exynos/exynos7_drm_decon: free resources when clk_set_parent() failed.
  mediatek: mt76: mac80211: Fix missing of_node_put() in mt76_led_init()
  media: platform: mtk-mdp: Fix mdp_ipi_comm structure alignment
  crypto: hisilicon - Kunpeng916 crypto driver don't sleep when in softirq
  drm/msm/mdp5: Fix global state lock backoff
  drm: bridge: sii8620: fix possible off-by-one
  drm/mediatek: dpi: Only enable dpi after the bridge is enabled
  drm/mediatek: dpi: Remove output format of YUV
  drm/rockchip: Fix an error handling path rockchip_dp_probe()
  drm/rockchip: vop: Don't crash for invalid duplicate_state()
  crypto: arm64/gcm - Select AEAD for GHASH_ARM64_CE
  drm/vc4: dsi: Correct DSI divider calculations
  drm/vc4: plane: Fix margin calculations for the right/bottom edges
  drm/vc4: plane: Remove subpixel positioning check
  media: hdpvr: fix error value returns in hdpvr_read
  drm/mcde: Fix refcount leak in mcde_dsi_bind
  drm: bridge: adv7511: Add check for mipi_dsi_driver_register
  wifi: iwlegacy: 4965: fix potential off-by-one overflow in il4965_rs_fill_link_cmd()
  ath9k: fix use-after-free in ath9k_hif_usb_rx_cb
  media: tw686x: Register the irq at the end of probe
  i2c: Fix a potential use after free
  drm: adv7511: override i2c address of cec before accessing it
  drm/mediatek: Add pull-down MIPI operation in mtk_dsi_poweroff function
  drm/radeon: fix potential buffer overflow in ni_set_mc_special_registers()
  drm/mipi-dbi: align max_chunk to 2 in spi_transfer
  wifi: rtlwifi: fix error codes in rtl_debugfs_set_write_h2c()
  ath10k: do not enforce interrupt trigger type
  dm: return early from dm_pr_call() if DM device is suspended
  thermal/tools/tmon: Include pthread and time headers in tmon.h
  nohz/full, sched/rt: Fix missed tick-reenabling bug in dequeue_task_rt()
  regulator: of: Fix refcount leak bug in of_get_regulation_constraints()
  blk-mq: don't create hctx debugfs dir until q->debugfs_dir is created
  erofs: avoid consecutive detection for Highmem memory
  arm64: dts: mt7622: fix BPI-R64 WPS button
  bus: hisi_lpc: fix missing platform_device_put() in hisi_lpc_acpi_probe()
  ARM: dts: qcom: pm8841: add required thermal-sensor-cells
  soc: qcom: aoss: Fix refcount leak in qmp_cooling_devices_register
  cpufreq: zynq: Fix refcount leak in zynq_get_revision
  ARM: OMAP2+: Fix refcount leak in omap3xxx_prm_late_init
  ARM: OMAP2+: Fix refcount leak in omapdss_init_of
  ARM: dts: qcom: mdm9615: add missing PMIC GPIO reg
  soc: fsl: guts: machine variable might be unset
  ARM: dts: ast2600-evb: fix board compatible
  ARM: dts: ast2500-evb: fix board compatible
  x86/pmem: Fix platform-device leak in error path
  ARM: bcm: Fix refcount leak in bcm_kona_smc_init
  meson-mx-socinfo: Fix refcount leak in meson_mx_socinfo_init
  ARM: findbit: fix overflowing offset
  spi: spi-rspi: Fix PIO fallback on RZ platforms
  selinux: Add boundary check in put_entry()
  PM: hibernate: defer device probing when resuming from hibernation
  ARM: shmobile: rcar-gen2: Increase refcount for new reference
  arm64: dts: allwinner: a64: orangepi-win: Fix LED node name
  arm64: dts: qcom: ipq8074: fix NAND node name
  ACPI: LPSS: Fix missing check in register_device_clock()
  ACPI: PM: save NVS memory for Lenovo G40-45
  ACPI: EC: Remove duplicate ThinkPad X1 Carbon 6th entry from DMI quirks
  ARM: OMAP2+: display: Fix refcount leak bug
  spi: synquacer: Add missing clk_disable_unprepare()
  ARM: dts: imx6ul: fix qspi node compatible
  ARM: dts: imx6ul: fix lcdif node compatible
  ARM: dts: imx6ul: fix csi node compatible
  ARM: dts: imx6ul: change operating-points to uint32-matrix
  ARM: dts: imx6ul: add missing properties for sram
  wait: Fix __wait_event_hrtimeout for RT/DL tasks
  genirq: Don't return error on missing optional irq_request_resources()
  ext2: Add more validity checks for inode counts
  arm64: fix oops in concurrently setting insn_emulation sysctls
  arm64: Do not forget syscall when starting a new thread.
  x86: Handle idle=nomwait cmdline properly for x86_idle
  epoll: autoremove wakers even more aggressively
  netfilter: nf_tables: fix null deref due to zeroed list head
  netfilter: nf_tables: do not allow RULE_ID to refer to another chain
  netfilter: nf_tables: do not allow SET_ID to refer to another table
  arm64: dts: uniphier: Fix USB interrupts for PXs3 SoC
  ARM: dts: uniphier: Fix USB interrupts for PXs2 SoC
  USB: HCD: Fix URB giveback issue in tasklet function
  coresight: Clear the connection field properly
  MIPS: cpuinfo: Fix a warning for CONFIG_CPUMASK_OFFSTACK
  powerpc/powernv: Avoid crashing if rng is NULL
  powerpc/ptdump: Fix display of RW pages on FSL_BOOK3E
  powerpc/fsl-pci: Fix Class Code of PCIe Root Port
  PCI: Add defines for normal and subtractive PCI bridges
  ia64, processor: fix -Wincompatible-pointer-types in ia64_get_irr()
  md-raid10: fix KASAN warning
  serial: mvebu-uart: uart2 error bits clearing
  fuse: limit nsec
  iio: light: isl29028: Fix the warning in isl29028_remove()
  drm/amdgpu: Check BO's requested pinning domains against its preferred_domains
  drm/nouveau: fix another off-by-one in nvbios_addr
  drm/gem: Properly annotate WW context on drm_gem_lock_reservations() error
  parisc: io_pgetevents_time64() needs compat syscall in 32-bit compat mode
  parisc: Fix device names in /proc/iomem
  ovl: drop WARN_ON() dentry is NULL in ovl_encode_fh()
  usbnet: Fix linkwatch use-after-free on disconnect
  fbcon: Fix boundary checks for fbcon=vc:n1-n2 parameters
  thermal: sysfs: Fix cooling_device_stats_setup() error code path
  fs: Add missing umask strip in vfs_tmpfile
  vfs: Check the truncate maximum size in inode_newsize_ok()
  tty: vt: initialize unicode screen buffer
  ALSA: hda/realtek: Add quirk for another Asus K42JZ model
  ALSA: hda/cirrus - support for iMac 12,1 model
  ALSA: hda/conexant: Add quirk for LENOVO 20149 Notebook model
  mm/mremap: hold the rmap lock in write mode when moving page table entries.
  KVM: x86: Set error code to segment selector on LLDT/LTR non-canonical #GP
  KVM: x86: Mark TSS busy during LTR emulation _after_ all fault checks
  KVM: nVMX: Let userspace set nVMX MSR to any _host_ supported value
  KVM: SVM: Don't BUG if userspace injects an interrupt with GIF=0
  KVM: nVMX: Snapshot pre-VM-Enter DEBUGCTL for !nested_run_pending case
  KVM: nVMX: Snapshot pre-VM-Enter BNDCFGS for !nested_run_pending case
  HID: wacom: Don't register pad_input for touch switch
  HID: wacom: Only report rotation for art pen
  add barriers to buffer_uptodate and set_buffer_uptodate
  wifi: mac80211_hwsim: use 32-bit skb cookie
  wifi: mac80211_hwsim: add back erroneously removed cast
  wifi: mac80211_hwsim: fix race condition in pending packet
  igc: Remove _I_PHY_ID checking
  ALSA: bcd2000: Fix a UAF bug on the error path of probing
  scsi: Revert "scsi: qla2xxx: Fix disk failure to rediscover"
  x86: link vdso and boot with -z noexecstack --no-warn-rwx-segments
  Makefile: link with -z noexecstack --no-warn-rwx-segments

 Conflicts:
	Documentation/devicetree/bindings
	Documentation/devicetree/bindings/arm/qcom.yaml
	Documentation/devicetree/bindings/dma/moxa,moxart-dma.txt
	drivers/mmc/core/sd.c
	drivers/net/usb/ax88179_178a.c
	drivers/rpmsg/qcom_glink_native.c
	drivers/usb/dwc3/core.c
	drivers/usb/typec/ucsi/ucsi.c
	net/core/dev.c
	net/wireless/scan.c

Change-Id: Id1996866ef5d9b7c097c39a5bdb00db413763104
Signed-off-by: Srinivasarao Pathipati <quic_c_spathi@quicinc.com>
2023-01-27 00:57:29 +05:30

2988 lines
78 KiB
C

// SPDX-License-Identifier: GPL-2.0
/*
* Memory Migration functionality - linux/mm/migrate.c
*
* Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter
*
* Page migration was first developed in the context of the memory hotplug
* project. The main authors of the migration code are:
*
* IWAMOTO Toshihiro <iwamoto@valinux.co.jp>
* Hirokazu Takahashi <taka@valinux.co.jp>
* Dave Hansen <haveblue@us.ibm.com>
* Christoph Lameter
*/
#include <linux/migrate.h>
#include <linux/export.h>
#include <linux/swap.h>
#include <linux/swapops.h>
#include <linux/pagemap.h>
#include <linux/buffer_head.h>
#include <linux/mm_inline.h>
#include <linux/nsproxy.h>
#include <linux/pagevec.h>
#include <linux/ksm.h>
#include <linux/rmap.h>
#include <linux/topology.h>
#include <linux/cpu.h>
#include <linux/cpuset.h>
#include <linux/writeback.h>
#include <linux/mempolicy.h>
#include <linux/vmalloc.h>
#include <linux/security.h>
#include <linux/backing-dev.h>
#include <linux/compaction.h>
#include <linux/syscalls.h>
#include <linux/compat.h>
#include <linux/hugetlb.h>
#include <linux/hugetlb_cgroup.h>
#include <linux/gfp.h>
#include <linux/pagewalk.h>
#include <linux/pfn_t.h>
#include <linux/memremap.h>
#include <linux/userfaultfd_k.h>
#include <linux/balloon_compaction.h>
#include <linux/mmu_notifier.h>
#include <linux/page_idle.h>
#include <linux/page_owner.h>
#include <linux/sched/mm.h>
#include <linux/ptrace.h>
#include <asm/tlbflush.h>
#define CREATE_TRACE_POINTS
#include <trace/events/migrate.h>
#include "internal.h"
/*
* migrate_prep() needs to be called before we start compiling a list of pages
* to be migrated using isolate_lru_page(). If scheduling work on other CPUs is
* undesirable, use migrate_prep_local()
*/
int migrate_prep(void)
{
/*
* Clear the LRU lists so pages can be isolated.
* Note that pages may be moved off the LRU after we have
* drained them. Those pages will fail to migrate like other
* pages that may be busy.
*/
lru_add_drain_all();
return 0;
}
/* Do the necessary work of migrate_prep but not if it involves other CPUs */
int migrate_prep_local(void)
{
lru_add_drain();
return 0;
}
int isolate_movable_page(struct page *page, isolate_mode_t mode)
{
struct address_space *mapping;
/*
* Avoid burning cycles with pages that are yet under __free_pages(),
* or just got freed under us.
*
* In case we 'win' a race for a movable page being freed under us and
* raise its refcount preventing __free_pages() from doing its job
* the put_page() at the end of this block will take care of
* release this page, thus avoiding a nasty leakage.
*/
if (unlikely(!get_page_unless_zero(page)))
goto out;
/*
* Check PageMovable before holding a PG_lock because page's owner
* assumes anybody doesn't touch PG_lock of newly allocated page
* so unconditionally grabbing the lock ruins page's owner side.
*/
if (unlikely(!__PageMovable(page)))
goto out_putpage;
/*
* As movable pages are not isolated from LRU lists, concurrent
* compaction threads can race against page migration functions
* as well as race against the releasing a page.
*
* In order to avoid having an already isolated movable page
* being (wrongly) re-isolated while it is under migration,
* or to avoid attempting to isolate pages being released,
* lets be sure we have the page lock
* before proceeding with the movable page isolation steps.
*/
if (unlikely(!trylock_page(page)))
goto out_putpage;
if (!PageMovable(page) || PageIsolated(page))
goto out_no_isolated;
mapping = page_mapping(page);
VM_BUG_ON_PAGE(!mapping, page);
if (!mapping->a_ops->isolate_page(page, mode))
goto out_no_isolated;
/* Driver shouldn't use PG_isolated bit of page->flags */
WARN_ON_ONCE(PageIsolated(page));
__SetPageIsolated(page);
unlock_page(page);
return 0;
out_no_isolated:
unlock_page(page);
out_putpage:
put_page(page);
out:
return -EBUSY;
}
/* It should be called on page which is PG_movable */
void putback_movable_page(struct page *page)
{
struct address_space *mapping;
VM_BUG_ON_PAGE(!PageLocked(page), page);
VM_BUG_ON_PAGE(!PageMovable(page), page);
VM_BUG_ON_PAGE(!PageIsolated(page), page);
mapping = page_mapping(page);
mapping->a_ops->putback_page(page);
__ClearPageIsolated(page);
}
/*
* Put previously isolated pages back onto the appropriate lists
* from where they were once taken off for compaction/migration.
*
* This function shall be used whenever the isolated pageset has been
* built from lru, balloon, hugetlbfs page. See isolate_migratepages_range()
* and isolate_huge_page().
*/
void putback_movable_pages(struct list_head *l)
{
struct page *page;
struct page *page2;
list_for_each_entry_safe(page, page2, l, lru) {
if (unlikely(PageHuge(page))) {
putback_active_hugepage(page);
continue;
}
list_del(&page->lru);
/*
* We isolated non-lru movable page so here we can use
* __PageMovable because LRU page's mapping cannot have
* PAGE_MAPPING_MOVABLE.
*/
if (unlikely(__PageMovable(page))) {
VM_BUG_ON_PAGE(!PageIsolated(page), page);
lock_page(page);
if (PageMovable(page))
putback_movable_page(page);
else
__ClearPageIsolated(page);
unlock_page(page);
put_page(page);
} else {
mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON +
page_is_file_cache(page), -hpage_nr_pages(page));
putback_lru_page(page);
}
}
}
/*
* Restore a potential migration pte to a working pte entry
*/
static bool remove_migration_pte(struct page *page, struct vm_area_struct *vma,
unsigned long addr, void *old)
{
struct page_vma_mapped_walk pvmw = {
.page = old,
.vma = vma,
.address = addr,
.flags = PVMW_SYNC | PVMW_MIGRATION,
};
struct page *new;
pte_t pte;
swp_entry_t entry;
VM_BUG_ON_PAGE(PageTail(page), page);
while (page_vma_mapped_walk(&pvmw)) {
if (PageKsm(page))
new = page;
else
new = page - pvmw.page->index +
linear_page_index(vma, pvmw.address);
#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
/* PMD-mapped THP migration entry */
if (!pvmw.pte) {
VM_BUG_ON_PAGE(PageHuge(page) || !PageTransCompound(page), page);
remove_migration_pmd(&pvmw, new);
continue;
}
#endif
get_page(new);
pte = pte_mkold(mk_pte(new, READ_ONCE(vma->vm_page_prot)));
if (pte_swp_soft_dirty(*pvmw.pte))
pte = pte_mksoft_dirty(pte);
/*
* Recheck VMA as permissions can change since migration started
*/
entry = pte_to_swp_entry(*pvmw.pte);
if (is_write_migration_entry(entry))
pte = maybe_mkwrite(pte, vma->vm_flags);
if (unlikely(is_zone_device_page(new))) {
if (is_device_private_page(new)) {
entry = make_device_private_entry(new, pte_write(pte));
pte = swp_entry_to_pte(entry);
}
}
#ifdef CONFIG_HUGETLB_PAGE
if (PageHuge(new)) {
pte = pte_mkhuge(pte);
pte = arch_make_huge_pte(pte, vma, new, 0);
set_huge_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte);
if (PageAnon(new))
hugepage_add_anon_rmap(new, vma, pvmw.address);
else
page_dup_rmap(new, true);
} else
#endif
{
set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte);
if (PageAnon(new))
page_add_anon_rmap(new, vma, pvmw.address, false);
else
page_add_file_rmap(new, false);
}
if (vma->vm_flags & VM_LOCKED && !PageTransCompound(new))
mlock_vma_page(new);
if (PageTransHuge(page) && PageMlocked(page))
clear_page_mlock(page);
/* No need to invalidate - it was non-present before */
update_mmu_cache(vma, pvmw.address, pvmw.pte);
}
return true;
}
/*
* Get rid of all migration entries and replace them by
* references to the indicated page.
*/
void remove_migration_ptes(struct page *old, struct page *new, bool locked)
{
struct rmap_walk_control rwc = {
.rmap_one = remove_migration_pte,
.arg = old,
};
if (locked)
rmap_walk_locked(new, &rwc);
else
rmap_walk(new, &rwc);
}
/*
* Something used the pte of a page under migration. We need to
* get to the page and wait until migration is finished.
* When we return from this function the fault will be retried.
*/
void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep,
spinlock_t *ptl)
{
pte_t pte;
swp_entry_t entry;
struct page *page;
spin_lock(ptl);
pte = *ptep;
if (!is_swap_pte(pte))
goto out;
entry = pte_to_swp_entry(pte);
if (!is_migration_entry(entry))
goto out;
page = migration_entry_to_page(entry);
page = compound_head(page);
/*
* Once page cache replacement of page migration started, page_count
* is zero; but we must not call put_and_wait_on_page_locked() without
* a ref. Use get_page_unless_zero(), and just fault again if it fails.
*/
if (!get_page_unless_zero(page))
goto out;
pte_unmap_unlock(ptep, ptl);
put_and_wait_on_page_locked(page);
return;
out:
pte_unmap_unlock(ptep, ptl);
}
void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
unsigned long address)
{
spinlock_t *ptl = pte_lockptr(mm, pmd);
pte_t *ptep = pte_offset_map(pmd, address);
__migration_entry_wait(mm, ptep, ptl);
}
void migration_entry_wait_huge(struct vm_area_struct *vma,
struct mm_struct *mm, pte_t *pte)
{
spinlock_t *ptl = huge_pte_lockptr(hstate_vma(vma), mm, pte);
__migration_entry_wait(mm, pte, ptl);
}
#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd)
{
spinlock_t *ptl;
struct page *page;
ptl = pmd_lock(mm, pmd);
if (!is_pmd_migration_entry(*pmd))
goto unlock;
page = migration_entry_to_page(pmd_to_swp_entry(*pmd));
if (!get_page_unless_zero(page))
goto unlock;
spin_unlock(ptl);
put_and_wait_on_page_locked(page);
return;
unlock:
spin_unlock(ptl);
}
#endif
static int expected_page_refs(struct address_space *mapping, struct page *page)
{
int expected_count = 1;
/*
* Device public or private pages have an extra refcount as they are
* ZONE_DEVICE pages.
*/
expected_count += is_device_private_page(page);
if (mapping)
expected_count += hpage_nr_pages(page) + page_has_private(page);
return expected_count;
}
/*
* Replace the page in the mapping.
*
* The number of remaining references must be:
* 1 for anonymous pages without a mapping
* 2 for pages with a mapping
* 3 for pages with a mapping and PagePrivate/PagePrivate2 set.
*/
int migrate_page_move_mapping(struct address_space *mapping,
struct page *newpage, struct page *page, int extra_count)
{
XA_STATE(xas, &mapping->i_pages, page_index(page));
struct zone *oldzone, *newzone;
int dirty;
int expected_count = expected_page_refs(mapping, page) + extra_count;
if (!mapping) {
/* Anonymous page without mapping */
if (page_count(page) != expected_count)
return -EAGAIN;
/* No turning back from here */
newpage->index = page->index;
newpage->mapping = page->mapping;
if (PageSwapBacked(page))
__SetPageSwapBacked(newpage);
return MIGRATEPAGE_SUCCESS;
}
oldzone = page_zone(page);
newzone = page_zone(newpage);
xas_lock_irq(&xas);
if (page_count(page) != expected_count || xas_load(&xas) != page) {
xas_unlock_irq(&xas);
return -EAGAIN;
}
if (!page_ref_freeze(page, expected_count)) {
xas_unlock_irq(&xas);
return -EAGAIN;
}
/*
* Now we know that no one else is looking at the page:
* no turning back from here.
*/
newpage->index = page->index;
newpage->mapping = page->mapping;
page_ref_add(newpage, hpage_nr_pages(page)); /* add cache reference */
if (PageSwapBacked(page)) {
__SetPageSwapBacked(newpage);
if (PageSwapCache(page)) {
SetPageSwapCache(newpage);
set_page_private(newpage, page_private(page));
}
} else {
VM_BUG_ON_PAGE(PageSwapCache(page), page);
}
/* Move dirty while page refs frozen and newpage not yet exposed */
dirty = PageDirty(page);
if (dirty) {
ClearPageDirty(page);
SetPageDirty(newpage);
}
xas_store(&xas, newpage);
if (PageTransHuge(page)) {
int i;
for (i = 1; i < HPAGE_PMD_NR; i++) {
xas_next(&xas);
xas_store(&xas, newpage);
}
}
/*
* Drop cache reference from old page by unfreezing
* to one less reference.
* We know this isn't the last reference.
*/
page_ref_unfreeze(page, expected_count - hpage_nr_pages(page));
xas_unlock(&xas);
/* Leave irq disabled to prevent preemption while updating stats */
/*
* If moved to a different zone then also account
* the page for that zone. Other VM counters will be
* taken care of when we establish references to the
* new page and drop references to the old page.
*
* Note that anonymous pages are accounted for
* via NR_FILE_PAGES and NR_ANON_MAPPED if they
* are mapped to swap space.
*/
if (newzone != oldzone) {
__dec_node_state(oldzone->zone_pgdat, NR_FILE_PAGES);
__inc_node_state(newzone->zone_pgdat, NR_FILE_PAGES);
if (PageSwapBacked(page) && !PageSwapCache(page)) {
__dec_node_state(oldzone->zone_pgdat, NR_SHMEM);
__inc_node_state(newzone->zone_pgdat, NR_SHMEM);
}
if (dirty && mapping_cap_account_dirty(mapping)) {
__dec_node_state(oldzone->zone_pgdat, NR_FILE_DIRTY);
__dec_zone_state(oldzone, NR_ZONE_WRITE_PENDING);
__inc_node_state(newzone->zone_pgdat, NR_FILE_DIRTY);
__inc_zone_state(newzone, NR_ZONE_WRITE_PENDING);
}
}
local_irq_enable();
return MIGRATEPAGE_SUCCESS;
}
EXPORT_SYMBOL(migrate_page_move_mapping);
/*
* The expected number of remaining references is the same as that
* of migrate_page_move_mapping().
*/
int migrate_huge_page_move_mapping(struct address_space *mapping,
struct page *newpage, struct page *page)
{
XA_STATE(xas, &mapping->i_pages, page_index(page));
int expected_count;
xas_lock_irq(&xas);
expected_count = 2 + page_has_private(page);
if (page_count(page) != expected_count || xas_load(&xas) != page) {
xas_unlock_irq(&xas);
return -EAGAIN;
}
if (!page_ref_freeze(page, expected_count)) {
xas_unlock_irq(&xas);
return -EAGAIN;
}
newpage->index = page->index;
newpage->mapping = page->mapping;
get_page(newpage);
xas_store(&xas, newpage);
page_ref_unfreeze(page, expected_count - 1);
xas_unlock_irq(&xas);
return MIGRATEPAGE_SUCCESS;
}
/*
* Gigantic pages are so large that we do not guarantee that page++ pointer
* arithmetic will work across the entire page. We need something more
* specialized.
*/
static void __copy_gigantic_page(struct page *dst, struct page *src,
int nr_pages)
{
int i;
struct page *dst_base = dst;
struct page *src_base = src;
for (i = 0; i < nr_pages; ) {
cond_resched();
copy_highpage(dst, src);
i++;
dst = mem_map_next(dst, dst_base, i);
src = mem_map_next(src, src_base, i);
}
}
static void copy_huge_page(struct page *dst, struct page *src)
{
int i;
int nr_pages;
if (PageHuge(src)) {
/* hugetlbfs page */
struct hstate *h = page_hstate(src);
nr_pages = pages_per_huge_page(h);
if (unlikely(nr_pages > MAX_ORDER_NR_PAGES)) {
__copy_gigantic_page(dst, src, nr_pages);
return;
}
} else {
/* thp page */
BUG_ON(!PageTransHuge(src));
nr_pages = hpage_nr_pages(src);
}
for (i = 0; i < nr_pages; i++) {
cond_resched();
copy_highpage(dst + i, src + i);
}
}
/*
* Copy the page to its new location
*/
void migrate_page_states(struct page *newpage, struct page *page)
{
int cpupid;
if (PageError(page))
SetPageError(newpage);
if (PageReferenced(page))
SetPageReferenced(newpage);
if (PageUptodate(page))
SetPageUptodate(newpage);
if (TestClearPageActive(page)) {
VM_BUG_ON_PAGE(PageUnevictable(page), page);
SetPageActive(newpage);
} else if (TestClearPageUnevictable(page))
SetPageUnevictable(newpage);
if (PageWorkingset(page))
SetPageWorkingset(newpage);
if (PageChecked(page))
SetPageChecked(newpage);
if (PageMappedToDisk(page))
SetPageMappedToDisk(newpage);
/* Move dirty on pages not done by migrate_page_move_mapping() */
if (PageDirty(page))
SetPageDirty(newpage);
if (page_is_young(page))
set_page_young(newpage);
if (page_is_idle(page))
set_page_idle(newpage);
/*
* Copy NUMA information to the new page, to prevent over-eager
* future migrations of this same page.
*/
cpupid = page_cpupid_xchg_last(page, -1);
page_cpupid_xchg_last(newpage, cpupid);
ksm_migrate_page(newpage, page);
/*
* Please do not reorder this without considering how mm/ksm.c's
* get_ksm_page() depends upon ksm_migrate_page() and PageSwapCache().
*/
if (PageSwapCache(page))
ClearPageSwapCache(page);
ClearPagePrivate(page);
set_page_private(page, 0);
/*
* If any waiters have accumulated on the new page then
* wake them up.
*/
if (PageWriteback(newpage))
end_page_writeback(newpage);
copy_page_owner(page, newpage);
mem_cgroup_migrate(page, newpage);
}
EXPORT_SYMBOL(migrate_page_states);
void migrate_page_copy(struct page *newpage, struct page *page)
{
if (PageHuge(page) || PageTransHuge(page))
copy_huge_page(newpage, page);
else
copy_highpage(newpage, page);
migrate_page_states(newpage, page);
}
EXPORT_SYMBOL(migrate_page_copy);
/************************************************************
* Migration functions
***********************************************************/
/*
* Common logic to directly migrate a single LRU page suitable for
* pages that do not use PagePrivate/PagePrivate2.
*
* Pages are locked upon entry and exit.
*/
int migrate_page(struct address_space *mapping,
struct page *newpage, struct page *page,
enum migrate_mode mode)
{
int rc;
BUG_ON(PageWriteback(page)); /* Writeback must be complete */
rc = migrate_page_move_mapping(mapping, newpage, page, 0);
if (rc != MIGRATEPAGE_SUCCESS)
return rc;
if (mode != MIGRATE_SYNC_NO_COPY)
migrate_page_copy(newpage, page);
else
migrate_page_states(newpage, page);
return MIGRATEPAGE_SUCCESS;
}
EXPORT_SYMBOL(migrate_page);
#ifdef CONFIG_BLOCK
/* Returns true if all buffers are successfully locked */
static bool buffer_migrate_lock_buffers(struct buffer_head *head,
enum migrate_mode mode)
{
struct buffer_head *bh = head;
/* Simple case, sync compaction */
if (mode != MIGRATE_ASYNC) {
do {
lock_buffer(bh);
bh = bh->b_this_page;
} while (bh != head);
return true;
}
/* async case, we cannot block on lock_buffer so use trylock_buffer */
do {
if (!trylock_buffer(bh)) {
/*
* We failed to lock the buffer and cannot stall in
* async migration. Release the taken locks
*/
struct buffer_head *failed_bh = bh;
bh = head;
while (bh != failed_bh) {
unlock_buffer(bh);
bh = bh->b_this_page;
}
return false;
}
bh = bh->b_this_page;
} while (bh != head);
return true;
}
static int __buffer_migrate_page(struct address_space *mapping,
struct page *newpage, struct page *page, enum migrate_mode mode,
bool check_refs)
{
struct buffer_head *bh, *head;
int rc;
int expected_count;
if (!page_has_buffers(page))
return migrate_page(mapping, newpage, page, mode);
/* Check whether page does not have extra refs before we do more work */
expected_count = expected_page_refs(mapping, page);
if (page_count(page) != expected_count)
return -EAGAIN;
head = page_buffers(page);
if (!buffer_migrate_lock_buffers(head, mode))
return -EAGAIN;
if (check_refs) {
bool busy;
bool invalidated = false;
recheck_buffers:
busy = false;
spin_lock(&mapping->private_lock);
bh = head;
do {
if (atomic_read(&bh->b_count)) {
busy = true;
break;
}
bh = bh->b_this_page;
} while (bh != head);
if (busy) {
if (invalidated) {
rc = -EAGAIN;
goto unlock_buffers;
}
spin_unlock(&mapping->private_lock);
invalidate_bh_lrus();
invalidated = true;
goto recheck_buffers;
}
}
rc = migrate_page_move_mapping(mapping, newpage, page, 0);
if (rc != MIGRATEPAGE_SUCCESS)
goto unlock_buffers;
ClearPagePrivate(page);
set_page_private(newpage, page_private(page));
set_page_private(page, 0);
put_page(page);
get_page(newpage);
bh = head;
do {
set_bh_page(bh, newpage, bh_offset(bh));
bh = bh->b_this_page;
} while (bh != head);
SetPagePrivate(newpage);
if (mode != MIGRATE_SYNC_NO_COPY)
migrate_page_copy(newpage, page);
else
migrate_page_states(newpage, page);
rc = MIGRATEPAGE_SUCCESS;
unlock_buffers:
if (check_refs)
spin_unlock(&mapping->private_lock);
bh = head;
do {
unlock_buffer(bh);
bh = bh->b_this_page;
} while (bh != head);
return rc;
}
/*
* Migration function for pages with buffers. This function can only be used
* if the underlying filesystem guarantees that no other references to "page"
* exist. For example attached buffer heads are accessed only under page lock.
*/
int buffer_migrate_page(struct address_space *mapping,
struct page *newpage, struct page *page, enum migrate_mode mode)
{
return __buffer_migrate_page(mapping, newpage, page, mode, false);
}
EXPORT_SYMBOL(buffer_migrate_page);
/*
* Same as above except that this variant is more careful and checks that there
* are also no buffer head references. This function is the right one for
* mappings where buffer heads are directly looked up and referenced (such as
* block device mappings).
*/
int buffer_migrate_page_norefs(struct address_space *mapping,
struct page *newpage, struct page *page, enum migrate_mode mode)
{
return __buffer_migrate_page(mapping, newpage, page, mode, true);
}
#endif
/*
* Writeback a page to clean the dirty state
*/
static int writeout(struct address_space *mapping, struct page *page)
{
struct writeback_control wbc = {
.sync_mode = WB_SYNC_NONE,
.nr_to_write = 1,
.range_start = 0,
.range_end = LLONG_MAX,
.for_reclaim = 1
};
int rc;
if (!mapping->a_ops->writepage)
/* No write method for the address space */
return -EINVAL;
if (!clear_page_dirty_for_io(page))
/* Someone else already triggered a write */
return -EAGAIN;
/*
* A dirty page may imply that the underlying filesystem has
* the page on some queue. So the page must be clean for
* migration. Writeout may mean we loose the lock and the
* page state is no longer what we checked for earlier.
* At this point we know that the migration attempt cannot
* be successful.
*/
remove_migration_ptes(page, page, false);
rc = mapping->a_ops->writepage(page, &wbc);
if (rc != AOP_WRITEPAGE_ACTIVATE)
/* unlocked. Relock */
lock_page(page);
return (rc < 0) ? -EIO : -EAGAIN;
}
/*
* Default handling if a filesystem does not provide a migration function.
*/
static int fallback_migrate_page(struct address_space *mapping,
struct page *newpage, struct page *page, enum migrate_mode mode)
{
if (PageDirty(page)) {
/* Only writeback pages in full synchronous migration */
switch (mode) {
case MIGRATE_SYNC:
case MIGRATE_SYNC_NO_COPY:
break;
default:
return -EBUSY;
}
return writeout(mapping, page);
}
/*
* Buffers may be managed in a filesystem specific way.
* We must have no buffers or drop them.
*/
if (page_has_private(page) &&
!try_to_release_page(page, GFP_KERNEL))
return mode == MIGRATE_SYNC ? -EAGAIN : -EBUSY;
return migrate_page(mapping, newpage, page, mode);
}
/*
* Move a page to a newly allocated page
* The page is locked and all ptes have been successfully removed.
*
* The new page will have replaced the old page if this function
* is successful.
*
* Return value:
* < 0 - error code
* MIGRATEPAGE_SUCCESS - success
*/
static int move_to_new_page(struct page *newpage, struct page *page,
enum migrate_mode mode)
{
struct address_space *mapping;
int rc = -EAGAIN;
bool is_lru = !__PageMovable(page);
VM_BUG_ON_PAGE(!PageLocked(page), page);
VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
mapping = page_mapping(page);
if (likely(is_lru)) {
if (!mapping)
rc = migrate_page(mapping, newpage, page, mode);
else if (mapping->a_ops->migratepage)
/*
* Most pages have a mapping and most filesystems
* provide a migratepage callback. Anonymous pages
* are part of swap space which also has its own
* migratepage callback. This is the most common path
* for page migration.
*/
rc = mapping->a_ops->migratepage(mapping, newpage,
page, mode);
else
rc = fallback_migrate_page(mapping, newpage,
page, mode);
} else {
/*
* In case of non-lru page, it could be released after
* isolation step. In that case, we shouldn't try migration.
*/
VM_BUG_ON_PAGE(!PageIsolated(page), page);
if (!PageMovable(page)) {
rc = MIGRATEPAGE_SUCCESS;
__ClearPageIsolated(page);
goto out;
}
rc = mapping->a_ops->migratepage(mapping, newpage,
page, mode);
WARN_ON_ONCE(rc == MIGRATEPAGE_SUCCESS &&
!PageIsolated(page));
}
/*
* When successful, old pagecache page->mapping must be cleared before
* page is freed; but stats require that PageAnon be left as PageAnon.
*/
if (rc == MIGRATEPAGE_SUCCESS) {
if (__PageMovable(page)) {
VM_BUG_ON_PAGE(!PageIsolated(page), page);
/*
* We clear PG_movable under page_lock so any compactor
* cannot try to migrate this page.
*/
__ClearPageIsolated(page);
}
/*
* Anonymous and movable page->mapping will be cleard by
* free_pages_prepare so don't reset it here for keeping
* the type to work PageAnon, for example.
*/
if (!PageMappingFlags(page))
page->mapping = NULL;
if (likely(!is_zone_device_page(newpage))) {
int i, nr = compound_nr(newpage);
for (i = 0; i < nr; i++)
flush_dcache_page(newpage + i);
}
}
out:
return rc;
}
static int __unmap_and_move(struct page *page, struct page *newpage,
int force, enum migrate_mode mode)
{
int rc = -EAGAIN;
int page_was_mapped = 0;
struct anon_vma *anon_vma = NULL;
bool is_lru = !__PageMovable(page);
if (!trylock_page(page)) {
if (!force || mode == MIGRATE_ASYNC)
goto out;
/*
* It's not safe for direct compaction to call lock_page.
* For example, during page readahead pages are added locked
* to the LRU. Later, when the IO completes the pages are
* marked uptodate and unlocked. However, the queueing
* could be merging multiple pages for one bio (e.g.
* mpage_readpages). If an allocation happens for the
* second or third page, the process can end up locking
* the same page twice and deadlocking. Rather than
* trying to be clever about what pages can be locked,
* avoid the use of lock_page for direct compaction
* altogether.
*/
if (current->flags & PF_MEMALLOC)
goto out;
lock_page(page);
}
if (PageWriteback(page)) {
/*
* Only in the case of a full synchronous migration is it
* necessary to wait for PageWriteback. In the async case,
* the retry loop is too short and in the sync-light case,
* the overhead of stalling is too much
*/
switch (mode) {
case MIGRATE_SYNC:
case MIGRATE_SYNC_NO_COPY:
break;
default:
rc = -EBUSY;
goto out_unlock;
}
if (!force)
goto out_unlock;
wait_on_page_writeback(page);
}
/*
* By try_to_unmap(), page->mapcount goes down to 0 here. In this case,
* we cannot notice that anon_vma is freed while we migrates a page.
* This get_anon_vma() delays freeing anon_vma pointer until the end
* of migration. File cache pages are no problem because of page_lock()
* File Caches may use write_page() or lock_page() in migration, then,
* just care Anon page here.
*
* Only page_get_anon_vma() understands the subtleties of
* getting a hold on an anon_vma from outside one of its mms.
* But if we cannot get anon_vma, then we won't need it anyway,
* because that implies that the anon page is no longer mapped
* (and cannot be remapped so long as we hold the page lock).
*/
if (PageAnon(page) && !PageKsm(page))
anon_vma = page_get_anon_vma(page);
/*
* Block others from accessing the new page when we get around to
* establishing additional references. We are usually the only one
* holding a reference to newpage at this point. We used to have a BUG
* here if trylock_page(newpage) fails, but would like to allow for
* cases where there might be a race with the previous use of newpage.
* This is much like races on refcount of oldpage: just don't BUG().
*/
if (unlikely(!trylock_page(newpage)))
goto out_unlock;
if (unlikely(!is_lru)) {
rc = move_to_new_page(newpage, page, mode);
goto out_unlock_both;
}
/*
* Corner case handling:
* 1. When a new swap-cache page is read into, it is added to the LRU
* and treated as swapcache but it has no rmap yet.
* Calling try_to_unmap() against a page->mapping==NULL page will
* trigger a BUG. So handle it here.
* 2. An orphaned page (see truncate_complete_page) might have
* fs-private metadata. The page can be picked up due to memory
* offlining. Everywhere else except page reclaim, the page is
* invisible to the vm, so the page can not be migrated. So try to
* free the metadata, so the page can be freed.
*/
if (!page->mapping) {
VM_BUG_ON_PAGE(PageAnon(page), page);
if (page_has_private(page)) {
try_to_free_buffers(page);
goto out_unlock_both;
}
} else if (page_mapped(page)) {
/* Establish migration ptes */
VM_BUG_ON_PAGE(PageAnon(page) && !PageKsm(page) && !anon_vma,
page);
try_to_unmap(page,
TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS, NULL);
page_was_mapped = 1;
}
if (!page_mapped(page))
rc = move_to_new_page(newpage, page, mode);
if (page_was_mapped)
remove_migration_ptes(page,
rc == MIGRATEPAGE_SUCCESS ? newpage : page, false);
out_unlock_both:
unlock_page(newpage);
out_unlock:
/* Drop an anon_vma reference if we took one */
if (anon_vma)
put_anon_vma(anon_vma);
unlock_page(page);
out:
/*
* If migration is successful, decrease refcount of the newpage
* which will not free the page because new page owner increased
* refcounter. As well, if it is LRU page, add the page to LRU
* list in here. Use the old state of the isolated source page to
* determine if we migrated a LRU page. newpage was already unlocked
* and possibly modified by its owner - don't rely on the page
* state.
*/
if (rc == MIGRATEPAGE_SUCCESS) {
if (unlikely(!is_lru))
put_page(newpage);
else
putback_lru_page(newpage);
}
return rc;
}
/*
* gcc 4.7 and 4.8 on arm get an ICEs when inlining unmap_and_move(). Work
* around it.
*/
#if defined(CONFIG_ARM) && \
defined(GCC_VERSION) && GCC_VERSION < 40900 && GCC_VERSION >= 40700
#define ICE_noinline noinline
#else
#define ICE_noinline
#endif
/*
* Obtain the lock on page, remove all ptes and migrate the page
* to the newly allocated page in newpage.
*/
static ICE_noinline int unmap_and_move(new_page_t get_new_page,
free_page_t put_new_page,
unsigned long private, struct page *page,
int force, enum migrate_mode mode,
enum migrate_reason reason)
{
int rc = MIGRATEPAGE_SUCCESS;
struct page *newpage;
if (!thp_migration_supported() && PageTransHuge(page))
return -ENOMEM;
newpage = get_new_page(page, private);
if (!newpage)
return -ENOMEM;
if (page_count(page) == 1) {
/* page was freed from under us. So we are done. */
ClearPageActive(page);
ClearPageUnevictable(page);
if (unlikely(__PageMovable(page))) {
lock_page(page);
if (!PageMovable(page))
__ClearPageIsolated(page);
unlock_page(page);
}
if (put_new_page)
put_new_page(newpage, private);
else
put_page(newpage);
goto out;
}
rc = __unmap_and_move(page, newpage, force, mode);
if (rc == MIGRATEPAGE_SUCCESS)
set_page_owner_migrate_reason(newpage, reason);
out:
if (rc != -EAGAIN) {
/*
* A page that has been migrated has all references
* removed and will be freed. A page that has not been
* migrated will have kepts its references and be
* restored.
*/
list_del(&page->lru);
/*
* Compaction can migrate also non-LRU pages which are
* not accounted to NR_ISOLATED_*. They can be recognized
* as __PageMovable
*/
if (likely(!__PageMovable(page)))
mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON +
page_is_file_cache(page), -hpage_nr_pages(page));
}
/*
* If migration is successful, releases reference grabbed during
* isolation. Otherwise, restore the page to right list unless
* we want to retry.
*/
if (rc == MIGRATEPAGE_SUCCESS) {
put_page(page);
if (reason == MR_MEMORY_FAILURE) {
/*
* Set PG_HWPoison on just freed page
* intentionally. Although it's rather weird,
* it's how HWPoison flag works at the moment.
*/
if (set_hwpoison_free_buddy_page(page))
num_poisoned_pages_inc();
}
} else {
if (rc != -EAGAIN) {
if (likely(!__PageMovable(page))) {
putback_lru_page(page);
goto put_new;
}
lock_page(page);
if (PageMovable(page))
putback_movable_page(page);
else
__ClearPageIsolated(page);
unlock_page(page);
put_page(page);
}
put_new:
if (put_new_page)
put_new_page(newpage, private);
else
put_page(newpage);
}
return rc;
}
/*
* Counterpart of unmap_and_move_page() for hugepage migration.
*
* This function doesn't wait the completion of hugepage I/O
* because there is no race between I/O and migration for hugepage.
* Note that currently hugepage I/O occurs only in direct I/O
* where no lock is held and PG_writeback is irrelevant,
* and writeback status of all subpages are counted in the reference
* count of the head page (i.e. if all subpages of a 2MB hugepage are
* under direct I/O, the reference of the head page is 512 and a bit more.)
* This means that when we try to migrate hugepage whose subpages are
* doing direct I/O, some references remain after try_to_unmap() and
* hugepage migration fails without data corruption.
*
* There is also no race when direct I/O is issued on the page under migration,
* because then pte is replaced with migration swap entry and direct I/O code
* will wait in the page fault for migration to complete.
*/
static int unmap_and_move_huge_page(new_page_t get_new_page,
free_page_t put_new_page, unsigned long private,
struct page *hpage, int force,
enum migrate_mode mode, int reason)
{
int rc = -EAGAIN;
int page_was_mapped = 0;
struct page *new_hpage;
struct anon_vma *anon_vma = NULL;
/*
* Migratability of hugepages depends on architectures and their size.
* This check is necessary because some callers of hugepage migration
* like soft offline and memory hotremove don't walk through page
* tables or check whether the hugepage is pmd-based or not before
* kicking migration.
*/
if (!hugepage_migration_supported(page_hstate(hpage))) {
putback_active_hugepage(hpage);
return -ENOSYS;
}
new_hpage = get_new_page(hpage, private);
if (!new_hpage)
return -ENOMEM;
if (!trylock_page(hpage)) {
if (!force)
goto out;
switch (mode) {
case MIGRATE_SYNC:
case MIGRATE_SYNC_NO_COPY:
break;
default:
goto out;
}
lock_page(hpage);
}
/*
* Check for pages which are in the process of being freed. Without
* page_mapping() set, hugetlbfs specific move page routine will not
* be called and we could leak usage counts for subpools.
*/
if (page_private(hpage) && !page_mapping(hpage)) {
rc = -EBUSY;
goto out_unlock;
}
if (PageAnon(hpage))
anon_vma = page_get_anon_vma(hpage);
if (unlikely(!trylock_page(new_hpage)))
goto put_anon;
if (page_mapped(hpage)) {
try_to_unmap(hpage,
TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS, NULL);
page_was_mapped = 1;
}
if (!page_mapped(hpage))
rc = move_to_new_page(new_hpage, hpage, mode);
if (page_was_mapped)
remove_migration_ptes(hpage,
rc == MIGRATEPAGE_SUCCESS ? new_hpage : hpage, false);
unlock_page(new_hpage);
put_anon:
if (anon_vma)
put_anon_vma(anon_vma);
if (rc == MIGRATEPAGE_SUCCESS) {
move_hugetlb_state(hpage, new_hpage, reason);
put_new_page = NULL;
}
out_unlock:
unlock_page(hpage);
out:
if (rc != -EAGAIN)
putback_active_hugepage(hpage);
/*
* If migration was not successful and there's a freeing callback, use
* it. Otherwise, put_page() will drop the reference grabbed during
* isolation.
*/
if (put_new_page)
put_new_page(new_hpage, private);
else
putback_active_hugepage(new_hpage);
return rc;
}
/*
* migrate_pages - migrate the pages specified in a list, to the free pages
* supplied as the target for the page migration
*
* @from: The list of pages to be migrated.
* @get_new_page: The function used to allocate free pages to be used
* as the target of the page migration.
* @put_new_page: The function used to free target pages if migration
* fails, or NULL if no special handling is necessary.
* @private: Private data to be passed on to get_new_page()
* @mode: The migration mode that specifies the constraints for
* page migration, if any.
* @reason: The reason for page migration.
*
* The function returns after 10 attempts or if no pages are movable any more
* because the list has become empty or no retryable pages exist any more.
* The caller should call putback_movable_pages() to return pages to the LRU
* or free list only if ret != 0.
*
* Returns the number of pages that were not migrated, or an error code.
*/
int migrate_pages(struct list_head *from, new_page_t get_new_page,
free_page_t put_new_page, unsigned long private,
enum migrate_mode mode, int reason)
{
int retry = 1;
int nr_failed = 0;
int nr_succeeded = 0;
int pass = 0;
struct page *page;
struct page *page2;
int swapwrite = current->flags & PF_SWAPWRITE;
int rc;
trace_mm_migrate_pages_start(mode, reason);
if (!swapwrite)
current->flags |= PF_SWAPWRITE;
for(pass = 0; pass < 10 && retry; pass++) {
retry = 0;
list_for_each_entry_safe(page, page2, from, lru) {
retry:
cond_resched();
if (PageHuge(page))
rc = unmap_and_move_huge_page(get_new_page,
put_new_page, private, page,
pass > 2, mode, reason);
else
rc = unmap_and_move(get_new_page, put_new_page,
private, page, pass > 2, mode,
reason);
switch(rc) {
case -ENOMEM:
/*
* THP migration might be unsupported or the
* allocation could've failed so we should
* retry on the same page with the THP split
* to base pages.
*
* Head page is retried immediately and tail
* pages are added to the tail of the list so
* we encounter them after the rest of the list
* is processed.
*/
if (PageTransHuge(page) && !PageHuge(page)) {
lock_page(page);
rc = split_huge_page_to_list(page, from);
unlock_page(page);
if (!rc) {
list_safe_reset_next(page, page2, lru);
goto retry;
}
}
nr_failed++;
goto out;
case -EAGAIN:
retry++;
break;
case MIGRATEPAGE_SUCCESS:
nr_succeeded++;
break;
default:
/*
* Permanent failure (-EBUSY, -ENOSYS, etc.):
* unlike -EAGAIN case, the failed page is
* removed from migration page list and not
* retried in the next outer loop.
*/
nr_failed++;
break;
}
}
}
nr_failed += retry;
rc = nr_failed;
out:
if (nr_succeeded)
count_vm_events(PGMIGRATE_SUCCESS, nr_succeeded);
if (nr_failed)
count_vm_events(PGMIGRATE_FAIL, nr_failed);
trace_mm_migrate_pages(nr_succeeded, nr_failed, mode, reason);
if (!swapwrite)
current->flags &= ~PF_SWAPWRITE;
return rc;
}
#ifdef CONFIG_NUMA
static int store_status(int __user *status, int start, int value, int nr)
{
while (nr-- > 0) {
if (put_user(value, status + start))
return -EFAULT;
start++;
}
return 0;
}
static int do_move_pages_to_node(struct mm_struct *mm,
struct list_head *pagelist, int node)
{
int err;
if (list_empty(pagelist))
return 0;
err = migrate_pages(pagelist, alloc_new_node_page, NULL, node,
MIGRATE_SYNC, MR_SYSCALL);
if (err)
putback_movable_pages(pagelist);
return err;
}
/*
* Resolves the given address to a struct page, isolates it from the LRU and
* puts it to the given pagelist.
* Returns:
* errno - if the page cannot be found/isolated
* 0 - when it doesn't have to be migrated because it is already on the
* target node
* 1 - when it has been queued
*/
static int add_page_for_migration(struct mm_struct *mm, unsigned long addr,
int node, struct list_head *pagelist, bool migrate_all)
{
struct vm_area_struct *vma;
struct page *page;
unsigned int follflags;
int err;
down_read(&mm->mmap_sem);
err = -EFAULT;
vma = find_vma(mm, addr);
if (!vma || addr < vma->vm_start || !vma_migratable(vma))
goto out;
/* FOLL_DUMP to ignore special (like zero) pages */
follflags = FOLL_GET | FOLL_DUMP;
page = follow_page(vma, addr, follflags);
err = PTR_ERR(page);
if (IS_ERR(page))
goto out;
err = -ENOENT;
if (!page)
goto out;
err = 0;
if (page_to_nid(page) == node)
goto out_putpage;
err = -EACCES;
if (page_mapcount(page) > 1 && !migrate_all)
goto out_putpage;
if (PageHuge(page)) {
if (PageHead(page)) {
isolate_huge_page(page, pagelist);
err = 1;
}
} else {
struct page *head;
head = compound_head(page);
err = isolate_lru_page(head);
if (err)
goto out_putpage;
err = 1;
list_add_tail(&head->lru, pagelist);
mod_node_page_state(page_pgdat(head),
NR_ISOLATED_ANON + page_is_file_cache(head),
hpage_nr_pages(head));
}
out_putpage:
/*
* Either remove the duplicate refcount from
* isolate_lru_page() or drop the page ref if it was
* not isolated.
*/
put_page(page);
out:
up_read(&mm->mmap_sem);
return err;
}
/*
* Migrate an array of page address onto an array of nodes and fill
* the corresponding array of status.
*/
static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes,
unsigned long nr_pages,
const void __user * __user *pages,
const int __user *nodes,
int __user *status, int flags)
{
int current_node = NUMA_NO_NODE;
LIST_HEAD(pagelist);
int start, i;
int err = 0, err1;
migrate_prep();
for (i = start = 0; i < nr_pages; i++) {
const void __user *p;
unsigned long addr;
int node;
err = -EFAULT;
if (get_user(p, pages + i))
goto out_flush;
if (get_user(node, nodes + i))
goto out_flush;
addr = (unsigned long)untagged_addr(p);
err = -ENODEV;
if (node < 0 || node >= MAX_NUMNODES)
goto out_flush;
if (!node_state(node, N_MEMORY))
goto out_flush;
err = -EACCES;
if (!node_isset(node, task_nodes))
goto out_flush;
if (current_node == NUMA_NO_NODE) {
current_node = node;
start = i;
} else if (node != current_node) {
err = do_move_pages_to_node(mm, &pagelist, current_node);
if (err) {
/*
* Positive err means the number of failed
* pages to migrate. Since we are going to
* abort and return the number of non-migrated
* pages, so need to incude the rest of the
* nr_pages that have not been attempted as
* well.
*/
if (err > 0)
err += nr_pages - i - 1;
goto out;
}
err = store_status(status, start, current_node, i - start);
if (err)
goto out;
start = i;
current_node = node;
}
/*
* Errors in the page lookup or isolation are not fatal and we simply
* report them via status
*/
err = add_page_for_migration(mm, addr, current_node,
&pagelist, flags & MPOL_MF_MOVE_ALL);
if (!err) {
/* The page is already on the target node */
err = store_status(status, i, current_node, 1);
if (err)
goto out_flush;
continue;
} else if (err > 0) {
/* The page is successfully queued for migration */
continue;
}
err = store_status(status, i, err, 1);
if (err)
goto out_flush;
err = do_move_pages_to_node(mm, &pagelist, current_node);
if (err) {
if (err > 0)
err += nr_pages - i - 1;
goto out;
}
if (i > start) {
err = store_status(status, start, current_node, i - start);
if (err)
goto out;
}
current_node = NUMA_NO_NODE;
}
out_flush:
if (list_empty(&pagelist))
return err;
/* Make sure we do not overwrite the existing error */
err1 = do_move_pages_to_node(mm, &pagelist, current_node);
/*
* Don't have to report non-attempted pages here since:
* - If the above loop is done gracefully all pages have been
* attempted.
* - If the above loop is aborted it means a fatal error
* happened, should return ret.
*/
if (!err1)
err1 = store_status(status, start, current_node, i - start);
if (err >= 0)
err = err1;
out:
return err;
}
/*
* Determine the nodes of an array of pages and store it in an array of status.
*/
static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages,
const void __user **pages, int *status)
{
unsigned long i;
down_read(&mm->mmap_sem);
for (i = 0; i < nr_pages; i++) {
unsigned long addr = (unsigned long)(*pages);
struct vm_area_struct *vma;
struct page *page;
int err = -EFAULT;
vma = find_vma(mm, addr);
if (!vma || addr < vma->vm_start)
goto set_status;
/* FOLL_DUMP to ignore special (like zero) pages */
page = follow_page(vma, addr, FOLL_DUMP);
err = PTR_ERR(page);
if (IS_ERR(page))
goto set_status;
err = page ? page_to_nid(page) : -ENOENT;
set_status:
*status = err;
pages++;
status++;
}
up_read(&mm->mmap_sem);
}
/*
* Determine the nodes of a user array of pages and store it in
* a user array of status.
*/
static int do_pages_stat(struct mm_struct *mm, unsigned long nr_pages,
const void __user * __user *pages,
int __user *status)
{
#define DO_PAGES_STAT_CHUNK_NR 16
const void __user *chunk_pages[DO_PAGES_STAT_CHUNK_NR];
int chunk_status[DO_PAGES_STAT_CHUNK_NR];
while (nr_pages) {
unsigned long chunk_nr;
chunk_nr = nr_pages;
if (chunk_nr > DO_PAGES_STAT_CHUNK_NR)
chunk_nr = DO_PAGES_STAT_CHUNK_NR;
if (copy_from_user(chunk_pages, pages, chunk_nr * sizeof(*chunk_pages)))
break;
do_pages_stat_array(mm, chunk_nr, chunk_pages, chunk_status);
if (copy_to_user(status, chunk_status, chunk_nr * sizeof(*status)))
break;
pages += chunk_nr;
status += chunk_nr;
nr_pages -= chunk_nr;
}
return nr_pages ? -EFAULT : 0;
}
/*
* Move a list of pages in the address space of the currently executing
* process.
*/
static int kernel_move_pages(pid_t pid, unsigned long nr_pages,
const void __user * __user *pages,
const int __user *nodes,
int __user *status, int flags)
{
struct task_struct *task;
struct mm_struct *mm;
int err;
nodemask_t task_nodes;
/* Check flags */
if (flags & ~(MPOL_MF_MOVE|MPOL_MF_MOVE_ALL))
return -EINVAL;
if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
return -EPERM;
/* Find the mm_struct */
rcu_read_lock();
task = pid ? find_task_by_vpid(pid) : current;
if (!task) {
rcu_read_unlock();
return -ESRCH;
}
get_task_struct(task);
/*
* Check if this process has the right to modify the specified
* process. Use the regular "ptrace_may_access()" checks.
*/
if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS)) {
rcu_read_unlock();
err = -EPERM;
goto out;
}
rcu_read_unlock();
err = security_task_movememory(task);
if (err)
goto out;
task_nodes = cpuset_mems_allowed(task);
mm = get_task_mm(task);
put_task_struct(task);
if (!mm)
return -EINVAL;
if (nodes)
err = do_pages_move(mm, task_nodes, nr_pages, pages,
nodes, status, flags);
else
err = do_pages_stat(mm, nr_pages, pages, status);
mmput(mm);
return err;
out:
put_task_struct(task);
return err;
}
SYSCALL_DEFINE6(move_pages, pid_t, pid, unsigned long, nr_pages,
const void __user * __user *, pages,
const int __user *, nodes,
int __user *, status, int, flags)
{
return kernel_move_pages(pid, nr_pages, pages, nodes, status, flags);
}
#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE6(move_pages, pid_t, pid, compat_ulong_t, nr_pages,
compat_uptr_t __user *, pages32,
const int __user *, nodes,
int __user *, status,
int, flags)
{
const void __user * __user *pages;
int i;
pages = compat_alloc_user_space(nr_pages * sizeof(void *));
for (i = 0; i < nr_pages; i++) {
compat_uptr_t p;
if (get_user(p, pages32 + i) ||
put_user(compat_ptr(p), pages + i))
return -EFAULT;
}
return kernel_move_pages(pid, nr_pages, pages, nodes, status, flags);
}
#endif /* CONFIG_COMPAT */
#ifdef CONFIG_NUMA_BALANCING
/*
* Returns true if this is a safe migration target node for misplaced NUMA
* pages. Currently it only checks the watermarks which crude
*/
static bool migrate_balanced_pgdat(struct pglist_data *pgdat,
unsigned long nr_migrate_pages)
{
int z;
for (z = pgdat->nr_zones - 1; z >= 0; z--) {
struct zone *zone = pgdat->node_zones + z;
if (!populated_zone(zone))
continue;
/* Avoid waking kswapd by allocating pages_to_migrate pages. */
if (!zone_watermark_ok(zone, 0,
high_wmark_pages(zone) +
nr_migrate_pages,
0, 0))
continue;
return true;
}
return false;
}
static struct page *alloc_misplaced_dst_page(struct page *page,
unsigned long data)
{
int nid = (int) data;
struct page *newpage;
newpage = __alloc_pages_node(nid,
(GFP_HIGHUSER_MOVABLE |
__GFP_THISNODE | __GFP_NOMEMALLOC |
__GFP_NORETRY | __GFP_NOWARN) &
~__GFP_RECLAIM, 0);
return newpage;
}
static int numamigrate_isolate_page(pg_data_t *pgdat, struct page *page)
{
int page_lru;
VM_BUG_ON_PAGE(compound_order(page) && !PageTransHuge(page), page);
/* Avoid migrating to a node that is nearly full */
if (!migrate_balanced_pgdat(pgdat, compound_nr(page)))
return 0;
if (isolate_lru_page(page))
return 0;
/*
* migrate_misplaced_transhuge_page() skips page migration's usual
* check on page_count(), so we must do it here, now that the page
* has been isolated: a GUP pin, or any other pin, prevents migration.
* The expected page count is 3: 1 for page's mapcount and 1 for the
* caller's pin and 1 for the reference taken by isolate_lru_page().
*/
if (PageTransHuge(page) && page_count(page) != 3) {
putback_lru_page(page);
return 0;
}
page_lru = page_is_file_cache(page);
mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON + page_lru,
hpage_nr_pages(page));
/*
* Isolating the page has taken another reference, so the
* caller's reference can be safely dropped without the page
* disappearing underneath us during migration.
*/
put_page(page);
return 1;
}
bool pmd_trans_migrating(pmd_t pmd)
{
struct page *page = pmd_page(pmd);
return PageLocked(page);
}
/*
* Attempt to migrate a misplaced page to the specified destination
* node. Caller is expected to have an elevated reference count on
* the page that will be dropped by this function before returning.
*/
int migrate_misplaced_page(struct page *page, struct vm_fault *vmf,
int node)
{
pg_data_t *pgdat = NODE_DATA(node);
int isolated;
int nr_remaining;
LIST_HEAD(migratepages);
/*
* Don't migrate file pages that are mapped in multiple processes
* with execute permissions as they are probably shared libraries.
*/
if (page_mapcount(page) != 1 && page_is_file_cache(page) &&
(vmf->vma_flags & VM_EXEC))
goto out;
/*
* Also do not migrate dirty pages as not all filesystems can move
* dirty pages in MIGRATE_ASYNC mode which is a waste of cycles.
*/
if (page_is_file_cache(page) && PageDirty(page))
goto out;
isolated = numamigrate_isolate_page(pgdat, page);
if (!isolated)
goto out;
list_add(&page->lru, &migratepages);
nr_remaining = migrate_pages(&migratepages, alloc_misplaced_dst_page,
NULL, node, MIGRATE_ASYNC,
MR_NUMA_MISPLACED);
if (nr_remaining) {
if (!list_empty(&migratepages)) {
list_del(&page->lru);
dec_node_page_state(page, NR_ISOLATED_ANON +
page_is_file_cache(page));
putback_lru_page(page);
}
isolated = 0;
} else
count_vm_numa_event(NUMA_PAGE_MIGRATE);
BUG_ON(!list_empty(&migratepages));
return isolated;
out:
put_page(page);
return 0;
}
#endif /* CONFIG_NUMA_BALANCING */
#if defined(CONFIG_NUMA_BALANCING) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
/*
* Migrates a THP to a given target node. page must be locked and is unlocked
* before returning.
*/
int migrate_misplaced_transhuge_page(struct mm_struct *mm,
struct vm_area_struct *vma,
pmd_t *pmd, pmd_t entry,
unsigned long address,
struct page *page, int node)
{
spinlock_t *ptl;
pg_data_t *pgdat = NODE_DATA(node);
int isolated = 0;
struct page *new_page = NULL;
int page_lru = page_is_file_cache(page);
unsigned long start = address & HPAGE_PMD_MASK;
new_page = alloc_pages_node(node,
(GFP_TRANSHUGE_LIGHT | __GFP_THISNODE),
HPAGE_PMD_ORDER);
if (!new_page)
goto out_fail;
prep_transhuge_page(new_page);
isolated = numamigrate_isolate_page(pgdat, page);
if (!isolated) {
put_page(new_page);
goto out_fail;
}
/* Prepare a page as a migration target */
__SetPageLocked(new_page);
if (PageSwapBacked(page))
__SetPageSwapBacked(new_page);
/* anon mapping, we can simply copy page->mapping to the new page: */
new_page->mapping = page->mapping;
new_page->index = page->index;
/* flush the cache before copying using the kernel virtual address */
flush_cache_range(vma, start, start + HPAGE_PMD_SIZE);
migrate_page_copy(new_page, page);
WARN_ON(PageLRU(new_page));
/* Recheck the target PMD */
ptl = pmd_lock(mm, pmd);
if (unlikely(!pmd_same(*pmd, entry) || !page_ref_freeze(page, 2))) {
spin_unlock(ptl);
/* Reverse changes made by migrate_page_copy() */
if (TestClearPageActive(new_page))
SetPageActive(page);
if (TestClearPageUnevictable(new_page))
SetPageUnevictable(page);
unlock_page(new_page);
put_page(new_page); /* Free it */
/* Retake the callers reference and putback on LRU */
get_page(page);
putback_lru_page(page);
mod_node_page_state(page_pgdat(page),
NR_ISOLATED_ANON + page_lru, -HPAGE_PMD_NR);
goto out_unlock;
}
entry = mk_huge_pmd(new_page, vma->vm_page_prot);
entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
/*
* Overwrite the old entry under pagetable lock and establish
* the new PTE. Any parallel GUP will either observe the old
* page blocking on the page lock, block on the page table
* lock or observe the new page. The SetPageUptodate on the
* new page and page_add_new_anon_rmap guarantee the copy is
* visible before the pagetable update.
*/
page_add_anon_rmap(new_page, vma, start, true);
/*
* At this point the pmd is numa/protnone (i.e. non present) and the TLB
* has already been flushed globally. So no TLB can be currently
* caching this non present pmd mapping. There's no need to clear the
* pmd before doing set_pmd_at(), nor to flush the TLB after
* set_pmd_at(). Clearing the pmd here would introduce a race
* condition against MADV_DONTNEED, because MADV_DONTNEED only holds the
* mmap_sem for reading. If the pmd is set to NULL at any given time,
* MADV_DONTNEED won't wait on the pmd lock and it'll skip clearing this
* pmd.
*/
set_pmd_at(mm, start, pmd, entry);
update_mmu_cache_pmd(vma, address, &entry);
page_ref_unfreeze(page, 2);
mlock_migrate_page(new_page, page);
page_remove_rmap(page, true);
set_page_owner_migrate_reason(new_page, MR_NUMA_MISPLACED);
spin_unlock(ptl);
/* Take an "isolate" reference and put new page on the LRU. */
get_page(new_page);
putback_lru_page(new_page);
unlock_page(new_page);
unlock_page(page);
put_page(page); /* Drop the rmap reference */
put_page(page); /* Drop the LRU isolation reference */
count_vm_events(PGMIGRATE_SUCCESS, HPAGE_PMD_NR);
count_vm_numa_events(NUMA_PAGE_MIGRATE, HPAGE_PMD_NR);
mod_node_page_state(page_pgdat(page),
NR_ISOLATED_ANON + page_lru,
-HPAGE_PMD_NR);
return isolated;
out_fail:
count_vm_events(PGMIGRATE_FAIL, HPAGE_PMD_NR);
ptl = pmd_lock(mm, pmd);
if (pmd_same(*pmd, entry)) {
entry = pmd_modify(entry, vma->vm_page_prot);
set_pmd_at(mm, start, pmd, entry);
update_mmu_cache_pmd(vma, address, &entry);
}
spin_unlock(ptl);
out_unlock:
unlock_page(page);
put_page(page);
return 0;
}
#endif /* CONFIG_NUMA_BALANCING */
#endif /* CONFIG_NUMA */
#ifdef CONFIG_DEVICE_PRIVATE
static int migrate_vma_collect_hole(unsigned long start,
unsigned long end,
struct mm_walk *walk)
{
struct migrate_vma *migrate = walk->private;
unsigned long addr;
for (addr = start & PAGE_MASK; addr < end; addr += PAGE_SIZE) {
migrate->src[migrate->npages] = MIGRATE_PFN_MIGRATE;
migrate->dst[migrate->npages] = 0;
migrate->npages++;
migrate->cpages++;
}
return 0;
}
static int migrate_vma_collect_skip(unsigned long start,
unsigned long end,
struct mm_walk *walk)
{
struct migrate_vma *migrate = walk->private;
unsigned long addr;
for (addr = start & PAGE_MASK; addr < end; addr += PAGE_SIZE) {
migrate->dst[migrate->npages] = 0;
migrate->src[migrate->npages++] = 0;
}
return 0;
}
static int migrate_vma_collect_pmd(pmd_t *pmdp,
unsigned long start,
unsigned long end,
struct mm_walk *walk)
{
struct migrate_vma *migrate = walk->private;
struct vm_area_struct *vma = walk->vma;
struct mm_struct *mm = vma->vm_mm;
unsigned long addr = start, unmapped = 0;
spinlock_t *ptl;
pte_t *ptep;
again:
if (pmd_none(*pmdp))
return migrate_vma_collect_hole(start, end, walk);
if (pmd_trans_huge(*pmdp)) {
struct page *page;
ptl = pmd_lock(mm, pmdp);
if (unlikely(!pmd_trans_huge(*pmdp))) {
spin_unlock(ptl);
goto again;
}
page = pmd_page(*pmdp);
if (is_huge_zero_page(page)) {
spin_unlock(ptl);
split_huge_pmd(vma, pmdp, addr);
if (pmd_trans_unstable(pmdp))
return migrate_vma_collect_skip(start, end,
walk);
} else {
int ret;
get_page(page);
spin_unlock(ptl);
if (unlikely(!trylock_page(page)))
return migrate_vma_collect_skip(start, end,
walk);
ret = split_huge_page(page);
unlock_page(page);
put_page(page);
if (ret)
return migrate_vma_collect_skip(start, end,
walk);
if (pmd_none(*pmdp))
return migrate_vma_collect_hole(start, end,
walk);
}
}
if (unlikely(pmd_bad(*pmdp)))
return migrate_vma_collect_skip(start, end, walk);
ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl);
arch_enter_lazy_mmu_mode();
for (; addr < end; addr += PAGE_SIZE, ptep++) {
unsigned long mpfn, pfn;
struct page *page;
swp_entry_t entry;
pte_t pte;
pte = *ptep;
if (pte_none(pte)) {
mpfn = MIGRATE_PFN_MIGRATE;
migrate->cpages++;
goto next;
}
if (!pte_present(pte)) {
mpfn = 0;
/*
* Only care about unaddressable device page special
* page table entry. Other special swap entries are not
* migratable, and we ignore regular swapped page.
*/
entry = pte_to_swp_entry(pte);
if (!is_device_private_entry(entry))
goto next;
page = device_private_entry_to_page(entry);
mpfn = migrate_pfn(page_to_pfn(page)) |
MIGRATE_PFN_MIGRATE;
if (is_write_device_private_entry(entry))
mpfn |= MIGRATE_PFN_WRITE;
} else {
pfn = pte_pfn(pte);
if (is_zero_pfn(pfn)) {
mpfn = MIGRATE_PFN_MIGRATE;
migrate->cpages++;
goto next;
}
page = vm_normal_page(migrate->vma, addr, pte);
mpfn = migrate_pfn(pfn) | MIGRATE_PFN_MIGRATE;
mpfn |= pte_write(pte) ? MIGRATE_PFN_WRITE : 0;
}
/* FIXME support THP */
if (!page || !page->mapping || PageTransCompound(page)) {
mpfn = 0;
goto next;
}
/*
* By getting a reference on the page we pin it and that blocks
* any kind of migration. Side effect is that it "freezes" the
* pte.
*
* We drop this reference after isolating the page from the lru
* for non device page (device page are not on the lru and thus
* can't be dropped from it).
*/
get_page(page);
migrate->cpages++;
/*
* Optimize for the common case where page is only mapped once
* in one process. If we can lock the page, then we can safely
* set up a special migration page table entry now.
*/
if (trylock_page(page)) {
pte_t swp_pte;
mpfn |= MIGRATE_PFN_LOCKED;
ptep_get_and_clear(mm, addr, ptep);
/* Setup special migration page table entry */
entry = make_migration_entry(page, mpfn &
MIGRATE_PFN_WRITE);
swp_pte = swp_entry_to_pte(entry);
if (pte_soft_dirty(pte))
swp_pte = pte_swp_mksoft_dirty(swp_pte);
set_pte_at(mm, addr, ptep, swp_pte);
/*
* This is like regular unmap: we remove the rmap and
* drop page refcount. Page won't be freed, as we took
* a reference just above.
*/
page_remove_rmap(page, false);
put_page(page);
if (pte_present(pte))
unmapped++;
}
next:
migrate->dst[migrate->npages] = 0;
migrate->src[migrate->npages++] = mpfn;
}
/* Only flush the TLB if we actually modified any entries */
if (unmapped)
flush_tlb_range(walk->vma, start, end);
arch_leave_lazy_mmu_mode();
pte_unmap_unlock(ptep - 1, ptl);
return 0;
}
static const struct mm_walk_ops migrate_vma_walk_ops = {
.pmd_entry = migrate_vma_collect_pmd,
.pte_hole = migrate_vma_collect_hole,
};
/*
* migrate_vma_collect() - collect pages over a range of virtual addresses
* @migrate: migrate struct containing all migration information
*
* This will walk the CPU page table. For each virtual address backed by a
* valid page, it updates the src array and takes a reference on the page, in
* order to pin the page until we lock it and unmap it.
*/
static void migrate_vma_collect(struct migrate_vma *migrate)
{
struct mmu_notifier_range range;
mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, NULL,
migrate->vma->vm_mm, migrate->start, migrate->end);
mmu_notifier_invalidate_range_start(&range);
walk_page_range(migrate->vma->vm_mm, migrate->start, migrate->end,
&migrate_vma_walk_ops, migrate);
mmu_notifier_invalidate_range_end(&range);
migrate->end = migrate->start + (migrate->npages << PAGE_SHIFT);
}
/*
* migrate_vma_check_page() - check if page is pinned or not
* @page: struct page to check
*
* Pinned pages cannot be migrated. This is the same test as in
* migrate_page_move_mapping(), except that here we allow migration of a
* ZONE_DEVICE page.
*/
static bool migrate_vma_check_page(struct page *page)
{
/*
* One extra ref because caller holds an extra reference, either from
* isolate_lru_page() for a regular page, or migrate_vma_collect() for
* a device page.
*/
int extra = 1;
/*
* FIXME support THP (transparent huge page), it is bit more complex to
* check them than regular pages, because they can be mapped with a pmd
* or with a pte (split pte mapping).
*/
if (PageCompound(page))
return false;
/* Page from ZONE_DEVICE have one extra reference */
if (is_zone_device_page(page)) {
/*
* Private page can never be pin as they have no valid pte and
* GUP will fail for those. Yet if there is a pending migration
* a thread might try to wait on the pte migration entry and
* will bump the page reference count. Sadly there is no way to
* differentiate a regular pin from migration wait. Hence to
* avoid 2 racing thread trying to migrate back to CPU to enter
* infinite loop (one stoping migration because the other is
* waiting on pte migration entry). We always return true here.
*
* FIXME proper solution is to rework migration_entry_wait() so
* it does not need to take a reference on page.
*/
return is_device_private_page(page);
}
/* For file back page */
if (page_mapping(page))
extra += 1 + page_has_private(page);
if ((page_count(page) - extra) > page_mapcount(page))
return false;
return true;
}
/*
* migrate_vma_prepare() - lock pages and isolate them from the lru
* @migrate: migrate struct containing all migration information
*
* This locks pages that have been collected by migrate_vma_collect(). Once each
* page is locked it is isolated from the lru (for non-device pages). Finally,
* the ref taken by migrate_vma_collect() is dropped, as locked pages cannot be
* migrated by concurrent kernel threads.
*/
static void migrate_vma_prepare(struct migrate_vma *migrate)
{
const unsigned long npages = migrate->npages;
const unsigned long start = migrate->start;
unsigned long addr, i, restore = 0;
bool allow_drain = true;
lru_add_drain();
for (i = 0; (i < npages) && migrate->cpages; i++) {
struct page *page = migrate_pfn_to_page(migrate->src[i]);
bool remap = true;
if (!page)
continue;
if (!(migrate->src[i] & MIGRATE_PFN_LOCKED)) {
/*
* Because we are migrating several pages there can be
* a deadlock between 2 concurrent migration where each
* are waiting on each other page lock.
*
* Make migrate_vma() a best effort thing and backoff
* for any page we can not lock right away.
*/
if (!trylock_page(page)) {
migrate->src[i] = 0;
migrate->cpages--;
put_page(page);
continue;
}
remap = false;
migrate->src[i] |= MIGRATE_PFN_LOCKED;
}
/* ZONE_DEVICE pages are not on LRU */
if (!is_zone_device_page(page)) {
if (!PageLRU(page) && allow_drain) {
/* Drain CPU's pagevec */
lru_add_drain_all();
allow_drain = false;
}
if (isolate_lru_page(page)) {
if (remap) {
migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
migrate->cpages--;
restore++;
} else {
migrate->src[i] = 0;
unlock_page(page);
migrate->cpages--;
put_page(page);
}
continue;
}
/* Drop the reference we took in collect */
put_page(page);
}
if (!migrate_vma_check_page(page)) {
if (remap) {
migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
migrate->cpages--;
restore++;
if (!is_zone_device_page(page)) {
get_page(page);
putback_lru_page(page);
}
} else {
migrate->src[i] = 0;
unlock_page(page);
migrate->cpages--;
if (!is_zone_device_page(page))
putback_lru_page(page);
else
put_page(page);
}
}
}
for (i = 0, addr = start; i < npages && restore; i++, addr += PAGE_SIZE) {
struct page *page = migrate_pfn_to_page(migrate->src[i]);
if (!page || (migrate->src[i] & MIGRATE_PFN_MIGRATE))
continue;
remove_migration_pte(page, migrate->vma, addr, page);
migrate->src[i] = 0;
unlock_page(page);
put_page(page);
restore--;
}
}
/*
* migrate_vma_unmap() - replace page mapping with special migration pte entry
* @migrate: migrate struct containing all migration information
*
* Replace page mapping (CPU page table pte) with a special migration pte entry
* and check again if it has been pinned. Pinned pages are restored because we
* cannot migrate them.
*
* This is the last step before we call the device driver callback to allocate
* destination memory and copy contents of original page over to new page.
*/
static void migrate_vma_unmap(struct migrate_vma *migrate)
{
int flags = TTU_MIGRATION | TTU_IGNORE_MLOCK | TTU_IGNORE_ACCESS;
const unsigned long npages = migrate->npages;
const unsigned long start = migrate->start;
unsigned long addr, i, restore = 0;
for (i = 0; i < npages; i++) {
struct page *page = migrate_pfn_to_page(migrate->src[i]);
if (!page || !(migrate->src[i] & MIGRATE_PFN_MIGRATE))
continue;
if (page_mapped(page)) {
try_to_unmap(page, flags, NULL);
if (page_mapped(page))
goto restore;
}
if (migrate_vma_check_page(page))
continue;
restore:
migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
migrate->cpages--;
restore++;
}
for (addr = start, i = 0; i < npages && restore; addr += PAGE_SIZE, i++) {
struct page *page = migrate_pfn_to_page(migrate->src[i]);
if (!page || (migrate->src[i] & MIGRATE_PFN_MIGRATE))
continue;
remove_migration_ptes(page, page, false);
migrate->src[i] = 0;
unlock_page(page);
restore--;
if (is_zone_device_page(page))
put_page(page);
else
putback_lru_page(page);
}
}
/**
* migrate_vma_setup() - prepare to migrate a range of memory
* @args: contains the vma, start, and and pfns arrays for the migration
*
* Returns: negative errno on failures, 0 when 0 or more pages were migrated
* without an error.
*
* Prepare to migrate a range of memory virtual address range by collecting all
* the pages backing each virtual address in the range, saving them inside the
* src array. Then lock those pages and unmap them. Once the pages are locked
* and unmapped, check whether each page is pinned or not. Pages that aren't
* pinned have the MIGRATE_PFN_MIGRATE flag set (by this function) in the
* corresponding src array entry. Then restores any pages that are pinned, by
* remapping and unlocking those pages.
*
* The caller should then allocate destination memory and copy source memory to
* it for all those entries (ie with MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE
* flag set). Once these are allocated and copied, the caller must update each
* corresponding entry in the dst array with the pfn value of the destination
* page and with the MIGRATE_PFN_VALID and MIGRATE_PFN_LOCKED flags set
* (destination pages must have their struct pages locked, via lock_page()).
*
* Note that the caller does not have to migrate all the pages that are marked
* with MIGRATE_PFN_MIGRATE flag in src array unless this is a migration from
* device memory to system memory. If the caller cannot migrate a device page
* back to system memory, then it must return VM_FAULT_SIGBUS, which has severe
* consequences for the userspace process, so it must be avoided if at all
* possible.
*
* For empty entries inside CPU page table (pte_none() or pmd_none() is true) we
* do set MIGRATE_PFN_MIGRATE flag inside the corresponding source array thus
* allowing the caller to allocate device memory for those unback virtual
* address. For this the caller simply has to allocate device memory and
* properly set the destination entry like for regular migration. Note that
* this can still fails and thus inside the device driver must check if the
* migration was successful for those entries after calling migrate_vma_pages()
* just like for regular migration.
*
* After that, the callers must call migrate_vma_pages() to go over each entry
* in the src array that has the MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE flag
* set. If the corresponding entry in dst array has MIGRATE_PFN_VALID flag set,
* then migrate_vma_pages() to migrate struct page information from the source
* struct page to the destination struct page. If it fails to migrate the
* struct page information, then it clears the MIGRATE_PFN_MIGRATE flag in the
* src array.
*
* At this point all successfully migrated pages have an entry in the src
* array with MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE flag set and the dst
* array entry with MIGRATE_PFN_VALID flag set.
*
* Once migrate_vma_pages() returns the caller may inspect which pages were
* successfully migrated, and which were not. Successfully migrated pages will
* have the MIGRATE_PFN_MIGRATE flag set for their src array entry.
*
* It is safe to update device page table after migrate_vma_pages() because
* both destination and source page are still locked, and the mmap_sem is held
* in read mode (hence no one can unmap the range being migrated).
*
* Once the caller is done cleaning up things and updating its page table (if it
* chose to do so, this is not an obligation) it finally calls
* migrate_vma_finalize() to update the CPU page table to point to new pages
* for successfully migrated pages or otherwise restore the CPU page table to
* point to the original source pages.
*/
int migrate_vma_setup(struct migrate_vma *args)
{
long nr_pages = (args->end - args->start) >> PAGE_SHIFT;
args->start &= PAGE_MASK;
args->end &= PAGE_MASK;
if (!args->vma || is_vm_hugetlb_page(args->vma) ||
(args->vma->vm_flags & VM_SPECIAL) || vma_is_dax(args->vma))
return -EINVAL;
if (nr_pages <= 0)
return -EINVAL;
if (args->start < args->vma->vm_start ||
args->start >= args->vma->vm_end)
return -EINVAL;
if (args->end <= args->vma->vm_start || args->end > args->vma->vm_end)
return -EINVAL;
if (!args->src || !args->dst)
return -EINVAL;
memset(args->src, 0, sizeof(*args->src) * nr_pages);
args->cpages = 0;
args->npages = 0;
migrate_vma_collect(args);
if (args->cpages)
migrate_vma_prepare(args);
if (args->cpages)
migrate_vma_unmap(args);
/*
* At this point pages are locked and unmapped, and thus they have
* stable content and can safely be copied to destination memory that
* is allocated by the drivers.
*/
return 0;
}
EXPORT_SYMBOL(migrate_vma_setup);
static void migrate_vma_insert_page(struct migrate_vma *migrate,
unsigned long addr,
struct page *page,
unsigned long *src,
unsigned long *dst)
{
struct vm_area_struct *vma = migrate->vma;
struct mm_struct *mm = vma->vm_mm;
struct mem_cgroup *memcg;
bool flush = false;
spinlock_t *ptl;
pte_t entry;
pgd_t *pgdp;
p4d_t *p4dp;
pud_t *pudp;
pmd_t *pmdp;
pte_t *ptep;
/* Only allow populating anonymous memory */
if (!vma_is_anonymous(vma))
goto abort;
pgdp = pgd_offset(mm, addr);
p4dp = p4d_alloc(mm, pgdp, addr);
if (!p4dp)
goto abort;
pudp = pud_alloc(mm, p4dp, addr);
if (!pudp)
goto abort;
pmdp = pmd_alloc(mm, pudp, addr);
if (!pmdp)
goto abort;
if (pmd_trans_huge(*pmdp) || pmd_devmap(*pmdp))
goto abort;
/*
* Use pte_alloc() instead of pte_alloc_map(). We can't run
* pte_offset_map() on pmds where a huge pmd might be created
* from a different thread.
*
* pte_alloc_map() is safe to use under down_write(mmap_sem) or when
* parallel threads are excluded by other means.
*
* Here we only have down_read(mmap_sem).
*/
if (pte_alloc(mm, pmdp))
goto abort;
/* See the comment in pte_alloc_one_map() */
if (unlikely(pmd_trans_unstable(pmdp)))
goto abort;
if (unlikely(anon_vma_prepare(vma)))
goto abort;
if (mem_cgroup_try_charge(page, vma->vm_mm, GFP_KERNEL, &memcg, false))
goto abort;
/*
* The memory barrier inside __SetPageUptodate makes sure that
* preceding stores to the page contents become visible before
* the set_pte_at() write.
*/
__SetPageUptodate(page);
if (is_zone_device_page(page)) {
if (is_device_private_page(page)) {
swp_entry_t swp_entry;
swp_entry = make_device_private_entry(page, vma->vm_flags & VM_WRITE);
entry = swp_entry_to_pte(swp_entry);
} else {
/*
* For now we only support migrating to un-addressable
* device memory.
*/
pr_warn_once("Unsupported ZONE_DEVICE page type.\n");
goto abort;
}
} else {
entry = mk_pte(page, vma->vm_page_prot);
if (vma->vm_flags & VM_WRITE)
entry = pte_mkwrite(pte_mkdirty(entry));
}
ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl);
if (pte_present(*ptep)) {
unsigned long pfn = pte_pfn(*ptep);
if (!is_zero_pfn(pfn)) {
pte_unmap_unlock(ptep, ptl);
mem_cgroup_cancel_charge(page, memcg, false);
goto abort;
}
flush = true;
} else if (!pte_none(*ptep)) {
pte_unmap_unlock(ptep, ptl);
mem_cgroup_cancel_charge(page, memcg, false);
goto abort;
}
/*
* Check for usefaultfd but do not deliver the fault. Instead,
* just back off.
*/
if (userfaultfd_missing(vma)) {
pte_unmap_unlock(ptep, ptl);
mem_cgroup_cancel_charge(page, memcg, false);
goto abort;
}
inc_mm_counter(mm, MM_ANONPAGES);
page_add_new_anon_rmap(page, vma, addr, false);
mem_cgroup_commit_charge(page, memcg, false, false);
if (!is_zone_device_page(page))
lru_cache_add_active_or_unevictable(page, vma);
get_page(page);
if (flush) {
flush_cache_page(vma, addr, pte_pfn(*ptep));
ptep_clear_flush_notify(vma, addr, ptep);
set_pte_at_notify(mm, addr, ptep, entry);
update_mmu_cache(vma, addr, ptep);
} else {
/* No need to invalidate - it was non-present before */
set_pte_at(mm, addr, ptep, entry);
update_mmu_cache(vma, addr, ptep);
}
pte_unmap_unlock(ptep, ptl);
*src = MIGRATE_PFN_MIGRATE;
return;
abort:
*src &= ~MIGRATE_PFN_MIGRATE;
}
/**
* migrate_vma_pages() - migrate meta-data from src page to dst page
* @migrate: migrate struct containing all migration information
*
* This migrates struct page meta-data from source struct page to destination
* struct page. This effectively finishes the migration from source page to the
* destination page.
*/
void migrate_vma_pages(struct migrate_vma *migrate)
{
const unsigned long npages = migrate->npages;
const unsigned long start = migrate->start;
struct mmu_notifier_range range;
unsigned long addr, i;
bool notified = false;
for (i = 0, addr = start; i < npages; addr += PAGE_SIZE, i++) {
struct page *newpage = migrate_pfn_to_page(migrate->dst[i]);
struct page *page = migrate_pfn_to_page(migrate->src[i]);
struct address_space *mapping;
int r;
if (!newpage) {
migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
continue;
}
if (!page) {
if (!(migrate->src[i] & MIGRATE_PFN_MIGRATE)) {
continue;
}
if (!notified) {
notified = true;
mmu_notifier_range_init(&range,
MMU_NOTIFY_CLEAR, 0,
NULL,
migrate->vma->vm_mm,
addr, migrate->end);
mmu_notifier_invalidate_range_start(&range);
}
migrate_vma_insert_page(migrate, addr, newpage,
&migrate->src[i],
&migrate->dst[i]);
continue;
}
mapping = page_mapping(page);
if (is_zone_device_page(newpage)) {
if (is_device_private_page(newpage)) {
/*
* For now only support private anonymous when
* migrating to un-addressable device memory.
*/
if (mapping) {
migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
continue;
}
} else {
/*
* Other types of ZONE_DEVICE page are not
* supported.
*/
migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
continue;
}
}
r = migrate_page(mapping, newpage, page, MIGRATE_SYNC_NO_COPY);
if (r != MIGRATEPAGE_SUCCESS)
migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
}
/*
* No need to double call mmu_notifier->invalidate_range() callback as
* the above ptep_clear_flush_notify() inside migrate_vma_insert_page()
* did already call it.
*/
if (notified)
mmu_notifier_invalidate_range_only_end(&range);
}
EXPORT_SYMBOL(migrate_vma_pages);
/**
* migrate_vma_finalize() - restore CPU page table entry
* @migrate: migrate struct containing all migration information
*
* This replaces the special migration pte entry with either a mapping to the
* new page if migration was successful for that page, or to the original page
* otherwise.
*
* This also unlocks the pages and puts them back on the lru, or drops the extra
* refcount, for device pages.
*/
void migrate_vma_finalize(struct migrate_vma *migrate)
{
const unsigned long npages = migrate->npages;
unsigned long i;
for (i = 0; i < npages; i++) {
struct page *newpage = migrate_pfn_to_page(migrate->dst[i]);
struct page *page = migrate_pfn_to_page(migrate->src[i]);
if (!page) {
if (newpage) {
unlock_page(newpage);
put_page(newpage);
}
continue;
}
if (!(migrate->src[i] & MIGRATE_PFN_MIGRATE) || !newpage) {
if (newpage) {
unlock_page(newpage);
put_page(newpage);
}
newpage = page;
}
remove_migration_ptes(page, newpage, false);
unlock_page(page);
migrate->cpages--;
if (is_zone_device_page(page))
put_page(page);
else
putback_lru_page(page);
if (newpage != page) {
unlock_page(newpage);
if (is_zone_device_page(newpage))
put_page(newpage);
else
putback_lru_page(newpage);
}
}
}
EXPORT_SYMBOL(migrate_vma_finalize);
#endif /* CONFIG_DEVICE_PRIVATE */