Файловый менеджер - Редактировать - /var/www/html/64.zip
Ðазад
PK ! `��� � mmu.hnu �[��� /* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_BOOK3S_64_MMU_H_ #define _ASM_POWERPC_BOOK3S_64_MMU_H_ #include <asm/page.h> #ifndef __ASSEMBLY__ /* * Page size definition * * shift : is the "PAGE_SHIFT" value for that page size * sllp : is a bit mask with the value of SLB L || LP to be or'ed * directly to a slbmte "vsid" value * penc : is the HPTE encoding mask for the "LP" field: * */ struct mmu_psize_def { unsigned int shift; /* number of bits */ int penc[MMU_PAGE_COUNT]; /* HPTE encoding */ unsigned int tlbiel; /* tlbiel supported for that page size */ unsigned long avpnm; /* bits to mask out in AVPN in the HPTE */ unsigned long h_rpt_pgsize; /* H_RPT_INVALIDATE page size encoding */ union { unsigned long sllp; /* SLB L||LP (exact mask to use in slbmte) */ unsigned long ap; /* Ap encoding used by PowerISA 3.0 */ }; }; extern struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT]; #endif /* __ASSEMBLY__ */ /* 64-bit classic hash table MMU */ #include <asm/book3s/64/mmu-hash.h> #ifndef __ASSEMBLY__ /* * ISA 3.0 partition and process table entry format */ struct prtb_entry { __be64 prtb0; __be64 prtb1; }; extern struct prtb_entry *process_tb; struct patb_entry { __be64 patb0; __be64 patb1; }; extern struct patb_entry *partition_tb; /* Bits in patb0 field */ #define PATB_HR (1UL << 63) #define RPDB_MASK 0x0fffffffffffff00UL #define RPDB_SHIFT (1UL << 8) #define RTS1_SHIFT 61 /* top 2 bits of radix tree size */ #define RTS1_MASK (3UL << RTS1_SHIFT) #define RTS2_SHIFT 5 /* bottom 3 bits of radix tree size */ #define RTS2_MASK (7UL << RTS2_SHIFT) #define RPDS_MASK 0x1f /* root page dir. size field */ /* Bits in patb1 field */ #define PATB_GR (1UL << 63) /* guest uses radix; must match HR */ #define PRTS_MASK 0x1f /* process table size field */ #define PRTB_MASK 0x0ffffffffffff000UL /* Number of supported PID bits */ extern unsigned int mmu_pid_bits; /* Base PID to allocate from */ extern unsigned int mmu_base_pid; /* * memory block size used with radix translation. */ extern unsigned long __ro_after_init radix_mem_block_size; #define PRTB_SIZE_SHIFT (mmu_pid_bits + 4) #define PRTB_ENTRIES (1ul << mmu_pid_bits) /* * Power9 currently only support 64K partition table size. */ #define PATB_SIZE_SHIFT 16 typedef unsigned long mm_context_id_t; struct spinlock; /* Maximum possible number of NPUs in a system. */ #define NV_MAX_NPUS 8 typedef struct { union { /* * We use id as the PIDR content for radix. On hash we can use * more than one id. The extended ids are used when we start * having address above 512TB. We allocate one extended id * for each 512TB. The new id is then used with the 49 bit * EA to build a new VA. We always use ESID_BITS_1T_MASK bits * from EA and new context ids to build the new VAs. */ mm_context_id_t id; mm_context_id_t extended_id[TASK_SIZE_USER64/TASK_CONTEXT_SIZE]; }; /* Number of bits in the mm_cpumask */ atomic_t active_cpus; /* Number of users of the external (Nest) MMU */ atomic_t copros; /* Number of user space windows opened in process mm_context */ atomic_t vas_windows; struct hash_mm_context *hash_context; void __user *vdso; /* * pagetable fragment support */ void *pte_frag; void *pmd_frag; #ifdef CONFIG_SPAPR_TCE_IOMMU struct list_head iommu_group_mem_list; #endif #ifdef CONFIG_PPC_MEM_KEYS /* * Each bit represents one protection key. * bit set -> key allocated * bit unset -> key available for allocation */ u32 pkey_allocation_map; s16 execute_only_pkey; /* key holding execute-only protection */ #endif } mm_context_t; static inline u16 mm_ctx_user_psize(mm_context_t *ctx) { return ctx->hash_context->user_psize; } static inline void mm_ctx_set_user_psize(mm_context_t *ctx, u16 user_psize) { ctx->hash_context->user_psize = user_psize; } static inline unsigned char *mm_ctx_low_slices(mm_context_t *ctx) { return ctx->hash_context->low_slices_psize; } static inline unsigned char *mm_ctx_high_slices(mm_context_t *ctx) { return ctx->hash_context->high_slices_psize; } static inline unsigned long mm_ctx_slb_addr_limit(mm_context_t *ctx) { return ctx->hash_context->slb_addr_limit; } static inline void mm_ctx_set_slb_addr_limit(mm_context_t *ctx, unsigned long limit) { ctx->hash_context->slb_addr_limit = limit; } static inline struct slice_mask *slice_mask_for_size(mm_context_t *ctx, int psize) { #ifdef CONFIG_PPC_64K_PAGES if (psize == MMU_PAGE_64K) return &ctx->hash_context->mask_64k; #endif #ifdef CONFIG_HUGETLB_PAGE if (psize == MMU_PAGE_16M) return &ctx->hash_context->mask_16m; if (psize == MMU_PAGE_16G) return &ctx->hash_context->mask_16g; #endif BUG_ON(psize != MMU_PAGE_4K); return &ctx->hash_context->mask_4k; } #ifdef CONFIG_PPC_SUBPAGE_PROT static inline struct subpage_prot_table *mm_ctx_subpage_prot(mm_context_t *ctx) { return ctx->hash_context->spt; } #endif /* * The current system page and segment sizes */ extern int mmu_linear_psize; extern int mmu_virtual_psize; extern int mmu_vmalloc_psize; extern int mmu_vmemmap_psize; extern int mmu_io_psize; /* MMU initialization */ void mmu_early_init_devtree(void); void hash__early_init_devtree(void); void radix__early_init_devtree(void); #ifdef CONFIG_PPC_PKEY void pkey_early_init_devtree(void); #else static inline void pkey_early_init_devtree(void) {} #endif extern void hash__early_init_mmu(void); extern void radix__early_init_mmu(void); static inline void __init early_init_mmu(void) { if (radix_enabled()) return radix__early_init_mmu(); return hash__early_init_mmu(); } extern void hash__early_init_mmu_secondary(void); extern void radix__early_init_mmu_secondary(void); static inline void early_init_mmu_secondary(void) { if (radix_enabled()) return radix__early_init_mmu_secondary(); return hash__early_init_mmu_secondary(); } extern void hash__setup_initial_memory_limit(phys_addr_t first_memblock_base, phys_addr_t first_memblock_size); static inline void setup_initial_memory_limit(phys_addr_t first_memblock_base, phys_addr_t first_memblock_size) { /* * Hash has more strict restrictions. At this point we don't * know which translations we will pick. Hence go with hash * restrictions. */ return hash__setup_initial_memory_limit(first_memblock_base, first_memblock_size); } #ifdef CONFIG_PPC_PSERIES extern void radix_init_pseries(void); #else static inline void radix_init_pseries(void) { } #endif #ifdef CONFIG_HOTPLUG_CPU #define arch_clear_mm_cpumask_cpu(cpu, mm) \ do { \ if (cpumask_test_cpu(cpu, mm_cpumask(mm))) { \ atomic_dec(&(mm)->context.active_cpus); \ cpumask_clear_cpu(cpu, mm_cpumask(mm)); \ } \ } while (0) void cleanup_cpu_mmu_context(void); #endif static inline int get_user_context(mm_context_t *ctx, unsigned long ea) { int index = ea >> MAX_EA_BITS_PER_CONTEXT; if (likely(index < ARRAY_SIZE(ctx->extended_id))) return ctx->extended_id[index]; /* should never happen */ WARN_ON(1); return 0; } static inline unsigned long get_user_vsid(mm_context_t *ctx, unsigned long ea, int ssize) { unsigned long context = get_user_context(ctx, ea); return get_vsid(context, ea, ssize); } #endif /* __ASSEMBLY__ */ #endif /* _ASM_POWERPC_BOOK3S_64_MMU_H_ */ PK ! � �}p }p mmu-hash.hnu �[��� /* SPDX-License-Identifier: GPL-2.0-or-later */ #ifndef _ASM_POWERPC_BOOK3S_64_MMU_HASH_H_ #define _ASM_POWERPC_BOOK3S_64_MMU_HASH_H_ /* * PowerPC64 memory management structures * * Dave Engebretsen & Mike Corrigan <{engebret|mikejc}@us.ibm.com> * PPC64 rework. */ #include <asm/page.h> #include <asm/bug.h> #include <asm/asm-const.h> /* * This is necessary to get the definition of PGTABLE_RANGE which we * need for various slices related matters. Note that this isn't the * complete pgtable.h but only a portion of it. */ #include <asm/book3s/64/pgtable.h> #include <asm/task_size_64.h> #include <asm/cpu_has_feature.h> /* * SLB */ #define SLB_NUM_BOLTED 2 #define SLB_CACHE_ENTRIES 8 #define SLB_MIN_SIZE 32 /* Bits in the SLB ESID word */ #define SLB_ESID_V ASM_CONST(0x0000000008000000) /* valid */ /* Bits in the SLB VSID word */ #define SLB_VSID_SHIFT 12 #define SLB_VSID_SHIFT_256M SLB_VSID_SHIFT #define SLB_VSID_SHIFT_1T 24 #define SLB_VSID_SSIZE_SHIFT 62 #define SLB_VSID_B ASM_CONST(0xc000000000000000) #define SLB_VSID_B_256M ASM_CONST(0x0000000000000000) #define SLB_VSID_B_1T ASM_CONST(0x4000000000000000) #define SLB_VSID_KS ASM_CONST(0x0000000000000800) #define SLB_VSID_KP ASM_CONST(0x0000000000000400) #define SLB_VSID_N ASM_CONST(0x0000000000000200) /* no-execute */ #define SLB_VSID_L ASM_CONST(0x0000000000000100) #define SLB_VSID_C ASM_CONST(0x0000000000000080) /* class */ #define SLB_VSID_LP ASM_CONST(0x0000000000000030) #define SLB_VSID_LP_00 ASM_CONST(0x0000000000000000) #define SLB_VSID_LP_01 ASM_CONST(0x0000000000000010) #define SLB_VSID_LP_10 ASM_CONST(0x0000000000000020) #define SLB_VSID_LP_11 ASM_CONST(0x0000000000000030) #define SLB_VSID_LLP (SLB_VSID_L|SLB_VSID_LP) #define SLB_VSID_KERNEL (SLB_VSID_KP) #define SLB_VSID_USER (SLB_VSID_KP|SLB_VSID_KS|SLB_VSID_C) #define SLBIE_C (0x08000000) #define SLBIE_SSIZE_SHIFT 25 /* * Hash table */ #define HPTES_PER_GROUP 8 #define HPTE_V_SSIZE_SHIFT 62 #define HPTE_V_AVPN_SHIFT 7 #define HPTE_V_COMMON_BITS ASM_CONST(0x000fffffffffffff) #define HPTE_V_AVPN ASM_CONST(0x3fffffffffffff80) #define HPTE_V_AVPN_3_0 ASM_CONST(0x000fffffffffff80) #define HPTE_V_AVPN_VAL(x) (((x) & HPTE_V_AVPN) >> HPTE_V_AVPN_SHIFT) #define HPTE_V_COMPARE(x,y) (!(((x) ^ (y)) & 0xffffffffffffff80UL)) #define HPTE_V_BOLTED ASM_CONST(0x0000000000000010) #define HPTE_V_LOCK ASM_CONST(0x0000000000000008) #define HPTE_V_LARGE ASM_CONST(0x0000000000000004) #define HPTE_V_SECONDARY ASM_CONST(0x0000000000000002) #define HPTE_V_VALID ASM_CONST(0x0000000000000001) /* * ISA 3.0 has a different HPTE format. */ #define HPTE_R_3_0_SSIZE_SHIFT 58 #define HPTE_R_3_0_SSIZE_MASK (3ull << HPTE_R_3_0_SSIZE_SHIFT) #define HPTE_R_PP0 ASM_CONST(0x8000000000000000) #define HPTE_R_TS ASM_CONST(0x4000000000000000) #define HPTE_R_KEY_HI ASM_CONST(0x3000000000000000) #define HPTE_R_KEY_BIT4 ASM_CONST(0x2000000000000000) #define HPTE_R_KEY_BIT3 ASM_CONST(0x1000000000000000) #define HPTE_R_RPN_SHIFT 12 #define HPTE_R_RPN ASM_CONST(0x0ffffffffffff000) #define HPTE_R_RPN_3_0 ASM_CONST(0x01fffffffffff000) #define HPTE_R_PP ASM_CONST(0x0000000000000003) #define HPTE_R_PPP ASM_CONST(0x8000000000000003) #define HPTE_R_N ASM_CONST(0x0000000000000004) #define HPTE_R_G ASM_CONST(0x0000000000000008) #define HPTE_R_M ASM_CONST(0x0000000000000010) #define HPTE_R_I ASM_CONST(0x0000000000000020) #define HPTE_R_W ASM_CONST(0x0000000000000040) #define HPTE_R_WIMG ASM_CONST(0x0000000000000078) #define HPTE_R_C ASM_CONST(0x0000000000000080) #define HPTE_R_R ASM_CONST(0x0000000000000100) #define HPTE_R_KEY_LO ASM_CONST(0x0000000000000e00) #define HPTE_R_KEY_BIT2 ASM_CONST(0x0000000000000800) #define HPTE_R_KEY_BIT1 ASM_CONST(0x0000000000000400) #define HPTE_R_KEY_BIT0 ASM_CONST(0x0000000000000200) #define HPTE_R_KEY (HPTE_R_KEY_LO | HPTE_R_KEY_HI) #define HPTE_V_1TB_SEG ASM_CONST(0x4000000000000000) #define HPTE_V_VRMA_MASK ASM_CONST(0x4001ffffff000000) /* Values for PP (assumes Ks=0, Kp=1) */ #define PP_RWXX 0 /* Supervisor read/write, User none */ #define PP_RWRX 1 /* Supervisor read/write, User read */ #define PP_RWRW 2 /* Supervisor read/write, User read/write */ #define PP_RXRX 3 /* Supervisor read, User read */ #define PP_RXXX (HPTE_R_PP0 | 2) /* Supervisor read, user none */ /* Fields for tlbiel instruction in architecture 2.06 */ #define TLBIEL_INVAL_SEL_MASK 0xc00 /* invalidation selector */ #define TLBIEL_INVAL_PAGE 0x000 /* invalidate a single page */ #define TLBIEL_INVAL_SET_LPID 0x800 /* invalidate a set for current LPID */ #define TLBIEL_INVAL_SET 0xc00 /* invalidate a set for all LPIDs */ #define TLBIEL_INVAL_SET_MASK 0xfff000 /* set number to inval. */ #define TLBIEL_INVAL_SET_SHIFT 12 #define POWER7_TLB_SETS 128 /* # sets in POWER7 TLB */ #define POWER8_TLB_SETS 512 /* # sets in POWER8 TLB */ #define POWER9_TLB_SETS_HASH 256 /* # sets in POWER9 TLB Hash mode */ #define POWER9_TLB_SETS_RADIX 128 /* # sets in POWER9 TLB Radix mode */ #ifndef __ASSEMBLY__ struct mmu_hash_ops { void (*hpte_invalidate)(unsigned long slot, unsigned long vpn, int bpsize, int apsize, int ssize, int local); long (*hpte_updatepp)(unsigned long slot, unsigned long newpp, unsigned long vpn, int bpsize, int apsize, int ssize, unsigned long flags); void (*hpte_updateboltedpp)(unsigned long newpp, unsigned long ea, int psize, int ssize); long (*hpte_insert)(unsigned long hpte_group, unsigned long vpn, unsigned long prpn, unsigned long rflags, unsigned long vflags, int psize, int apsize, int ssize); long (*hpte_remove)(unsigned long hpte_group); int (*hpte_removebolted)(unsigned long ea, int psize, int ssize); void (*flush_hash_range)(unsigned long number, int local); void (*hugepage_invalidate)(unsigned long vsid, unsigned long addr, unsigned char *hpte_slot_array, int psize, int ssize, int local); int (*resize_hpt)(unsigned long shift); /* * Special for kexec. * To be called in real mode with interrupts disabled. No locks are * taken as such, concurrent access on pre POWER5 hardware could result * in a deadlock. * The linear mapping is destroyed as well. */ void (*hpte_clear_all)(void); }; extern struct mmu_hash_ops mmu_hash_ops; struct hash_pte { __be64 v; __be64 r; }; extern struct hash_pte *htab_address; extern unsigned long htab_size_bytes; extern unsigned long htab_hash_mask; static inline int shift_to_mmu_psize(unsigned int shift) { int psize; for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) if (mmu_psize_defs[psize].shift == shift) return psize; return -1; } static inline unsigned int mmu_psize_to_shift(unsigned int mmu_psize) { if (mmu_psize_defs[mmu_psize].shift) return mmu_psize_defs[mmu_psize].shift; BUG(); } static inline unsigned int ap_to_shift(unsigned long ap) { int psize; for (psize = 0; psize < MMU_PAGE_COUNT; psize++) { if (mmu_psize_defs[psize].ap == ap) return mmu_psize_defs[psize].shift; } return -1; } static inline unsigned long get_sllp_encoding(int psize) { unsigned long sllp; sllp = ((mmu_psize_defs[psize].sllp & SLB_VSID_L) >> 6) | ((mmu_psize_defs[psize].sllp & SLB_VSID_LP) >> 4); return sllp; } #endif /* __ASSEMBLY__ */ /* * Segment sizes. * These are the values used by hardware in the B field of * SLB entries and the first dword of MMU hashtable entries. * The B field is 2 bits; the values 2 and 3 are unused and reserved. */ #define MMU_SEGSIZE_256M 0 #define MMU_SEGSIZE_1T 1 /* * encode page number shift. * in order to fit the 78 bit va in a 64 bit variable we shift the va by * 12 bits. This enable us to address upto 76 bit va. * For hpt hash from a va we can ignore the page size bits of va and for * hpte encoding we ignore up to 23 bits of va. So ignoring lower 12 bits ensure * we work in all cases including 4k page size. */ #define VPN_SHIFT 12 /* * HPTE Large Page (LP) details */ #define LP_SHIFT 12 #define LP_BITS 8 #define LP_MASK(i) ((0xFF >> (i)) << LP_SHIFT) #ifndef __ASSEMBLY__ static inline int slb_vsid_shift(int ssize) { if (ssize == MMU_SEGSIZE_256M) return SLB_VSID_SHIFT; return SLB_VSID_SHIFT_1T; } static inline int segment_shift(int ssize) { if (ssize == MMU_SEGSIZE_256M) return SID_SHIFT; return SID_SHIFT_1T; } /* * This array is indexed by the LP field of the HPTE second dword. * Since this field may contain some RPN bits, some entries are * replicated so that we get the same value irrespective of RPN. * The top 4 bits are the page size index (MMU_PAGE_*) for the * actual page size, the bottom 4 bits are the base page size. */ extern u8 hpte_page_sizes[1 << LP_BITS]; static inline unsigned long __hpte_page_size(unsigned long h, unsigned long l, bool is_base_size) { unsigned int i, lp; if (!(h & HPTE_V_LARGE)) return 1ul << 12; /* Look at the 8 bit LP value */ lp = (l >> LP_SHIFT) & ((1 << LP_BITS) - 1); i = hpte_page_sizes[lp]; if (!i) return 0; if (!is_base_size) i >>= 4; return 1ul << mmu_psize_defs[i & 0xf].shift; } static inline unsigned long hpte_page_size(unsigned long h, unsigned long l) { return __hpte_page_size(h, l, 0); } static inline unsigned long hpte_base_page_size(unsigned long h, unsigned long l) { return __hpte_page_size(h, l, 1); } /* * The current system page and segment sizes */ extern int mmu_kernel_ssize; extern int mmu_highuser_ssize; extern u16 mmu_slb_size; extern unsigned long tce_alloc_start, tce_alloc_end; /* * If the processor supports 64k normal pages but not 64k cache * inhibited pages, we have to be prepared to switch processes * to use 4k pages when they create cache-inhibited mappings. * If this is the case, mmu_ci_restrictions will be set to 1. */ extern int mmu_ci_restrictions; /* * This computes the AVPN and B fields of the first dword of a HPTE, * for use when we want to match an existing PTE. The bottom 7 bits * of the returned value are zero. */ static inline unsigned long hpte_encode_avpn(unsigned long vpn, int psize, int ssize) { unsigned long v; /* * The AVA field omits the low-order 23 bits of the 78 bits VA. * These bits are not needed in the PTE, because the * low-order b of these bits are part of the byte offset * into the virtual page and, if b < 23, the high-order * 23-b of these bits are always used in selecting the * PTEGs to be searched */ v = (vpn >> (23 - VPN_SHIFT)) & ~(mmu_psize_defs[psize].avpnm); v <<= HPTE_V_AVPN_SHIFT; v |= ((unsigned long) ssize) << HPTE_V_SSIZE_SHIFT; return v; } /* * ISA v3.0 defines a new HPTE format, which differs from the old * format in having smaller AVPN and ARPN fields, and the B field * in the second dword instead of the first. */ static inline unsigned long hpte_old_to_new_v(unsigned long v) { /* trim AVPN, drop B */ return v & HPTE_V_COMMON_BITS; } static inline unsigned long hpte_old_to_new_r(unsigned long v, unsigned long r) { /* move B field from 1st to 2nd dword, trim ARPN */ return (r & ~HPTE_R_3_0_SSIZE_MASK) | (((v) >> HPTE_V_SSIZE_SHIFT) << HPTE_R_3_0_SSIZE_SHIFT); } static inline unsigned long hpte_new_to_old_v(unsigned long v, unsigned long r) { /* insert B field */ return (v & HPTE_V_COMMON_BITS) | ((r & HPTE_R_3_0_SSIZE_MASK) << (HPTE_V_SSIZE_SHIFT - HPTE_R_3_0_SSIZE_SHIFT)); } static inline unsigned long hpte_new_to_old_r(unsigned long r) { /* clear out B field */ return r & ~HPTE_R_3_0_SSIZE_MASK; } static inline unsigned long hpte_get_old_v(struct hash_pte *hptep) { unsigned long hpte_v; hpte_v = be64_to_cpu(hptep->v); if (cpu_has_feature(CPU_FTR_ARCH_300)) hpte_v = hpte_new_to_old_v(hpte_v, be64_to_cpu(hptep->r)); return hpte_v; } /* * This function sets the AVPN and L fields of the HPTE appropriately * using the base page size and actual page size. */ static inline unsigned long hpte_encode_v(unsigned long vpn, int base_psize, int actual_psize, int ssize) { unsigned long v; v = hpte_encode_avpn(vpn, base_psize, ssize); if (actual_psize != MMU_PAGE_4K) v |= HPTE_V_LARGE; return v; } /* * This function sets the ARPN, and LP fields of the HPTE appropriately * for the page size. We assume the pa is already "clean" that is properly * aligned for the requested page size */ static inline unsigned long hpte_encode_r(unsigned long pa, int base_psize, int actual_psize) { /* A 4K page needs no special encoding */ if (actual_psize == MMU_PAGE_4K) return pa & HPTE_R_RPN; else { unsigned int penc = mmu_psize_defs[base_psize].penc[actual_psize]; unsigned int shift = mmu_psize_defs[actual_psize].shift; return (pa & ~((1ul << shift) - 1)) | (penc << LP_SHIFT); } } /* * Build a VPN_SHIFT bit shifted va given VSID, EA and segment size. */ static inline unsigned long hpt_vpn(unsigned long ea, unsigned long vsid, int ssize) { unsigned long mask; int s_shift = segment_shift(ssize); mask = (1ul << (s_shift - VPN_SHIFT)) - 1; return (vsid << (s_shift - VPN_SHIFT)) | ((ea >> VPN_SHIFT) & mask); } /* * This hashes a virtual address */ static inline unsigned long hpt_hash(unsigned long vpn, unsigned int shift, int ssize) { unsigned long mask; unsigned long hash, vsid; /* VPN_SHIFT can be atmost 12 */ if (ssize == MMU_SEGSIZE_256M) { mask = (1ul << (SID_SHIFT - VPN_SHIFT)) - 1; hash = (vpn >> (SID_SHIFT - VPN_SHIFT)) ^ ((vpn & mask) >> (shift - VPN_SHIFT)); } else { mask = (1ul << (SID_SHIFT_1T - VPN_SHIFT)) - 1; vsid = vpn >> (SID_SHIFT_1T - VPN_SHIFT); hash = vsid ^ (vsid << 25) ^ ((vpn & mask) >> (shift - VPN_SHIFT)) ; } return hash & 0x7fffffffffUL; } #define HPTE_LOCAL_UPDATE 0x1 #define HPTE_NOHPTE_UPDATE 0x2 #define HPTE_USE_KERNEL_KEY 0x4 long hpte_insert_repeating(unsigned long hash, unsigned long vpn, unsigned long pa, unsigned long rlags, unsigned long vflags, int psize, int ssize); extern int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid, pte_t *ptep, unsigned long trap, unsigned long flags, int ssize, int subpage_prot); extern int __hash_page_64K(unsigned long ea, unsigned long access, unsigned long vsid, pte_t *ptep, unsigned long trap, unsigned long flags, int ssize); struct mm_struct; unsigned int hash_page_do_lazy_icache(unsigned int pp, pte_t pte, int trap); extern int hash_page_mm(struct mm_struct *mm, unsigned long ea, unsigned long access, unsigned long trap, unsigned long flags); extern int hash_page(unsigned long ea, unsigned long access, unsigned long trap, unsigned long dsisr); void low_hash_fault(struct pt_regs *regs, unsigned long address, int rc); int __hash_page(unsigned long trap, unsigned long ea, unsigned long dsisr, unsigned long msr); int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid, pte_t *ptep, unsigned long trap, unsigned long flags, int ssize, unsigned int shift, unsigned int mmu_psize); #ifdef CONFIG_TRANSPARENT_HUGEPAGE extern int __hash_page_thp(unsigned long ea, unsigned long access, unsigned long vsid, pmd_t *pmdp, unsigned long trap, unsigned long flags, int ssize, unsigned int psize); #else static inline int __hash_page_thp(unsigned long ea, unsigned long access, unsigned long vsid, pmd_t *pmdp, unsigned long trap, unsigned long flags, int ssize, unsigned int psize) { BUG(); return -1; } #endif extern void hash_failure_debug(unsigned long ea, unsigned long access, unsigned long vsid, unsigned long trap, int ssize, int psize, int lpsize, unsigned long pte); extern int htab_bolt_mapping(unsigned long vstart, unsigned long vend, unsigned long pstart, unsigned long prot, int psize, int ssize); int htab_remove_mapping(unsigned long vstart, unsigned long vend, int psize, int ssize); extern void pseries_add_gpage(u64 addr, u64 page_size, unsigned long number_of_pages); extern void demote_segment_4k(struct mm_struct *mm, unsigned long addr); extern void hash__setup_new_exec(void); #ifdef CONFIG_PPC_PSERIES void hpte_init_pseries(void); #else static inline void hpte_init_pseries(void) { } #endif extern void hpte_init_native(void); struct slb_entry { u64 esid; u64 vsid; }; extern void slb_initialize(void); void slb_flush_and_restore_bolted(void); void slb_flush_all_realmode(void); void __slb_restore_bolted_realmode(void); void slb_restore_bolted_realmode(void); void slb_save_contents(struct slb_entry *slb_ptr); void slb_dump_contents(struct slb_entry *slb_ptr); extern void slb_vmalloc_update(void); extern void slb_set_size(u16 size); #endif /* __ASSEMBLY__ */ /* * VSID allocation (256MB segment) * * We first generate a 37-bit "proto-VSID". Proto-VSIDs are generated * from mmu context id and effective segment id of the address. * * For user processes max context id is limited to MAX_USER_CONTEXT. * more details in get_user_context * * For kernel space get_kernel_context * * The proto-VSIDs are then scrambled into real VSIDs with the * multiplicative hash: * * VSID = (proto-VSID * VSID_MULTIPLIER) % VSID_MODULUS * * VSID_MULTIPLIER is prime, so in particular it is * co-prime to VSID_MODULUS, making this a 1:1 scrambling function. * Because the modulus is 2^n-1 we can compute it efficiently without * a divide or extra multiply (see below). The scramble function gives * robust scattering in the hash table (at least based on some initial * results). * * We use VSID 0 to indicate an invalid VSID. The means we can't use context id * 0, because a context id of 0 and an EA of 0 gives a proto-VSID of 0, which * will produce a VSID of 0. * * We also need to avoid the last segment of the last context, because that * would give a protovsid of 0x1fffffffff. That will result in a VSID 0 * because of the modulo operation in vsid scramble. */ /* * Max Va bits we support as of now is 68 bits. We want 19 bit * context ID. * Restrictions: * GPU has restrictions of not able to access beyond 128TB * (47 bit effective address). We also cannot do more than 20bit PID. * For p4 and p5 which can only do 65 bit VA, we restrict our CONTEXT_BITS * to 16 bits (ie, we can only have 2^16 pids at the same time). */ #define VA_BITS 68 #define CONTEXT_BITS 19 #define ESID_BITS (VA_BITS - (SID_SHIFT + CONTEXT_BITS)) #define ESID_BITS_1T (VA_BITS - (SID_SHIFT_1T + CONTEXT_BITS)) #define ESID_BITS_MASK ((1 << ESID_BITS) - 1) #define ESID_BITS_1T_MASK ((1 << ESID_BITS_1T) - 1) /* * Now certain config support MAX_PHYSMEM more than 512TB. Hence we will need * to use more than one context for linear mapping the kernel. * For vmalloc and memmap, we use just one context with 512TB. With 64 byte * struct page size, we need ony 32 TB in memmap for 2PB (51 bits (MAX_PHYSMEM_BITS)). */ #if (H_MAX_PHYSMEM_BITS > MAX_EA_BITS_PER_CONTEXT) #define MAX_KERNEL_CTX_CNT (1UL << (H_MAX_PHYSMEM_BITS - MAX_EA_BITS_PER_CONTEXT)) #else #define MAX_KERNEL_CTX_CNT 1 #endif #define MAX_VMALLOC_CTX_CNT 1 #define MAX_IO_CTX_CNT 1 #define MAX_VMEMMAP_CTX_CNT 1 /* * 256MB segment * The proto-VSID space has 2^(CONTEX_BITS + ESID_BITS) - 1 segments * available for user + kernel mapping. VSID 0 is reserved as invalid, contexts * 1-4 are used for kernel mapping. Each segment contains 2^28 bytes. Each * context maps 2^49 bytes (512TB). * * We also need to avoid the last segment of the last context, because that * would give a protovsid of 0x1fffffffff. That will result in a VSID 0 * because of the modulo operation in vsid scramble. * */ #define MAX_USER_CONTEXT ((ASM_CONST(1) << CONTEXT_BITS) - 2) // The + 2 accounts for INVALID_REGION and 1 more to avoid overlap with kernel #define MIN_USER_CONTEXT (MAX_KERNEL_CTX_CNT + MAX_VMALLOC_CTX_CNT + \ MAX_IO_CTX_CNT + MAX_VMEMMAP_CTX_CNT + 2) /* * For platforms that support on 65bit VA we limit the context bits */ #define MAX_USER_CONTEXT_65BIT_VA ((ASM_CONST(1) << (65 - (SID_SHIFT + ESID_BITS))) - 2) /* * This should be computed such that protovosid * vsid_mulitplier * doesn't overflow 64 bits. The vsid_mutliplier should also be * co-prime to vsid_modulus. We also need to make sure that number * of bits in multiplied result (dividend) is less than twice the number of * protovsid bits for our modulus optmization to work. * * The below table shows the current values used. * |-------+------------+----------------------+------------+-------------------| * | | Prime Bits | proto VSID_BITS_65VA | Total Bits | 2* prot VSID_BITS | * |-------+------------+----------------------+------------+-------------------| * | 1T | 24 | 25 | 49 | 50 | * |-------+------------+----------------------+------------+-------------------| * | 256MB | 24 | 37 | 61 | 74 | * |-------+------------+----------------------+------------+-------------------| * * |-------+------------+----------------------+------------+--------------------| * | | Prime Bits | proto VSID_BITS_68VA | Total Bits | 2* proto VSID_BITS | * |-------+------------+----------------------+------------+--------------------| * | 1T | 24 | 28 | 52 | 56 | * |-------+------------+----------------------+------------+--------------------| * | 256MB | 24 | 40 | 64 | 80 | * |-------+------------+----------------------+------------+--------------------| * */ #define VSID_MULTIPLIER_256M ASM_CONST(12538073) /* 24-bit prime */ #define VSID_BITS_256M (VA_BITS - SID_SHIFT) #define VSID_BITS_65_256M (65 - SID_SHIFT) /* * Modular multiplicative inverse of VSID_MULTIPLIER under modulo VSID_MODULUS */ #define VSID_MULINV_256M ASM_CONST(665548017062) #define VSID_MULTIPLIER_1T ASM_CONST(12538073) /* 24-bit prime */ #define VSID_BITS_1T (VA_BITS - SID_SHIFT_1T) #define VSID_BITS_65_1T (65 - SID_SHIFT_1T) #define VSID_MULINV_1T ASM_CONST(209034062) /* 1TB VSID reserved for VRMA */ #define VRMA_VSID 0x1ffffffUL #define USER_VSID_RANGE (1UL << (ESID_BITS + SID_SHIFT)) /* 4 bits per slice and we have one slice per 1TB */ #define SLICE_ARRAY_SIZE (H_PGTABLE_RANGE >> 41) #define LOW_SLICE_ARRAY_SZ (BITS_PER_LONG / BITS_PER_BYTE) #define TASK_SLICE_ARRAY_SZ(x) ((x)->hash_context->slb_addr_limit >> 41) #ifndef __ASSEMBLY__ #ifdef CONFIG_PPC_SUBPAGE_PROT /* * For the sub-page protection option, we extend the PGD with one of * these. Basically we have a 3-level tree, with the top level being * the protptrs array. To optimize speed and memory consumption when * only addresses < 4GB are being protected, pointers to the first * four pages of sub-page protection words are stored in the low_prot * array. * Each page of sub-page protection words protects 1GB (4 bytes * protects 64k). For the 3-level tree, each page of pointers then * protects 8TB. */ struct subpage_prot_table { unsigned long maxaddr; /* only addresses < this are protected */ unsigned int **protptrs[(TASK_SIZE_USER64 >> 43)]; unsigned int *low_prot[4]; }; #define SBP_L1_BITS (PAGE_SHIFT - 2) #define SBP_L2_BITS (PAGE_SHIFT - 3) #define SBP_L1_COUNT (1 << SBP_L1_BITS) #define SBP_L2_COUNT (1 << SBP_L2_BITS) #define SBP_L2_SHIFT (PAGE_SHIFT + SBP_L1_BITS) #define SBP_L3_SHIFT (SBP_L2_SHIFT + SBP_L2_BITS) extern void subpage_prot_free(struct mm_struct *mm); #else static inline void subpage_prot_free(struct mm_struct *mm) {} #endif /* CONFIG_PPC_SUBPAGE_PROT */ /* * One bit per slice. We have lower slices which cover 256MB segments * upto 4G range. That gets us 16 low slices. For the rest we track slices * in 1TB size. */ struct slice_mask { u64 low_slices; DECLARE_BITMAP(high_slices, SLICE_NUM_HIGH); }; struct hash_mm_context { u16 user_psize; /* page size index */ /* SLB page size encodings*/ unsigned char low_slices_psize[LOW_SLICE_ARRAY_SZ]; unsigned char high_slices_psize[SLICE_ARRAY_SIZE]; unsigned long slb_addr_limit; #ifdef CONFIG_PPC_64K_PAGES struct slice_mask mask_64k; #endif struct slice_mask mask_4k; #ifdef CONFIG_HUGETLB_PAGE struct slice_mask mask_16m; struct slice_mask mask_16g; #endif #ifdef CONFIG_PPC_SUBPAGE_PROT struct subpage_prot_table *spt; #endif /* CONFIG_PPC_SUBPAGE_PROT */ }; #if 0 /* * The code below is equivalent to this function for arguments * < 2^VSID_BITS, which is all this should ever be called * with. However gcc is not clever enough to compute the * modulus (2^n-1) without a second multiply. */ #define vsid_scramble(protovsid, size) \ ((((protovsid) * VSID_MULTIPLIER_##size) % VSID_MODULUS_##size)) /* simplified form avoiding mod operation */ #define vsid_scramble(protovsid, size) \ ({ \ unsigned long x; \ x = (protovsid) * VSID_MULTIPLIER_##size; \ x = (x >> VSID_BITS_##size) + (x & VSID_MODULUS_##size); \ (x + ((x+1) >> VSID_BITS_##size)) & VSID_MODULUS_##size; \ }) #else /* 1 */ static inline unsigned long vsid_scramble(unsigned long protovsid, unsigned long vsid_multiplier, int vsid_bits) { unsigned long vsid; unsigned long vsid_modulus = ((1UL << vsid_bits) - 1); /* * We have same multipler for both 256 and 1T segements now */ vsid = protovsid * vsid_multiplier; vsid = (vsid >> vsid_bits) + (vsid & vsid_modulus); return (vsid + ((vsid + 1) >> vsid_bits)) & vsid_modulus; } #endif /* 1 */ /* Returns the segment size indicator for a user address */ static inline int user_segment_size(unsigned long addr) { /* Use 1T segments if possible for addresses >= 1T */ if (addr >= (1UL << SID_SHIFT_1T)) return mmu_highuser_ssize; return MMU_SEGSIZE_256M; } static inline unsigned long get_vsid(unsigned long context, unsigned long ea, int ssize) { unsigned long va_bits = VA_BITS; unsigned long vsid_bits; unsigned long protovsid; /* * Bad address. We return VSID 0 for that */ if ((ea & EA_MASK) >= H_PGTABLE_RANGE) return 0; if (!mmu_has_feature(MMU_FTR_68_BIT_VA)) va_bits = 65; if (ssize == MMU_SEGSIZE_256M) { vsid_bits = va_bits - SID_SHIFT; protovsid = (context << ESID_BITS) | ((ea >> SID_SHIFT) & ESID_BITS_MASK); return vsid_scramble(protovsid, VSID_MULTIPLIER_256M, vsid_bits); } /* 1T segment */ vsid_bits = va_bits - SID_SHIFT_1T; protovsid = (context << ESID_BITS_1T) | ((ea >> SID_SHIFT_1T) & ESID_BITS_1T_MASK); return vsid_scramble(protovsid, VSID_MULTIPLIER_1T, vsid_bits); } /* * For kernel space, we use context ids as * below. Range is 512TB per context. * * 0x00001 - [ 0xc000000000000000 - 0xc001ffffffffffff] * 0x00002 - [ 0xc002000000000000 - 0xc003ffffffffffff] * 0x00003 - [ 0xc004000000000000 - 0xc005ffffffffffff] * 0x00004 - [ 0xc006000000000000 - 0xc007ffffffffffff] * * vmap, IO, vmemap * * 0x00005 - [ 0xc008000000000000 - 0xc009ffffffffffff] * 0x00006 - [ 0xc00a000000000000 - 0xc00bffffffffffff] * 0x00007 - [ 0xc00c000000000000 - 0xc00dffffffffffff] * */ static inline unsigned long get_kernel_context(unsigned long ea) { unsigned long region_id = get_region_id(ea); unsigned long ctx; /* * Depending on Kernel config, kernel region can have one context * or more. */ if (region_id == LINEAR_MAP_REGION_ID) { /* * We already verified ea to be not beyond the addr limit. */ ctx = 1 + ((ea & EA_MASK) >> MAX_EA_BITS_PER_CONTEXT); } else ctx = region_id + MAX_KERNEL_CTX_CNT - 1; return ctx; } /* * This is only valid for addresses >= PAGE_OFFSET */ static inline unsigned long get_kernel_vsid(unsigned long ea, int ssize) { unsigned long context; if (!is_kernel_addr(ea)) return 0; context = get_kernel_context(ea); return get_vsid(context, ea, ssize); } unsigned htab_shift_for_mem_size(unsigned long mem_size); enum slb_index { LINEAR_INDEX = 0, /* Kernel linear map (0xc000000000000000) */ KSTACK_INDEX = 1, /* Kernel stack map */ }; #define slb_esid_mask(ssize) \ (((ssize) == MMU_SEGSIZE_256M) ? ESID_MASK : ESID_MASK_1T) static inline unsigned long mk_esid_data(unsigned long ea, int ssize, enum slb_index index) { return (ea & slb_esid_mask(ssize)) | SLB_ESID_V | index; } static inline unsigned long __mk_vsid_data(unsigned long vsid, int ssize, unsigned long flags) { return (vsid << slb_vsid_shift(ssize)) | flags | ((unsigned long)ssize << SLB_VSID_SSIZE_SHIFT); } static inline unsigned long mk_vsid_data(unsigned long ea, int ssize, unsigned long flags) { return __mk_vsid_data(get_kernel_vsid(ea, ssize), ssize, flags); } #endif /* __ASSEMBLY__ */ #endif /* _ASM_POWERPC_BOOK3S_64_MMU_HASH_H_ */ PK ! ���߉ � pgtable-64k.hnu �[��� /* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_BOOK3S_64_PGTABLE_64K_H #define _ASM_POWERPC_BOOK3S_64_PGTABLE_64K_H #ifndef __ASSEMBLY__ #ifdef CONFIG_HUGETLB_PAGE /* * We have PGD_INDEX_SIZ = 12 and PTE_INDEX_SIZE = 8, so that we can have * 16GB hugepage pte in PGD and 16MB hugepage pte at PMD; * * Defined in such a way that we can optimize away code block at build time * if CONFIG_HUGETLB_PAGE=n. * * returns true for pmd migration entries, THP, devmap, hugetlb * But compile time dependent on CONFIG_HUGETLB_PAGE */ static inline int pmd_huge(pmd_t pmd) { /* * leaf pte for huge page */ return !!(pmd_raw(pmd) & cpu_to_be64(_PAGE_PTE)); } static inline int pud_huge(pud_t pud) { /* * leaf pte for huge page */ return !!(pud_raw(pud) & cpu_to_be64(_PAGE_PTE)); } static inline int pgd_huge(pgd_t pgd) { /* * leaf pte for huge page */ return !!(pgd_raw(pgd) & cpu_to_be64(_PAGE_PTE)); } #define pgd_huge pgd_huge /* * With 64k page size, we have hugepage ptes in the pgd and pmd entries. We don't * need to setup hugepage directory for them. Our pte and page directory format * enable us to have this enabled. */ static inline int hugepd_ok(hugepd_t hpd) { return 0; } #define is_hugepd(pdep) 0 /* * This should never get called */ static inline int get_hugepd_cache_index(int index) { BUG(); } #endif /* CONFIG_HUGETLB_PAGE */ static inline int remap_4k_pfn(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, pgprot_t prot) { if (radix_enabled()) BUG(); return hash__remap_4k_pfn(vma, addr, pfn, prot); } #endif /* __ASSEMBLY__ */ #endif /*_ASM_POWERPC_BOOK3S_64_PGTABLE_64K_H */ PK ! ���� � tlbflush-hash.hnu �[��� /* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_BOOK3S_64_TLBFLUSH_HASH_H #define _ASM_POWERPC_BOOK3S_64_TLBFLUSH_HASH_H /* * TLB flushing for 64-bit hash-MMU CPUs */ #include <linux/percpu.h> #include <asm/page.h> #define PPC64_TLB_BATCH_NR 192 struct ppc64_tlb_batch { int active; unsigned long index; struct mm_struct *mm; real_pte_t pte[PPC64_TLB_BATCH_NR]; unsigned long vpn[PPC64_TLB_BATCH_NR]; unsigned int psize; int ssize; }; DECLARE_PER_CPU(struct ppc64_tlb_batch, ppc64_tlb_batch); extern void __flush_tlb_pending(struct ppc64_tlb_batch *batch); #define __HAVE_ARCH_ENTER_LAZY_MMU_MODE static inline void arch_enter_lazy_mmu_mode(void) { struct ppc64_tlb_batch *batch; if (radix_enabled()) return; batch = this_cpu_ptr(&ppc64_tlb_batch); batch->active = 1; } static inline void arch_leave_lazy_mmu_mode(void) { struct ppc64_tlb_batch *batch; if (radix_enabled()) return; batch = this_cpu_ptr(&ppc64_tlb_batch); if (batch->index) __flush_tlb_pending(batch); batch->active = 0; } #define arch_flush_lazy_mmu_mode() do {} while (0) extern void hash__tlbiel_all(unsigned int action); extern void flush_hash_page(unsigned long vpn, real_pte_t pte, int psize, int ssize, unsigned long flags); extern void flush_hash_range(unsigned long number, int local); extern void flush_hash_hugepage(unsigned long vsid, unsigned long addr, pmd_t *pmdp, unsigned int psize, int ssize, unsigned long flags); static inline void hash__local_flush_tlb_mm(struct mm_struct *mm) { } static inline void hash__flush_tlb_mm(struct mm_struct *mm) { } static inline void hash__local_flush_all_mm(struct mm_struct *mm) { /* * There's no Page Walk Cache for hash, so what is needed is * the same as flush_tlb_mm(), which doesn't really make sense * with hash. So the only thing we could do is flush the * entire LPID! Punt for now, as it's not being used. */ WARN_ON_ONCE(1); } static inline void hash__flush_all_mm(struct mm_struct *mm) { /* * There's no Page Walk Cache for hash, so what is needed is * the same as flush_tlb_mm(), which doesn't really make sense * with hash. So the only thing we could do is flush the * entire LPID! Punt for now, as it's not being used. */ WARN_ON_ONCE(1); } static inline void hash__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr) { } static inline void hash__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr) { } static inline void hash__flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { } static inline void hash__flush_tlb_kernel_range(unsigned long start, unsigned long end) { } struct mmu_gather; extern void hash__tlb_flush(struct mmu_gather *tlb); /* Private function for use by PCI IO mapping code */ extern void __flush_hash_table_range(unsigned long start, unsigned long end); extern void flush_tlb_pmd_range(struct mm_struct *mm, pmd_t *pmd, unsigned long addr); #endif /* _ASM_POWERPC_BOOK3S_64_TLBFLUSH_HASH_H */ PK ! ;��' �' radix.hnu �[��� /* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_PGTABLE_RADIX_H #define _ASM_POWERPC_PGTABLE_RADIX_H #include <asm/asm-const.h> #ifndef __ASSEMBLY__ #include <asm/cmpxchg.h> #endif #ifdef CONFIG_PPC_64K_PAGES #include <asm/book3s/64/radix-64k.h> #else #include <asm/book3s/64/radix-4k.h> #endif #ifndef __ASSEMBLY__ #include <asm/book3s/64/tlbflush-radix.h> #include <asm/cpu_has_feature.h> #endif /* An empty PTE can still have a R or C writeback */ #define RADIX_PTE_NONE_MASK (_PAGE_DIRTY | _PAGE_ACCESSED) /* Bits to set in a RPMD/RPUD/RPGD */ #define RADIX_PMD_VAL_BITS (0x8000000000000000UL | RADIX_PTE_INDEX_SIZE) #define RADIX_PUD_VAL_BITS (0x8000000000000000UL | RADIX_PMD_INDEX_SIZE) #define RADIX_PGD_VAL_BITS (0x8000000000000000UL | RADIX_PUD_INDEX_SIZE) /* Don't have anything in the reserved bits and leaf bits */ #define RADIX_PMD_BAD_BITS 0x60000000000000e0UL #define RADIX_PUD_BAD_BITS 0x60000000000000e0UL #define RADIX_P4D_BAD_BITS 0x60000000000000e0UL #define RADIX_PMD_SHIFT (PAGE_SHIFT + RADIX_PTE_INDEX_SIZE) #define RADIX_PUD_SHIFT (RADIX_PMD_SHIFT + RADIX_PMD_INDEX_SIZE) #define RADIX_PGD_SHIFT (RADIX_PUD_SHIFT + RADIX_PUD_INDEX_SIZE) /* * Size of EA range mapped by our pagetables. */ #define RADIX_PGTABLE_EADDR_SIZE (RADIX_PTE_INDEX_SIZE + RADIX_PMD_INDEX_SIZE + \ RADIX_PUD_INDEX_SIZE + RADIX_PGD_INDEX_SIZE + PAGE_SHIFT) #define RADIX_PGTABLE_RANGE (ASM_CONST(1) << RADIX_PGTABLE_EADDR_SIZE) /* * We support 52 bit address space, Use top bit for kernel * virtual mapping. Also make sure kernel fit in the top * quadrant. * * +------------------+ * +------------------+ Kernel virtual map (0xc008000000000000) * | | * | | * | | * 0b11......+------------------+ Kernel linear map (0xc....) * | | * | 2 quadrant | * | | * 0b10......+------------------+ * | | * | 1 quadrant | * | | * 0b01......+------------------+ * | | * | 0 quadrant | * | | * 0b00......+------------------+ * * * 3rd quadrant expanded: * +------------------------------+ * | | * | | * | | * +------------------------------+ Kernel vmemmap end (0xc010000000000000) * | | * | 512TB | * | | * +------------------------------+ Kernel IO map end/vmemap start * | | * | 512TB | * | | * +------------------------------+ Kernel vmap end/ IO map start * | | * | 512TB | * | | * +------------------------------+ Kernel virt start (0xc008000000000000) * | | * | | * | | * +------------------------------+ Kernel linear (0xc.....) */ /* * If we store section details in page->flags we can't increase the MAX_PHYSMEM_BITS * if we increase SECTIONS_WIDTH we will not store node details in page->flags and * page_to_nid does a page->section->node lookup * Hence only increase for VMEMMAP. Further depending on SPARSEMEM_EXTREME reduce * memory requirements with large number of sections. * 51 bits is the max physical real address on POWER9 */ #if defined(CONFIG_SPARSEMEM_VMEMMAP) && defined(CONFIG_SPARSEMEM_EXTREME) #define R_MAX_PHYSMEM_BITS 51 #else #define R_MAX_PHYSMEM_BITS 46 #endif #define RADIX_KERN_VIRT_START ASM_CONST(0xc008000000000000) /* * 49 = MAX_EA_BITS_PER_CONTEXT (hash specific). To make sure we pick * the same value as hash. */ #define RADIX_KERN_MAP_SIZE (1UL << 49) #define RADIX_VMALLOC_START RADIX_KERN_VIRT_START #define RADIX_VMALLOC_SIZE RADIX_KERN_MAP_SIZE #define RADIX_VMALLOC_END (RADIX_VMALLOC_START + RADIX_VMALLOC_SIZE) #define RADIX_KERN_IO_START RADIX_VMALLOC_END #define RADIX_KERN_IO_SIZE RADIX_KERN_MAP_SIZE #define RADIX_KERN_IO_END (RADIX_KERN_IO_START + RADIX_KERN_IO_SIZE) #define RADIX_VMEMMAP_START RADIX_KERN_IO_END #define RADIX_VMEMMAP_SIZE RADIX_KERN_MAP_SIZE #define RADIX_VMEMMAP_END (RADIX_VMEMMAP_START + RADIX_VMEMMAP_SIZE) #ifndef __ASSEMBLY__ #define RADIX_PTE_TABLE_SIZE (sizeof(pte_t) << RADIX_PTE_INDEX_SIZE) #define RADIX_PMD_TABLE_SIZE (sizeof(pmd_t) << RADIX_PMD_INDEX_SIZE) #define RADIX_PUD_TABLE_SIZE (sizeof(pud_t) << RADIX_PUD_INDEX_SIZE) #define RADIX_PGD_TABLE_SIZE (sizeof(pgd_t) << RADIX_PGD_INDEX_SIZE) #ifdef CONFIG_STRICT_KERNEL_RWX extern void radix__mark_rodata_ro(void); extern void radix__mark_initmem_nx(void); #endif extern void radix__ptep_set_access_flags(struct vm_area_struct *vma, pte_t *ptep, pte_t entry, unsigned long address, int psize); extern void radix__ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, pte_t old_pte, pte_t pte); static inline unsigned long __radix_pte_update(pte_t *ptep, unsigned long clr, unsigned long set) { __be64 old_be, tmp_be; __asm__ __volatile__( "1: ldarx %0,0,%3 # pte_update\n" " andc %1,%0,%5 \n" " or %1,%1,%4 \n" " stdcx. %1,0,%3 \n" " bne- 1b" : "=&r" (old_be), "=&r" (tmp_be), "=m" (*ptep) : "r" (ptep), "r" (cpu_to_be64(set)), "r" (cpu_to_be64(clr)) : "cc" ); return be64_to_cpu(old_be); } static inline unsigned long radix__pte_update(struct mm_struct *mm, unsigned long addr, pte_t *ptep, unsigned long clr, unsigned long set, int huge) { unsigned long old_pte; old_pte = __radix_pte_update(ptep, clr, set); if (!huge) assert_pte_locked(mm, addr); return old_pte; } static inline pte_t radix__ptep_get_and_clear_full(struct mm_struct *mm, unsigned long addr, pte_t *ptep, int full) { unsigned long old_pte; if (full) { old_pte = pte_val(*ptep); *ptep = __pte(0); } else old_pte = radix__pte_update(mm, addr, ptep, ~0ul, 0, 0); return __pte(old_pte); } static inline int radix__pte_same(pte_t pte_a, pte_t pte_b) { return ((pte_raw(pte_a) ^ pte_raw(pte_b)) == 0); } static inline int radix__pte_none(pte_t pte) { return (pte_val(pte) & ~RADIX_PTE_NONE_MASK) == 0; } static inline void radix__set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte, int percpu) { *ptep = pte; /* * The architecture suggests a ptesync after setting the pte, which * orders the store that updates the pte with subsequent page table * walk accesses which may load the pte. Without this it may be * possible for a subsequent access to result in spurious fault. * * This is not necessary for correctness, because a spurious fault * is tolerated by the page fault handler, and this store will * eventually be seen. In testing, there was no noticable increase * in user faults on POWER9. Avoiding ptesync here is a significant * win for things like fork. If a future microarchitecture benefits * from ptesync, it should probably go into update_mmu_cache, rather * than set_pte_at (which is used to set ptes unrelated to faults). * * Spurious faults from the kernel memory are not tolerated, so there * is a ptesync in flush_cache_vmap, and __map_kernel_page() follows * the pte update sequence from ISA Book III 6.10 Translation Table * Update Synchronization Requirements. */ } static inline int radix__pmd_bad(pmd_t pmd) { return !!(pmd_val(pmd) & RADIX_PMD_BAD_BITS); } static inline int radix__pmd_same(pmd_t pmd_a, pmd_t pmd_b) { return ((pmd_raw(pmd_a) ^ pmd_raw(pmd_b)) == 0); } static inline int radix__pud_bad(pud_t pud) { return !!(pud_val(pud) & RADIX_PUD_BAD_BITS); } static inline int radix__p4d_bad(p4d_t p4d) { return !!(p4d_val(p4d) & RADIX_P4D_BAD_BITS); } #ifdef CONFIG_TRANSPARENT_HUGEPAGE static inline int radix__pmd_trans_huge(pmd_t pmd) { return (pmd_val(pmd) & (_PAGE_PTE | _PAGE_DEVMAP)) == _PAGE_PTE; } static inline pmd_t radix__pmd_mkhuge(pmd_t pmd) { return __pmd(pmd_val(pmd) | _PAGE_PTE); } extern unsigned long radix__pmd_hugepage_update(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, unsigned long clr, unsigned long set); extern pmd_t radix__pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp); extern void radix__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, pgtable_t pgtable); extern pgtable_t radix__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp); extern pmd_t radix__pmdp_huge_get_and_clear(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp); static inline int radix__has_transparent_hugepage(void) { /* For radix 2M at PMD level means thp */ if (mmu_psize_defs[MMU_PAGE_2M].shift == PMD_SHIFT) return 1; return 0; } #endif static inline pmd_t radix__pmd_mkdevmap(pmd_t pmd) { return __pmd(pmd_val(pmd) | (_PAGE_PTE | _PAGE_DEVMAP)); } extern int __meminit radix__vmemmap_create_mapping(unsigned long start, unsigned long page_size, unsigned long phys); extern void radix__vmemmap_remove_mapping(unsigned long start, unsigned long page_size); extern int radix__map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t flags, unsigned int psz); static inline unsigned long radix__get_tree_size(void) { unsigned long rts_field; /* * We support 52 bits, hence: * bits 52 - 31 = 21, 0b10101 * RTS encoding details * bits 0 - 3 of rts -> bits 6 - 8 unsigned long * bits 4 - 5 of rts -> bits 62 - 63 of unsigned long */ rts_field = (0x5UL << 5); /* 6 - 8 bits */ rts_field |= (0x2UL << 61); return rts_field; } #ifdef CONFIG_MEMORY_HOTPLUG int radix__create_section_mapping(unsigned long start, unsigned long end, int nid, pgprot_t prot); int radix__remove_section_mapping(unsigned long start, unsigned long end); #endif /* CONFIG_MEMORY_HOTPLUG */ #endif /* __ASSEMBLY__ */ #endif PK ! �ɼ* * tlbflush-radix.hnu �[��� /* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_TLBFLUSH_RADIX_H #define _ASM_POWERPC_TLBFLUSH_RADIX_H #include <asm/hvcall.h> #define RIC_FLUSH_TLB 0 #define RIC_FLUSH_PWC 1 #define RIC_FLUSH_ALL 2 struct vm_area_struct; struct mm_struct; struct mmu_gather; static inline u64 psize_to_rpti_pgsize(unsigned long psize) { if (psize == MMU_PAGE_4K) return H_RPTI_PAGE_4K; if (psize == MMU_PAGE_64K) return H_RPTI_PAGE_64K; if (psize == MMU_PAGE_2M) return H_RPTI_PAGE_2M; if (psize == MMU_PAGE_1G) return H_RPTI_PAGE_1G; return H_RPTI_PAGE_ALL; } static inline int mmu_get_ap(int psize) { return mmu_psize_defs[psize].ap; } #ifdef CONFIG_PPC_RADIX_MMU extern void radix__tlbiel_all(unsigned int action); extern void radix__flush_tlb_lpid_page(unsigned int lpid, unsigned long addr, unsigned long page_size); extern void radix__flush_pwc_lpid(unsigned int lpid); extern void radix__flush_all_lpid(unsigned int lpid); extern void radix__flush_all_lpid_guest(unsigned int lpid); #else static inline void radix__tlbiel_all(unsigned int action) { WARN_ON(1); } static inline void radix__flush_tlb_lpid_page(unsigned int lpid, unsigned long addr, unsigned long page_size) { WARN_ON(1); } static inline void radix__flush_pwc_lpid(unsigned int lpid) { WARN_ON(1); } static inline void radix__flush_all_lpid(unsigned int lpid) { WARN_ON(1); } static inline void radix__flush_all_lpid_guest(unsigned int lpid) { WARN_ON(1); } #endif extern void radix__flush_hugetlb_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); extern void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start, unsigned long end, int psize); void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start, unsigned long end, int psize); extern void radix__flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); extern void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); extern void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end); extern void radix__local_flush_tlb_mm(struct mm_struct *mm); extern void radix__local_flush_all_mm(struct mm_struct *mm); extern void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr); extern void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr, int psize); extern void radix__tlb_flush(struct mmu_gather *tlb); #ifdef CONFIG_SMP extern void radix__flush_tlb_mm(struct mm_struct *mm); extern void radix__flush_all_mm(struct mm_struct *mm); extern void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr); extern void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr, int psize); #else #define radix__flush_tlb_mm(mm) radix__local_flush_tlb_mm(mm) #define radix__flush_all_mm(mm) radix__local_flush_all_mm(mm) #define radix__flush_tlb_page(vma,addr) radix__local_flush_tlb_page(vma,addr) #define radix__flush_tlb_page_psize(mm,addr,p) radix__local_flush_tlb_page_psize(mm,addr,p) #endif extern void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr); extern void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr); extern void radix__flush_tlb_all(void); #endif PK ! ^�y tlbflush.hnu �[��� /* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_BOOK3S_64_TLBFLUSH_H #define _ASM_POWERPC_BOOK3S_64_TLBFLUSH_H #define MMU_NO_CONTEXT ~0UL #include <linux/mm_types.h> #include <asm/book3s/64/tlbflush-hash.h> #include <asm/book3s/64/tlbflush-radix.h> /* TLB flush actions. Used as argument to tlbiel_all() */ enum { TLB_INVAL_SCOPE_GLOBAL = 0, /* invalidate all TLBs */ TLB_INVAL_SCOPE_LPID = 1, /* invalidate TLBs for current LPID */ }; #ifdef CONFIG_PPC_NATIVE static inline void tlbiel_all(void) { /* * This is used for host machine check and bootup. * * This uses early_radix_enabled and implementations use * early_cpu_has_feature etc because that works early in boot * and this is the machine check path which is not performance * critical. */ if (early_radix_enabled()) radix__tlbiel_all(TLB_INVAL_SCOPE_GLOBAL); else hash__tlbiel_all(TLB_INVAL_SCOPE_GLOBAL); } #else static inline void tlbiel_all(void) { BUG(); } #endif static inline void tlbiel_all_lpid(bool radix) { /* * This is used for guest machine check. */ if (radix) radix__tlbiel_all(TLB_INVAL_SCOPE_LPID); else hash__tlbiel_all(TLB_INVAL_SCOPE_LPID); } #define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE static inline void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { if (radix_enabled()) return radix__flush_pmd_tlb_range(vma, start, end); return hash__flush_tlb_range(vma, start, end); } #define __HAVE_ARCH_FLUSH_HUGETLB_TLB_RANGE static inline void flush_hugetlb_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { if (radix_enabled()) return radix__flush_hugetlb_tlb_range(vma, start, end); return hash__flush_tlb_range(vma, start, end); } static inline void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { if (radix_enabled()) return radix__flush_tlb_range(vma, start, end); return hash__flush_tlb_range(vma, start, end); } static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end) { if (radix_enabled()) return radix__flush_tlb_kernel_range(start, end); return hash__flush_tlb_kernel_range(start, end); } static inline void local_flush_tlb_mm(struct mm_struct *mm) { if (radix_enabled()) return radix__local_flush_tlb_mm(mm); return hash__local_flush_tlb_mm(mm); } static inline void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr) { if (radix_enabled()) return radix__local_flush_tlb_page(vma, vmaddr); return hash__local_flush_tlb_page(vma, vmaddr); } static inline void local_flush_all_mm(struct mm_struct *mm) { if (radix_enabled()) return radix__local_flush_all_mm(mm); return hash__local_flush_all_mm(mm); } static inline void tlb_flush(struct mmu_gather *tlb) { if (radix_enabled()) return radix__tlb_flush(tlb); return hash__tlb_flush(tlb); } #ifdef CONFIG_SMP static inline void flush_tlb_mm(struct mm_struct *mm) { if (radix_enabled()) return radix__flush_tlb_mm(mm); return hash__flush_tlb_mm(mm); } static inline void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr) { if (radix_enabled()) return radix__flush_tlb_page(vma, vmaddr); return hash__flush_tlb_page(vma, vmaddr); } static inline void flush_all_mm(struct mm_struct *mm) { if (radix_enabled()) return radix__flush_all_mm(mm); return hash__flush_all_mm(mm); } #else #define flush_tlb_mm(mm) local_flush_tlb_mm(mm) #define flush_tlb_page(vma, addr) local_flush_tlb_page(vma, addr) #define flush_all_mm(mm) local_flush_all_mm(mm) #endif /* CONFIG_SMP */ #define flush_tlb_fix_spurious_fault flush_tlb_fix_spurious_fault static inline void flush_tlb_fix_spurious_fault(struct vm_area_struct *vma, unsigned long address) { /* See ptep_set_access_flags comment */ if (atomic_read(&vma->vm_mm->context.copros) > 0) flush_tlb_page(vma, address); } extern bool tlbie_capable; extern bool tlbie_enabled; static inline bool cputlb_use_tlbie(void) { return tlbie_enabled; } #endif /* _ASM_POWERPC_BOOK3S_64_TLBFLUSH_H */ PK ! �PY Y radix-4k.hnu �[��� /* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_PGTABLE_RADIX_4K_H #define _ASM_POWERPC_PGTABLE_RADIX_4K_H /* * For 4K page size supported index is 13/9/9/9 */ #define RADIX_PTE_INDEX_SIZE 9 // size: 8B << 9 = 4KB, maps 2^9 x 4K = 2MB #define RADIX_PMD_INDEX_SIZE 9 // size: 8B << 9 = 4KB, maps 2^9 x 2MB = 1GB #define RADIX_PUD_INDEX_SIZE 9 // size: 8B << 9 = 4KB, maps 2^9 x 1GB = 512GB #define RADIX_PGD_INDEX_SIZE 13 // size: 8B << 13 = 64KB, maps 2^13 x 512GB = 4PB /* * One fragment per page */ #define RADIX_PTE_FRAG_SIZE_SHIFT (RADIX_PTE_INDEX_SIZE + 3) #define RADIX_PTE_FRAG_NR (PAGE_SIZE >> RADIX_PTE_FRAG_SIZE_SHIFT) #define RADIX_PMD_FRAG_SIZE_SHIFT (RADIX_PMD_INDEX_SIZE + 3) #define RADIX_PMD_FRAG_NR (PAGE_SIZE >> RADIX_PMD_FRAG_SIZE_SHIFT) #endif /* _ASM_POWERPC_PGTABLE_RADIX_4K_H */ PK ! ��f�3 3 slice.hnu �[��� /* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_BOOK3S_64_SLICE_H #define _ASM_POWERPC_BOOK3S_64_SLICE_H #define SLICE_LOW_SHIFT 28 #define SLICE_LOW_TOP (0x100000000ul) #define SLICE_NUM_LOW (SLICE_LOW_TOP >> SLICE_LOW_SHIFT) #define GET_LOW_SLICE_INDEX(addr) ((addr) >> SLICE_LOW_SHIFT) #define SLICE_HIGH_SHIFT 40 #define SLICE_NUM_HIGH (H_PGTABLE_RANGE >> SLICE_HIGH_SHIFT) #define GET_HIGH_SLICE_INDEX(addr) ((addr) >> SLICE_HIGH_SHIFT) #define SLB_ADDR_LIMIT_DEFAULT DEFAULT_MAP_WINDOW_USER64 #endif /* _ASM_POWERPC_BOOK3S_64_SLICE_H */ PK ! ~O�t pkeys.hnu �[��� /* SPDX-License-Identifier: GPL-2.0+ */ #ifndef _ASM_POWERPC_BOOK3S_64_PKEYS_H #define _ASM_POWERPC_BOOK3S_64_PKEYS_H #include <asm/book3s/64/hash-pkey.h> static inline u64 vmflag_to_pte_pkey_bits(u64 vm_flags) { if (!mmu_has_feature(MMU_FTR_PKEY)) return 0x0UL; if (radix_enabled()) BUG(); return hash__vmflag_to_pte_pkey_bits(vm_flags); } static inline u16 pte_to_pkey_bits(u64 pteflags) { if (radix_enabled()) BUG(); return hash__pte_to_pkey_bits(pteflags); } #endif /*_ASM_POWERPC_KEYS_H */ PK ! �� }* * hash.hnu �[��� /* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_BOOK3S_64_HASH_H #define _ASM_POWERPC_BOOK3S_64_HASH_H #ifdef __KERNEL__ #include <asm/asm-const.h> /* * Common bits between 4K and 64K pages in a linux-style PTE. * Additional bits may be defined in pgtable-hash64-*.h * */ #define H_PTE_NONE_MASK _PAGE_HPTEFLAGS #ifdef CONFIG_PPC_64K_PAGES #include <asm/book3s/64/hash-64k.h> #else #include <asm/book3s/64/hash-4k.h> #endif /* Bits to set in a PMD/PUD/PGD entry valid bit*/ #define HASH_PMD_VAL_BITS (0x8000000000000000UL) #define HASH_PUD_VAL_BITS (0x8000000000000000UL) #define HASH_PGD_VAL_BITS (0x8000000000000000UL) /* * Size of EA range mapped by our pagetables. */ #define H_PGTABLE_EADDR_SIZE (H_PTE_INDEX_SIZE + H_PMD_INDEX_SIZE + \ H_PUD_INDEX_SIZE + H_PGD_INDEX_SIZE + PAGE_SHIFT) #define H_PGTABLE_RANGE (ASM_CONST(1) << H_PGTABLE_EADDR_SIZE) /* * Top 2 bits are ignored in page table walk. */ #define EA_MASK (~(0xcUL << 60)) /* * We store the slot details in the second half of page table. * Increase the pud level table so that hugetlb ptes can be stored * at pud level. */ #if defined(CONFIG_HUGETLB_PAGE) && defined(CONFIG_PPC_64K_PAGES) #define H_PUD_CACHE_INDEX (H_PUD_INDEX_SIZE + 1) #else #define H_PUD_CACHE_INDEX (H_PUD_INDEX_SIZE) #endif /* * +------------------------------+ * | | * | | * | | * +------------------------------+ Kernel virtual map end (0xc00e000000000000) * | | * | | * | 512TB/16TB of vmemmap | * | | * | | * +------------------------------+ Kernel vmemmap start * | | * | 512TB/16TB of IO map | * | | * +------------------------------+ Kernel IO map start * | | * | 512TB/16TB of vmap | * | | * +------------------------------+ Kernel virt start (0xc008000000000000) * | | * | | * | | * +------------------------------+ Kernel linear (0xc.....) */ #define H_VMALLOC_START H_KERN_VIRT_START #define H_VMALLOC_SIZE H_KERN_MAP_SIZE #define H_VMALLOC_END (H_VMALLOC_START + H_VMALLOC_SIZE) #define H_KERN_IO_START H_VMALLOC_END #define H_KERN_IO_SIZE H_KERN_MAP_SIZE #define H_KERN_IO_END (H_KERN_IO_START + H_KERN_IO_SIZE) #define H_VMEMMAP_START H_KERN_IO_END #define H_VMEMMAP_SIZE H_KERN_MAP_SIZE #define H_VMEMMAP_END (H_VMEMMAP_START + H_VMEMMAP_SIZE) #define NON_LINEAR_REGION_ID(ea) ((((unsigned long)ea - H_KERN_VIRT_START) >> REGION_SHIFT) + 2) /* * Region IDs */ #define USER_REGION_ID 0 #define LINEAR_MAP_REGION_ID 1 #define VMALLOC_REGION_ID NON_LINEAR_REGION_ID(H_VMALLOC_START) #define IO_REGION_ID NON_LINEAR_REGION_ID(H_KERN_IO_START) #define VMEMMAP_REGION_ID NON_LINEAR_REGION_ID(H_VMEMMAP_START) #define INVALID_REGION_ID (VMEMMAP_REGION_ID + 1) /* * Defines the address of the vmemap area, in its own region on * hash table CPUs. */ #ifdef CONFIG_PPC_MM_SLICES #define HAVE_ARCH_UNMAPPED_AREA #define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN #endif /* CONFIG_PPC_MM_SLICES */ /* PTEIDX nibble */ #define _PTEIDX_SECONDARY 0x8 #define _PTEIDX_GROUP_IX 0x7 #define H_PMD_BAD_BITS (PTE_TABLE_SIZE-1) #define H_PUD_BAD_BITS (PMD_TABLE_SIZE-1) #ifndef __ASSEMBLY__ static inline int get_region_id(unsigned long ea) { int region_id; int id = (ea >> 60UL); if (id == 0) return USER_REGION_ID; if (id != (PAGE_OFFSET >> 60)) return INVALID_REGION_ID; if (ea < H_KERN_VIRT_START) return LINEAR_MAP_REGION_ID; BUILD_BUG_ON(NON_LINEAR_REGION_ID(H_VMALLOC_START) != 2); region_id = NON_LINEAR_REGION_ID(ea); return region_id; } #define hash__pmd_bad(pmd) (pmd_val(pmd) & H_PMD_BAD_BITS) #define hash__pud_bad(pud) (pud_val(pud) & H_PUD_BAD_BITS) static inline int hash__p4d_bad(p4d_t p4d) { return (p4d_val(p4d) == 0); } #ifdef CONFIG_STRICT_KERNEL_RWX extern void hash__mark_rodata_ro(void); extern void hash__mark_initmem_nx(void); #endif extern void hpte_need_flush(struct mm_struct *mm, unsigned long addr, pte_t *ptep, unsigned long pte, int huge); unsigned long htab_convert_pte_flags(unsigned long pteflags, unsigned long flags); /* Atomic PTE updates */ static inline unsigned long hash__pte_update(struct mm_struct *mm, unsigned long addr, pte_t *ptep, unsigned long clr, unsigned long set, int huge) { __be64 old_be, tmp_be; unsigned long old; __asm__ __volatile__( "1: ldarx %0,0,%3 # pte_update\n\ and. %1,%0,%6\n\ bne- 1b \n\ andc %1,%0,%4 \n\ or %1,%1,%7\n\ stdcx. %1,0,%3 \n\ bne- 1b" : "=&r" (old_be), "=&r" (tmp_be), "=m" (*ptep) : "r" (ptep), "r" (cpu_to_be64(clr)), "m" (*ptep), "r" (cpu_to_be64(H_PAGE_BUSY)), "r" (cpu_to_be64(set)) : "cc" ); /* huge pages use the old page table lock */ if (!huge) assert_pte_locked(mm, addr); old = be64_to_cpu(old_be); if (old & H_PAGE_HASHPTE) hpte_need_flush(mm, addr, ptep, old, huge); return old; } /* Set the dirty and/or accessed bits atomically in a linux PTE, this * function doesn't need to flush the hash entry */ static inline void hash__ptep_set_access_flags(pte_t *ptep, pte_t entry) { __be64 old, tmp, val, mask; mask = cpu_to_be64(_PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_READ | _PAGE_WRITE | _PAGE_EXEC | _PAGE_SOFT_DIRTY); val = pte_raw(entry) & mask; __asm__ __volatile__( "1: ldarx %0,0,%4\n\ and. %1,%0,%6\n\ bne- 1b \n\ or %0,%3,%0\n\ stdcx. %0,0,%4\n\ bne- 1b" :"=&r" (old), "=&r" (tmp), "=m" (*ptep) :"r" (val), "r" (ptep), "m" (*ptep), "r" (cpu_to_be64(H_PAGE_BUSY)) :"cc"); } static inline int hash__pte_same(pte_t pte_a, pte_t pte_b) { return (((pte_raw(pte_a) ^ pte_raw(pte_b)) & ~cpu_to_be64(_PAGE_HPTEFLAGS)) == 0); } static inline int hash__pte_none(pte_t pte) { return (pte_val(pte) & ~H_PTE_NONE_MASK) == 0; } unsigned long pte_get_hash_gslot(unsigned long vpn, unsigned long shift, int ssize, real_pte_t rpte, unsigned int subpg_index); /* This low level function performs the actual PTE insertion * Setting the PTE depends on the MMU type and other factors. It's * an horrible mess that I'm not going to try to clean up now but * I'm keeping it in one place rather than spread around */ static inline void hash__set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte, int percpu) { /* * Anything else just stores the PTE normally. That covers all 64-bit * cases, and 32-bit non-hash with 32-bit PTEs. */ *ptep = pte; } #ifdef CONFIG_TRANSPARENT_HUGEPAGE extern void hpte_do_hugepage_flush(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, unsigned long old_pmd); #else static inline void hpte_do_hugepage_flush(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, unsigned long old_pmd) { WARN(1, "%s called with THP disabled\n", __func__); } #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ int hash__map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot); extern int __meminit hash__vmemmap_create_mapping(unsigned long start, unsigned long page_size, unsigned long phys); extern void hash__vmemmap_remove_mapping(unsigned long start, unsigned long page_size); int hash__create_section_mapping(unsigned long start, unsigned long end, int nid, pgprot_t prot); int hash__remove_section_mapping(unsigned long start, unsigned long end); #endif /* !__ASSEMBLY__ */ #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_BOOK3S_64_HASH_H */ PK ! O0I� � radix-64k.hnu �[��� /* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_PGTABLE_RADIX_64K_H #define _ASM_POWERPC_PGTABLE_RADIX_64K_H /* * For 64K page size supported index is 13/9/9/5 */ #define RADIX_PTE_INDEX_SIZE 5 // size: 8B << 5 = 256B, maps 2^5 x 64K = 2MB #define RADIX_PMD_INDEX_SIZE 9 // size: 8B << 9 = 4KB, maps 2^9 x 2MB = 1GB #define RADIX_PUD_INDEX_SIZE 9 // size: 8B << 9 = 4KB, maps 2^9 x 1GB = 512GB #define RADIX_PGD_INDEX_SIZE 13 // size: 8B << 13 = 64KB, maps 2^13 x 512GB = 4PB /* * We use a 256 byte PTE page fragment in radix * 8 bytes per each PTE entry. */ #define RADIX_PTE_FRAG_SIZE_SHIFT (RADIX_PTE_INDEX_SIZE + 3) #define RADIX_PTE_FRAG_NR (PAGE_SIZE >> RADIX_PTE_FRAG_SIZE_SHIFT) #define RADIX_PMD_FRAG_SIZE_SHIFT (RADIX_PMD_INDEX_SIZE + 3) #define RADIX_PMD_FRAG_NR (PAGE_SIZE >> RADIX_PMD_FRAG_SIZE_SHIFT) #endif /* _ASM_POWERPC_PGTABLE_RADIX_64K_H */ PK ! �|C� � pgtable-4k.hnu �[��� /* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_BOOK3S_64_PGTABLE_4K_H #define _ASM_POWERPC_BOOK3S_64_PGTABLE_4K_H /* * hash 4k can't share hugetlb and also doesn't support THP */ #ifndef __ASSEMBLY__ #ifdef CONFIG_HUGETLB_PAGE static inline int pmd_huge(pmd_t pmd) { /* * leaf pte for huge page */ if (radix_enabled()) return !!(pmd_raw(pmd) & cpu_to_be64(_PAGE_PTE)); return 0; } static inline int pud_huge(pud_t pud) { /* * leaf pte for huge page */ if (radix_enabled()) return !!(pud_raw(pud) & cpu_to_be64(_PAGE_PTE)); return 0; } static inline int pgd_huge(pgd_t pgd) { /* * leaf pte for huge page */ if (radix_enabled()) return !!(pgd_raw(pgd) & cpu_to_be64(_PAGE_PTE)); return 0; } #define pgd_huge pgd_huge /* * With radix , we have hugepage ptes in the pud and pmd entries. We don't * need to setup hugepage directory for them. Our pte and page directory format * enable us to have this enabled. */ static inline int hugepd_ok(hugepd_t hpd) { if (radix_enabled()) return 0; return hash__hugepd_ok(hpd); } #define is_hugepd(hpd) (hugepd_ok(hpd)) /* * 16M and 16G huge page directory tables are allocated from slab cache * */ #define H_16M_CACHE_INDEX (PAGE_SHIFT + H_PTE_INDEX_SIZE + H_PMD_INDEX_SIZE - 24) #define H_16G_CACHE_INDEX \ (PAGE_SHIFT + H_PTE_INDEX_SIZE + H_PMD_INDEX_SIZE + H_PUD_INDEX_SIZE - 34) static inline int get_hugepd_cache_index(int index) { switch (index) { case H_16M_CACHE_INDEX: return HTLB_16M_INDEX; case H_16G_CACHE_INDEX: return HTLB_16G_INDEX; default: BUG(); } /* should not reach */ } #endif /* CONFIG_HUGETLB_PAGE */ #endif /* __ASSEMBLY__ */ #endif /*_ASM_POWERPC_BOOK3S_64_PGTABLE_4K_H */ PK ! 4���| | hash-4k.hnu �[��� /* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_BOOK3S_64_HASH_4K_H #define _ASM_POWERPC_BOOK3S_64_HASH_4K_H #define H_PTE_INDEX_SIZE 9 // size: 8B << 9 = 4KB, maps: 2^9 x 4KB = 2MB #define H_PMD_INDEX_SIZE 7 // size: 8B << 7 = 1KB, maps: 2^7 x 2MB = 256MB #define H_PUD_INDEX_SIZE 9 // size: 8B << 9 = 4KB, maps: 2^9 x 256MB = 128GB #define H_PGD_INDEX_SIZE 9 // size: 8B << 9 = 4KB, maps: 2^9 x 128GB = 64TB /* * Each context is 512TB. But on 4k we restrict our max TASK size to 64TB * Hence also limit max EA bits to 64TB. */ #define MAX_EA_BITS_PER_CONTEXT 46 /* * Our page table limit us to 64TB. For 64TB physical memory, we only need 64GB * of vmemmap space. To better support sparse memory layout, we use 61TB * linear map range, 1TB of vmalloc, 1TB of I/O and 1TB of vmememmap. */ #define REGION_SHIFT (40) #define H_KERN_MAP_SIZE (ASM_CONST(1) << REGION_SHIFT) /* * Limits the linear mapping range */ #define H_MAX_PHYSMEM_BITS 46 /* * Define the address range of the kernel non-linear virtual area (61TB) */ #define H_KERN_VIRT_START ASM_CONST(0xc0003d0000000000) #ifndef __ASSEMBLY__ #define H_PTE_TABLE_SIZE (sizeof(pte_t) << H_PTE_INDEX_SIZE) #define H_PMD_TABLE_SIZE (sizeof(pmd_t) << H_PMD_INDEX_SIZE) #define H_PUD_TABLE_SIZE (sizeof(pud_t) << H_PUD_INDEX_SIZE) #define H_PGD_TABLE_SIZE (sizeof(pgd_t) << H_PGD_INDEX_SIZE) #define H_PAGE_F_GIX_SHIFT _PAGE_PA_MAX #define H_PAGE_F_SECOND _RPAGE_PKEY_BIT0 /* HPTE is in 2ndary HPTEG */ #define H_PAGE_F_GIX (_RPAGE_RPN43 | _RPAGE_RPN42 | _RPAGE_RPN41) #define H_PAGE_BUSY _RPAGE_RSV1 #define H_PAGE_HASHPTE _RPAGE_PKEY_BIT4 /* PTE flags to conserve for HPTE identification */ #define _PAGE_HPTEFLAGS (H_PAGE_BUSY | H_PAGE_HASHPTE | \ H_PAGE_F_SECOND | H_PAGE_F_GIX) /* * Not supported by 4k linux page size */ #define H_PAGE_4K_PFN 0x0 #define H_PAGE_THP_HUGE 0x0 #define H_PAGE_COMBO 0x0 /* 8 bytes per each pte entry */ #define H_PTE_FRAG_SIZE_SHIFT (H_PTE_INDEX_SIZE + 3) #define H_PTE_FRAG_NR (PAGE_SIZE >> H_PTE_FRAG_SIZE_SHIFT) #define H_PMD_FRAG_SIZE_SHIFT (H_PMD_INDEX_SIZE + 3) #define H_PMD_FRAG_NR (PAGE_SIZE >> H_PMD_FRAG_SIZE_SHIFT) /* memory key bits, only 8 keys supported */ #define H_PTE_PKEY_BIT4 0 #define H_PTE_PKEY_BIT3 0 #define H_PTE_PKEY_BIT2 _RPAGE_PKEY_BIT3 #define H_PTE_PKEY_BIT1 _RPAGE_PKEY_BIT2 #define H_PTE_PKEY_BIT0 _RPAGE_PKEY_BIT1 /* * On all 4K setups, remap_4k_pfn() equates to remap_pfn_range() */ #define remap_4k_pfn(vma, addr, pfn, prot) \ remap_pfn_range((vma), (addr), (pfn), PAGE_SIZE, (prot)) #ifdef CONFIG_HUGETLB_PAGE static inline int hash__hugepd_ok(hugepd_t hpd) { unsigned long hpdval = hpd_val(hpd); /* * if it is not a pte and have hugepd shift mask * set, then it is a hugepd directory pointer */ if (!(hpdval & _PAGE_PTE) && (hpdval & _PAGE_PRESENT) && ((hpdval & HUGEPD_SHIFT_MASK) != 0)) return true; return false; } #endif /* * With 4K page size the real_pte machinery is all nops. */ static inline real_pte_t __real_pte(pte_t pte, pte_t *ptep, int offset) { return (real_pte_t){pte}; } #define __rpte_to_pte(r) ((r).pte) static inline unsigned long __rpte_to_hidx(real_pte_t rpte, unsigned long index) { return pte_val(__rpte_to_pte(rpte)) >> H_PAGE_F_GIX_SHIFT; } #define pte_iterate_hashed_subpages(rpte, psize, va, index, shift) \ do { \ index = 0; \ shift = mmu_psize_defs[psize].shift; \ #define pte_iterate_hashed_end() } while(0) /* * We expect this to be called only for user addresses or kernel virtual * addresses other than the linear mapping. */ #define pte_pagesize_index(mm, addr, pte) MMU_PAGE_4K /* * 4K PTE format is different from 64K PTE format. Saving the hash_slot is just * a matter of returning the PTE bits that need to be modified. On 64K PTE, * things are a little more involved and hence needs many more parameters to * accomplish the same. However we want to abstract this out from the caller by * keeping the prototype consistent across the two formats. */ static inline unsigned long pte_set_hidx(pte_t *ptep, real_pte_t rpte, unsigned int subpg_index, unsigned long hidx, int offset) { return (hidx << H_PAGE_F_GIX_SHIFT) & (H_PAGE_F_SECOND | H_PAGE_F_GIX); } #ifdef CONFIG_TRANSPARENT_HUGEPAGE static inline char *get_hpte_slot_array(pmd_t *pmdp) { BUG(); return NULL; } static inline unsigned int hpte_valid(unsigned char *hpte_slot_array, int index) { BUG(); return 0; } static inline unsigned int hpte_hash_index(unsigned char *hpte_slot_array, int index) { BUG(); return 0; } static inline void mark_hpte_slot_valid(unsigned char *hpte_slot_array, unsigned int index, unsigned int hidx) { BUG(); } static inline int hash__pmd_trans_huge(pmd_t pmd) { return 0; } static inline int hash__pmd_same(pmd_t pmd_a, pmd_t pmd_b) { BUG(); return 0; } static inline pmd_t hash__pmd_mkhuge(pmd_t pmd) { BUG(); return pmd; } extern unsigned long hash__pmd_hugepage_update(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, unsigned long clr, unsigned long set); extern pmd_t hash__pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp); extern void hash__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, pgtable_t pgtable); extern pgtable_t hash__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp); extern pmd_t hash__pmdp_huge_get_and_clear(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp); extern int hash__has_transparent_hugepage(void); #endif static inline pmd_t hash__pmd_mkdevmap(pmd_t pmd) { BUG(); return pmd; } #endif /* !__ASSEMBLY__ */ #endif /* _ASM_POWERPC_BOOK3S_64_HASH_4K_H */ PK ! �f>�( �( hash-64k.hnu �[��� /* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_BOOK3S_64_HASH_64K_H #define _ASM_POWERPC_BOOK3S_64_HASH_64K_H #define H_PTE_INDEX_SIZE 8 // size: 8B << 8 = 2KB, maps 2^8 x 64KB = 16MB #define H_PMD_INDEX_SIZE 10 // size: 8B << 10 = 8KB, maps 2^10 x 16MB = 16GB #define H_PUD_INDEX_SIZE 10 // size: 8B << 10 = 8KB, maps 2^10 x 16GB = 16TB #define H_PGD_INDEX_SIZE 8 // size: 8B << 8 = 2KB, maps 2^8 x 16TB = 4PB /* * If we store section details in page->flags we can't increase the MAX_PHYSMEM_BITS * if we increase SECTIONS_WIDTH we will not store node details in page->flags and * page_to_nid does a page->section->node lookup * Hence only increase for VMEMMAP. Further depending on SPARSEMEM_EXTREME reduce * memory requirements with large number of sections. * 51 bits is the max physical real address on POWER9 */ #if defined(CONFIG_SPARSEMEM_VMEMMAP) && defined(CONFIG_SPARSEMEM_EXTREME) #define H_MAX_PHYSMEM_BITS 51 #else #define H_MAX_PHYSMEM_BITS 46 #endif /* * Each context is 512TB size. SLB miss for first context/default context * is handled in the hotpath. */ #define MAX_EA_BITS_PER_CONTEXT 49 #define REGION_SHIFT MAX_EA_BITS_PER_CONTEXT /* * We use one context for each MAP area. */ #define H_KERN_MAP_SIZE (1UL << MAX_EA_BITS_PER_CONTEXT) /* * Define the address range of the kernel non-linear virtual area * 2PB */ #define H_KERN_VIRT_START ASM_CONST(0xc008000000000000) /* * 64k aligned address free up few of the lower bits of RPN for us * We steal that here. For more deatils look at pte_pfn/pfn_pte() */ #define H_PAGE_COMBO _RPAGE_RPN0 /* this is a combo 4k page */ #define H_PAGE_4K_PFN _RPAGE_RPN1 /* PFN is for a single 4k page */ #define H_PAGE_BUSY _RPAGE_RSV1 /* software: PTE & hash are busy */ #define H_PAGE_HASHPTE _RPAGE_RPN43 /* PTE has associated HPTE */ /* memory key bits. */ #define H_PTE_PKEY_BIT4 _RPAGE_PKEY_BIT4 #define H_PTE_PKEY_BIT3 _RPAGE_PKEY_BIT3 #define H_PTE_PKEY_BIT2 _RPAGE_PKEY_BIT2 #define H_PTE_PKEY_BIT1 _RPAGE_PKEY_BIT1 #define H_PTE_PKEY_BIT0 _RPAGE_PKEY_BIT0 /* * We need to differentiate between explicit huge page and THP huge * page, since THP huge page also need to track real subpage details */ #define H_PAGE_THP_HUGE H_PAGE_4K_PFN /* PTE flags to conserve for HPTE identification */ #define _PAGE_HPTEFLAGS (H_PAGE_BUSY | H_PAGE_HASHPTE | H_PAGE_COMBO) /* * We use a 2K PTE page fragment and another 2K for storing * real_pte_t hash index * 8 bytes per each pte entry and another 8 bytes for storing * slot details. */ #define H_PTE_FRAG_SIZE_SHIFT (H_PTE_INDEX_SIZE + 3 + 1) #define H_PTE_FRAG_NR (PAGE_SIZE >> H_PTE_FRAG_SIZE_SHIFT) #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLB_PAGE) #define H_PMD_FRAG_SIZE_SHIFT (H_PMD_INDEX_SIZE + 3 + 1) #else #define H_PMD_FRAG_SIZE_SHIFT (H_PMD_INDEX_SIZE + 3) #endif #define H_PMD_FRAG_NR (PAGE_SIZE >> H_PMD_FRAG_SIZE_SHIFT) #ifndef __ASSEMBLY__ #include <asm/errno.h> /* * With 64K pages on hash table, we have a special PTE format that * uses a second "half" of the page table to encode sub-page information * in order to deal with 64K made of 4K HW pages. Thus we override the * generic accessors and iterators here */ #define __real_pte __real_pte static inline real_pte_t __real_pte(pte_t pte, pte_t *ptep, int offset) { real_pte_t rpte; unsigned long *hidxp; rpte.pte = pte; /* * Ensure that we do not read the hidx before we read the PTE. Because * the writer side is expected to finish writing the hidx first followed * by the PTE, by using smp_wmb(). pte_set_hash_slot() ensures that. */ smp_rmb(); hidxp = (unsigned long *)(ptep + offset); rpte.hidx = *hidxp; return rpte; } /* * shift the hidx representation by one-modulo-0xf; i.e hidx 0 is respresented * as 1, 1 as 2,... , and 0xf as 0. This convention lets us represent a * invalid hidx 0xf with a 0x0 bit value. PTEs are anyway zero'd when * allocated. We dont have to zero them gain; thus save on the initialization. */ #define HIDX_UNSHIFT_BY_ONE(x) ((x + 0xfUL) & 0xfUL) /* shift backward by one */ #define HIDX_SHIFT_BY_ONE(x) ((x + 0x1UL) & 0xfUL) /* shift forward by one */ #define HIDX_BITS(x, index) (x << (index << 2)) #define BITS_TO_HIDX(x, index) ((x >> (index << 2)) & 0xfUL) #define INVALID_RPTE_HIDX 0x0UL static inline unsigned long __rpte_to_hidx(real_pte_t rpte, unsigned long index) { return HIDX_UNSHIFT_BY_ONE(BITS_TO_HIDX(rpte.hidx, index)); } /* * Commit the hidx and return PTE bits that needs to be modified. The caller is * expected to modify the PTE bits accordingly and commit the PTE to memory. */ static inline unsigned long pte_set_hidx(pte_t *ptep, real_pte_t rpte, unsigned int subpg_index, unsigned long hidx, int offset) { unsigned long *hidxp = (unsigned long *)(ptep + offset); rpte.hidx &= ~HIDX_BITS(0xfUL, subpg_index); *hidxp = rpte.hidx | HIDX_BITS(HIDX_SHIFT_BY_ONE(hidx), subpg_index); /* * Anyone reading PTE must ensure hidx bits are read after reading the * PTE by using the read-side barrier smp_rmb(). __real_pte() can be * used for that. */ smp_wmb(); /* No PTE bits to be modified, return 0x0UL */ return 0x0UL; } #define __rpte_to_pte(r) ((r).pte) extern bool __rpte_sub_valid(real_pte_t rpte, unsigned long index); /* * Trick: we set __end to va + 64k, which happens works for * a 16M page as well as we want only one iteration */ #define pte_iterate_hashed_subpages(rpte, psize, vpn, index, shift) \ do { \ unsigned long __end = vpn + (1UL << (PAGE_SHIFT - VPN_SHIFT)); \ unsigned __split = (psize == MMU_PAGE_4K || \ psize == MMU_PAGE_64K_AP); \ shift = mmu_psize_defs[psize].shift; \ for (index = 0; vpn < __end; index++, \ vpn += (1L << (shift - VPN_SHIFT))) { \ if (!__split || __rpte_sub_valid(rpte, index)) #define pte_iterate_hashed_end() } } while(0) #define pte_pagesize_index(mm, addr, pte) \ (((pte) & H_PAGE_COMBO)? MMU_PAGE_4K: MMU_PAGE_64K) extern int remap_pfn_range(struct vm_area_struct *, unsigned long addr, unsigned long pfn, unsigned long size, pgprot_t); static inline int hash__remap_4k_pfn(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, pgprot_t prot) { if (pfn > (PTE_RPN_MASK >> PAGE_SHIFT)) { WARN(1, "remap_4k_pfn called with wrong pfn value\n"); return -EINVAL; } return remap_pfn_range(vma, addr, pfn, PAGE_SIZE, __pgprot(pgprot_val(prot) | H_PAGE_4K_PFN)); } #define H_PTE_TABLE_SIZE PTE_FRAG_SIZE #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined (CONFIG_HUGETLB_PAGE) #define H_PMD_TABLE_SIZE ((sizeof(pmd_t) << PMD_INDEX_SIZE) + \ (sizeof(unsigned long) << PMD_INDEX_SIZE)) #else #define H_PMD_TABLE_SIZE (sizeof(pmd_t) << PMD_INDEX_SIZE) #endif #ifdef CONFIG_HUGETLB_PAGE #define H_PUD_TABLE_SIZE ((sizeof(pud_t) << PUD_INDEX_SIZE) + \ (sizeof(unsigned long) << PUD_INDEX_SIZE)) #else #define H_PUD_TABLE_SIZE (sizeof(pud_t) << PUD_INDEX_SIZE) #endif #define H_PGD_TABLE_SIZE (sizeof(pgd_t) << PGD_INDEX_SIZE) #ifdef CONFIG_TRANSPARENT_HUGEPAGE static inline char *get_hpte_slot_array(pmd_t *pmdp) { /* * The hpte hindex is stored in the pgtable whose address is in the * second half of the PMD * * Order this load with the test for pmd_trans_huge in the caller */ smp_rmb(); return *(char **)(pmdp + PTRS_PER_PMD); } /* * The linux hugepage PMD now include the pmd entries followed by the address * to the stashed pgtable_t. The stashed pgtable_t contains the hpte bits. * [ 000 | 1 bit secondary | 3 bit hidx | 1 bit valid]. We use one byte per * each HPTE entry. With 16MB hugepage and 64K HPTE we need 256 entries and * with 4K HPTE we need 4096 entries. Both will fit in a 4K pgtable_t. * * The top three bits are intentionally left as zero. This memory location * are also used as normal page PTE pointers. So if we have any pointers * left around while we collapse a hugepage, we need to make sure * _PAGE_PRESENT bit of that is zero when we look at them */ static inline unsigned int hpte_valid(unsigned char *hpte_slot_array, int index) { return hpte_slot_array[index] & 0x1; } static inline unsigned int hpte_hash_index(unsigned char *hpte_slot_array, int index) { return hpte_slot_array[index] >> 1; } static inline void mark_hpte_slot_valid(unsigned char *hpte_slot_array, unsigned int index, unsigned int hidx) { hpte_slot_array[index] = (hidx << 1) | 0x1; } /* * * For core kernel code by design pmd_trans_huge is never run on any hugetlbfs * page. The hugetlbfs page table walking and mangling paths are totally * separated form the core VM paths and they're differentiated by * VM_HUGETLB being set on vm_flags well before any pmd_trans_huge could run. * * pmd_trans_huge() is defined as false at build time if * CONFIG_TRANSPARENT_HUGEPAGE=n to optimize away code blocks at build * time in such case. * * For ppc64 we need to differntiate from explicit hugepages from THP, because * for THP we also track the subpage details at the pmd level. We don't do * that for explicit huge pages. * */ static inline int hash__pmd_trans_huge(pmd_t pmd) { return !!((pmd_val(pmd) & (_PAGE_PTE | H_PAGE_THP_HUGE | _PAGE_DEVMAP)) == (_PAGE_PTE | H_PAGE_THP_HUGE)); } static inline int hash__pmd_same(pmd_t pmd_a, pmd_t pmd_b) { return (((pmd_raw(pmd_a) ^ pmd_raw(pmd_b)) & ~cpu_to_be64(_PAGE_HPTEFLAGS)) == 0); } static inline pmd_t hash__pmd_mkhuge(pmd_t pmd) { return __pmd(pmd_val(pmd) | (_PAGE_PTE | H_PAGE_THP_HUGE)); } extern unsigned long hash__pmd_hugepage_update(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, unsigned long clr, unsigned long set); extern pmd_t hash__pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp); extern void hash__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, pgtable_t pgtable); extern pgtable_t hash__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp); extern pmd_t hash__pmdp_huge_get_and_clear(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp); extern int hash__has_transparent_hugepage(void); #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ static inline pmd_t hash__pmd_mkdevmap(pmd_t pmd) { return __pmd(pmd_val(pmd) | (_PAGE_PTE | H_PAGE_THP_HUGE | _PAGE_DEVMAP)); } #endif /* __ASSEMBLY__ */ #endif /* _ASM_POWERPC_BOOK3S_64_HASH_64K_H */ PK ! �3P� hash-pkey.hnu �[��� /* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_BOOK3S_64_HASH_PKEY_H #define _ASM_POWERPC_BOOK3S_64_HASH_PKEY_H /* We use key 3 for KERNEL */ #define HASH_DEFAULT_KERNEL_KEY (HPTE_R_KEY_BIT0 | HPTE_R_KEY_BIT1) static inline u64 hash__vmflag_to_pte_pkey_bits(u64 vm_flags) { return (((vm_flags & VM_PKEY_BIT0) ? H_PTE_PKEY_BIT0 : 0x0UL) | ((vm_flags & VM_PKEY_BIT1) ? H_PTE_PKEY_BIT1 : 0x0UL) | ((vm_flags & VM_PKEY_BIT2) ? H_PTE_PKEY_BIT2 : 0x0UL) | ((vm_flags & VM_PKEY_BIT3) ? H_PTE_PKEY_BIT3 : 0x0UL) | ((vm_flags & VM_PKEY_BIT4) ? H_PTE_PKEY_BIT4 : 0x0UL)); } static inline u64 pte_to_hpte_pkey_bits(u64 pteflags, unsigned long flags) { unsigned long pte_pkey; pte_pkey = (((pteflags & H_PTE_PKEY_BIT4) ? HPTE_R_KEY_BIT4 : 0x0UL) | ((pteflags & H_PTE_PKEY_BIT3) ? HPTE_R_KEY_BIT3 : 0x0UL) | ((pteflags & H_PTE_PKEY_BIT2) ? HPTE_R_KEY_BIT2 : 0x0UL) | ((pteflags & H_PTE_PKEY_BIT1) ? HPTE_R_KEY_BIT1 : 0x0UL) | ((pteflags & H_PTE_PKEY_BIT0) ? HPTE_R_KEY_BIT0 : 0x0UL)); if (mmu_has_feature(MMU_FTR_BOOK3S_KUAP) || mmu_has_feature(MMU_FTR_BOOK3S_KUEP)) { if ((pte_pkey == 0) && (flags & HPTE_USE_KERNEL_KEY)) return HASH_DEFAULT_KERNEL_KEY; } return pte_pkey; } static inline u16 hash__pte_to_pkey_bits(u64 pteflags) { return (((pteflags & H_PTE_PKEY_BIT4) ? 0x10 : 0x0UL) | ((pteflags & H_PTE_PKEY_BIT3) ? 0x8 : 0x0UL) | ((pteflags & H_PTE_PKEY_BIT2) ? 0x4 : 0x0UL) | ((pteflags & H_PTE_PKEY_BIT1) ? 0x2 : 0x0UL) | ((pteflags & H_PTE_PKEY_BIT0) ? 0x1 : 0x0UL)); } #endif PK ! uI4�- - kexec.hnu �[��� /* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_BOOK3S_64_KEXEC_H_ #define _ASM_POWERPC_BOOK3S_64_KEXEC_H_ #include <asm/plpar_wrappers.h> #define reset_sprs reset_sprs static inline void reset_sprs(void) { if (cpu_has_feature(CPU_FTR_ARCH_206)) { mtspr(SPRN_AMR, 0); mtspr(SPRN_UAMOR, 0); } if (cpu_has_feature(CPU_FTR_ARCH_207S)) { mtspr(SPRN_IAMR, 0); if (cpu_has_feature(CPU_FTR_HVMODE)) mtspr(SPRN_CIABR, 0); else plpar_set_ciabr(0); } /* Do we need isync()? We are going via a kexec reset */ isync(); } #endif PK ! r*w�W� W� pgtable.hnu �[��� /* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_BOOK3S_64_PGTABLE_H_ #define _ASM_POWERPC_BOOK3S_64_PGTABLE_H_ #include <asm-generic/pgtable-nop4d.h> #ifndef __ASSEMBLY__ #include <linux/mmdebug.h> #include <linux/bug.h> #include <linux/sizes.h> #endif /* * Common bits between hash and Radix page table */ #define _PAGE_BIT_SWAP_TYPE 0 #define _PAGE_EXEC 0x00001 /* execute permission */ #define _PAGE_WRITE 0x00002 /* write access allowed */ #define _PAGE_READ 0x00004 /* read access allowed */ #define _PAGE_RW (_PAGE_READ | _PAGE_WRITE) #define _PAGE_RWX (_PAGE_READ | _PAGE_WRITE | _PAGE_EXEC) #define _PAGE_PRIVILEGED 0x00008 /* kernel access only */ #define _PAGE_SAO 0x00010 /* Strong access order */ #define _PAGE_NON_IDEMPOTENT 0x00020 /* non idempotent memory */ #define _PAGE_TOLERANT 0x00030 /* tolerant memory, cache inhibited */ #define _PAGE_DIRTY 0x00080 /* C: page changed */ #define _PAGE_ACCESSED 0x00100 /* R: page referenced */ /* * Software bits */ #define _RPAGE_SW0 0x2000000000000000UL #define _RPAGE_SW1 0x00800 #define _RPAGE_SW2 0x00400 #define _RPAGE_SW3 0x00200 #define _RPAGE_RSV1 0x00040UL #define _RPAGE_PKEY_BIT4 0x1000000000000000UL #define _RPAGE_PKEY_BIT3 0x0800000000000000UL #define _RPAGE_PKEY_BIT2 0x0400000000000000UL #define _RPAGE_PKEY_BIT1 0x0200000000000000UL #define _RPAGE_PKEY_BIT0 0x0100000000000000UL #define _PAGE_PTE 0x4000000000000000UL /* distinguishes PTEs from pointers */ #define _PAGE_PRESENT 0x8000000000000000UL /* pte contains a translation */ /* * We need to mark a pmd pte invalid while splitting. We can do that by clearing * the _PAGE_PRESENT bit. But then that will be taken as a swap pte. In order to * differentiate between two use a SW field when invalidating. * * We do that temporary invalidate for regular pte entry in ptep_set_access_flags * * This is used only when _PAGE_PRESENT is cleared. */ #define _PAGE_INVALID _RPAGE_SW0 /* * Top and bottom bits of RPN which can be used by hash * translation mode, because we expect them to be zero * otherwise. */ #define _RPAGE_RPN0 0x01000 #define _RPAGE_RPN1 0x02000 #define _RPAGE_RPN43 0x0080000000000000UL #define _RPAGE_RPN42 0x0040000000000000UL #define _RPAGE_RPN41 0x0020000000000000UL /* Max physical address bit as per radix table */ #define _RPAGE_PA_MAX 56 /* * Max physical address bit we will use for now. * * This is mostly a hardware limitation and for now Power9 has * a 51 bit limit. * * This is different from the number of physical bit required to address * the last byte of memory. That is defined by MAX_PHYSMEM_BITS. * MAX_PHYSMEM_BITS is a linux limitation imposed by the maximum * number of sections we can support (SECTIONS_SHIFT). * * This is different from Radix page table limitation above and * should always be less than that. The limit is done such that * we can overload the bits between _RPAGE_PA_MAX and _PAGE_PA_MAX * for hash linux page table specific bits. * * In order to be compatible with future hardware generations we keep * some offsets and limit this for now to 53 */ #define _PAGE_PA_MAX 53 #define _PAGE_SOFT_DIRTY _RPAGE_SW3 /* software: software dirty tracking */ #define _PAGE_SPECIAL _RPAGE_SW2 /* software: special page */ #define _PAGE_DEVMAP _RPAGE_SW1 /* software: ZONE_DEVICE page */ /* * Drivers request for cache inhibited pte mapping using _PAGE_NO_CACHE * Instead of fixing all of them, add an alternate define which * maps CI pte mapping. */ #define _PAGE_NO_CACHE _PAGE_TOLERANT /* * We support _RPAGE_PA_MAX bit real address in pte. On the linux side * we are limited by _PAGE_PA_MAX. Clear everything above _PAGE_PA_MAX * and every thing below PAGE_SHIFT; */ #define PTE_RPN_MASK (((1UL << _PAGE_PA_MAX) - 1) & (PAGE_MASK)) /* * set of bits not changed in pmd_modify. Even though we have hash specific bits * in here, on radix we expect them to be zero. */ #define _HPAGE_CHG_MASK (PTE_RPN_MASK | _PAGE_HPTEFLAGS | _PAGE_DIRTY | \ _PAGE_ACCESSED | H_PAGE_THP_HUGE | _PAGE_PTE | \ _PAGE_SOFT_DIRTY | _PAGE_DEVMAP) /* * user access blocked by key */ #define _PAGE_KERNEL_RW (_PAGE_PRIVILEGED | _PAGE_RW | _PAGE_DIRTY) #define _PAGE_KERNEL_RO (_PAGE_PRIVILEGED | _PAGE_READ) #define _PAGE_KERNEL_ROX (_PAGE_PRIVILEGED | _PAGE_READ | _PAGE_EXEC) #define _PAGE_KERNEL_RWX (_PAGE_PRIVILEGED | _PAGE_DIRTY | \ _PAGE_RW | _PAGE_EXEC) /* * _PAGE_CHG_MASK masks of bits that are to be preserved across * pgprot changes */ #define _PAGE_CHG_MASK (PTE_RPN_MASK | _PAGE_HPTEFLAGS | _PAGE_DIRTY | \ _PAGE_ACCESSED | _PAGE_SPECIAL | _PAGE_PTE | \ _PAGE_SOFT_DIRTY | _PAGE_DEVMAP) /* * We define 2 sets of base prot bits, one for basic pages (ie, * cacheable kernel and user pages) and one for non cacheable * pages. We always set _PAGE_COHERENT when SMP is enabled or * the processor might need it for DMA coherency. */ #define _PAGE_BASE_NC (_PAGE_PRESENT | _PAGE_ACCESSED) #define _PAGE_BASE (_PAGE_BASE_NC) /* Permission masks used to generate the __P and __S table, * * Note:__pgprot is defined in arch/powerpc/include/asm/page.h * * Write permissions imply read permissions for now (we could make write-only * pages on BookE but we don't bother for now). Execute permission control is * possible on platforms that define _PAGE_EXEC */ #define PAGE_NONE __pgprot(_PAGE_BASE | _PAGE_PRIVILEGED) #define PAGE_SHARED __pgprot(_PAGE_BASE | _PAGE_RW) #define PAGE_SHARED_X __pgprot(_PAGE_BASE | _PAGE_RW | _PAGE_EXEC) #define PAGE_COPY __pgprot(_PAGE_BASE | _PAGE_READ) #define PAGE_COPY_X __pgprot(_PAGE_BASE | _PAGE_READ | _PAGE_EXEC) #define PAGE_READONLY __pgprot(_PAGE_BASE | _PAGE_READ) #define PAGE_READONLY_X __pgprot(_PAGE_BASE | _PAGE_READ | _PAGE_EXEC) /* Permission masks used for kernel mappings */ #define PAGE_KERNEL __pgprot(_PAGE_BASE | _PAGE_KERNEL_RW) #define PAGE_KERNEL_NC __pgprot(_PAGE_BASE_NC | _PAGE_KERNEL_RW | \ _PAGE_TOLERANT) #define PAGE_KERNEL_NCG __pgprot(_PAGE_BASE_NC | _PAGE_KERNEL_RW | \ _PAGE_NON_IDEMPOTENT) #define PAGE_KERNEL_X __pgprot(_PAGE_BASE | _PAGE_KERNEL_RWX) #define PAGE_KERNEL_RO __pgprot(_PAGE_BASE | _PAGE_KERNEL_RO) #define PAGE_KERNEL_ROX __pgprot(_PAGE_BASE | _PAGE_KERNEL_ROX) /* * Protection used for kernel text. We want the debuggers to be able to * set breakpoints anywhere, so don't write protect the kernel text * on platforms where such control is possible. */ #if defined(CONFIG_KGDB) || defined(CONFIG_XMON) || defined(CONFIG_BDI_SWITCH) || \ defined(CONFIG_KPROBES) || defined(CONFIG_DYNAMIC_FTRACE) #define PAGE_KERNEL_TEXT PAGE_KERNEL_X #else #define PAGE_KERNEL_TEXT PAGE_KERNEL_ROX #endif /* Make modules code happy. We don't set RO yet */ #define PAGE_KERNEL_EXEC PAGE_KERNEL_X #define PAGE_AGP (PAGE_KERNEL_NC) #ifndef __ASSEMBLY__ /* * page table defines */ extern unsigned long __pte_index_size; extern unsigned long __pmd_index_size; extern unsigned long __pud_index_size; extern unsigned long __pgd_index_size; extern unsigned long __pud_cache_index; #define PTE_INDEX_SIZE __pte_index_size #define PMD_INDEX_SIZE __pmd_index_size #define PUD_INDEX_SIZE __pud_index_size #define PGD_INDEX_SIZE __pgd_index_size /* pmd table use page table fragments */ #define PMD_CACHE_INDEX 0 #define PUD_CACHE_INDEX __pud_cache_index /* * Because of use of pte fragments and THP, size of page table * are not always derived out of index size above. */ extern unsigned long __pte_table_size; extern unsigned long __pmd_table_size; extern unsigned long __pud_table_size; extern unsigned long __pgd_table_size; #define PTE_TABLE_SIZE __pte_table_size #define PMD_TABLE_SIZE __pmd_table_size #define PUD_TABLE_SIZE __pud_table_size #define PGD_TABLE_SIZE __pgd_table_size extern unsigned long __pmd_val_bits; extern unsigned long __pud_val_bits; extern unsigned long __pgd_val_bits; #define PMD_VAL_BITS __pmd_val_bits #define PUD_VAL_BITS __pud_val_bits #define PGD_VAL_BITS __pgd_val_bits extern unsigned long __pte_frag_nr; #define PTE_FRAG_NR __pte_frag_nr extern unsigned long __pte_frag_size_shift; #define PTE_FRAG_SIZE_SHIFT __pte_frag_size_shift #define PTE_FRAG_SIZE (1UL << PTE_FRAG_SIZE_SHIFT) extern unsigned long __pmd_frag_nr; #define PMD_FRAG_NR __pmd_frag_nr extern unsigned long __pmd_frag_size_shift; #define PMD_FRAG_SIZE_SHIFT __pmd_frag_size_shift #define PMD_FRAG_SIZE (1UL << PMD_FRAG_SIZE_SHIFT) #define PTRS_PER_PTE (1 << PTE_INDEX_SIZE) #define PTRS_PER_PMD (1 << PMD_INDEX_SIZE) #define PTRS_PER_PUD (1 << PUD_INDEX_SIZE) #define PTRS_PER_PGD (1 << PGD_INDEX_SIZE) #define MAX_PTRS_PER_PGD (1 << (H_PGD_INDEX_SIZE > RADIX_PGD_INDEX_SIZE ? \ H_PGD_INDEX_SIZE : RADIX_PGD_INDEX_SIZE)) /* PMD_SHIFT determines what a second-level page table entry can map */ #define PMD_SHIFT (PAGE_SHIFT + PTE_INDEX_SIZE) #define PMD_SIZE (1UL << PMD_SHIFT) #define PMD_MASK (~(PMD_SIZE-1)) /* PUD_SHIFT determines what a third-level page table entry can map */ #define PUD_SHIFT (PMD_SHIFT + PMD_INDEX_SIZE) #define PUD_SIZE (1UL << PUD_SHIFT) #define PUD_MASK (~(PUD_SIZE-1)) /* PGDIR_SHIFT determines what a fourth-level page table entry can map */ #define PGDIR_SHIFT (PUD_SHIFT + PUD_INDEX_SIZE) #define PGDIR_SIZE (1UL << PGDIR_SHIFT) #define PGDIR_MASK (~(PGDIR_SIZE-1)) /* Bits to mask out from a PMD to get to the PTE page */ #define PMD_MASKED_BITS 0xc0000000000000ffUL /* Bits to mask out from a PUD to get to the PMD page */ #define PUD_MASKED_BITS 0xc0000000000000ffUL /* Bits to mask out from a PGD to get to the PUD page */ #define P4D_MASKED_BITS 0xc0000000000000ffUL /* * Used as an indicator for rcu callback functions */ enum pgtable_index { PTE_INDEX = 0, PMD_INDEX, PUD_INDEX, PGD_INDEX, /* * Below are used with 4k page size and hugetlb */ HTLB_16M_INDEX, HTLB_16G_INDEX, }; extern unsigned long __vmalloc_start; extern unsigned long __vmalloc_end; #define VMALLOC_START __vmalloc_start #define VMALLOC_END __vmalloc_end static inline unsigned int ioremap_max_order(void) { if (radix_enabled()) return PUD_SHIFT; return 7 + PAGE_SHIFT; /* default from linux/vmalloc.h */ } #define IOREMAP_MAX_ORDER ioremap_max_order() extern unsigned long __kernel_virt_start; extern unsigned long __kernel_io_start; extern unsigned long __kernel_io_end; #define KERN_VIRT_START __kernel_virt_start #define KERN_IO_START __kernel_io_start #define KERN_IO_END __kernel_io_end extern struct page *vmemmap; extern unsigned long pci_io_base; #endif /* __ASSEMBLY__ */ #include <asm/book3s/64/hash.h> #include <asm/book3s/64/radix.h> #if H_MAX_PHYSMEM_BITS > R_MAX_PHYSMEM_BITS #define MAX_PHYSMEM_BITS H_MAX_PHYSMEM_BITS #else #define MAX_PHYSMEM_BITS R_MAX_PHYSMEM_BITS #endif #ifdef CONFIG_PPC_64K_PAGES #include <asm/book3s/64/pgtable-64k.h> #else #include <asm/book3s/64/pgtable-4k.h> #endif #include <asm/barrier.h> /* * IO space itself carved into the PIO region (ISA and PHB IO space) and * the ioremap space * * ISA_IO_BASE = KERN_IO_START, 64K reserved area * PHB_IO_BASE = ISA_IO_BASE + 64K to ISA_IO_BASE + 2G, PHB IO spaces * IOREMAP_BASE = ISA_IO_BASE + 2G to VMALLOC_START + PGTABLE_RANGE */ #define FULL_IO_SIZE 0x80000000ul #define ISA_IO_BASE (KERN_IO_START) #define ISA_IO_END (KERN_IO_START + 0x10000ul) #define PHB_IO_BASE (ISA_IO_END) #define PHB_IO_END (KERN_IO_START + FULL_IO_SIZE) #define IOREMAP_BASE (PHB_IO_END) #define IOREMAP_START (ioremap_bot) #define IOREMAP_END (KERN_IO_END - FIXADDR_SIZE) #define FIXADDR_SIZE SZ_32M /* Advertise special mapping type for AGP */ #define HAVE_PAGE_AGP #ifndef __ASSEMBLY__ static inline unsigned long pte_update(struct mm_struct *mm, unsigned long addr, pte_t *ptep, unsigned long clr, unsigned long set, int huge) { if (radix_enabled()) return radix__pte_update(mm, addr, ptep, clr, set, huge); return hash__pte_update(mm, addr, ptep, clr, set, huge); } /* * For hash even if we have _PAGE_ACCESSED = 0, we do a pte_update. * We currently remove entries from the hashtable regardless of whether * the entry was young or dirty. * * We should be more intelligent about this but for the moment we override * these functions and force a tlb flush unconditionally * For radix: H_PAGE_HASHPTE should be zero. Hence we can use the same * function for both hash and radix. */ static inline int __ptep_test_and_clear_young(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { unsigned long old; if ((pte_raw(*ptep) & cpu_to_be64(_PAGE_ACCESSED | H_PAGE_HASHPTE)) == 0) return 0; old = pte_update(mm, addr, ptep, _PAGE_ACCESSED, 0, 0); return (old & _PAGE_ACCESSED) != 0; } #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG #define ptep_test_and_clear_young(__vma, __addr, __ptep) \ ({ \ __ptep_test_and_clear_young((__vma)->vm_mm, __addr, __ptep); \ }) /* * On Book3S CPUs, clearing the accessed bit without a TLB flush * doesn't cause data corruption. [ It could cause incorrect * page aging and the (mistaken) reclaim of hot pages, but the * chance of that should be relatively low. ] * * So as a performance optimization don't flush the TLB when * clearing the accessed bit, it will eventually be flushed by * a context switch or a VM operation anyway. [ In the rare * event of it not getting flushed for a long time the delay * shouldn't really matter because there's no real memory * pressure for swapout to react to. ] */ #define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH #define ptep_clear_flush_young ptep_test_and_clear_young #define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH #define pmdp_clear_flush_young pmdp_test_and_clear_young static inline int __pte_write(pte_t pte) { return !!(pte_raw(pte) & cpu_to_be64(_PAGE_WRITE)); } #ifdef CONFIG_NUMA_BALANCING #define pte_savedwrite pte_savedwrite static inline bool pte_savedwrite(pte_t pte) { /* * Saved write ptes are prot none ptes that doesn't have * privileged bit sit. We mark prot none as one which has * present and pviliged bit set and RWX cleared. To mark * protnone which used to have _PAGE_WRITE set we clear * the privileged bit. */ return !(pte_raw(pte) & cpu_to_be64(_PAGE_RWX | _PAGE_PRIVILEGED)); } #else #define pte_savedwrite pte_savedwrite static inline bool pte_savedwrite(pte_t pte) { return false; } #endif static inline int pte_write(pte_t pte) { return __pte_write(pte) || pte_savedwrite(pte); } static inline int pte_read(pte_t pte) { return !!(pte_raw(pte) & cpu_to_be64(_PAGE_READ)); } #define __HAVE_ARCH_PTEP_SET_WRPROTECT static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { if (__pte_write(*ptep)) pte_update(mm, addr, ptep, _PAGE_WRITE, 0, 0); else if (unlikely(pte_savedwrite(*ptep))) pte_update(mm, addr, ptep, 0, _PAGE_PRIVILEGED, 0); } #define __HAVE_ARCH_HUGE_PTEP_SET_WRPROTECT static inline void huge_ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { /* * We should not find protnone for hugetlb, but this complete the * interface. */ if (__pte_write(*ptep)) pte_update(mm, addr, ptep, _PAGE_WRITE, 0, 1); else if (unlikely(pte_savedwrite(*ptep))) pte_update(mm, addr, ptep, 0, _PAGE_PRIVILEGED, 1); } #define __HAVE_ARCH_PTEP_GET_AND_CLEAR static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { unsigned long old = pte_update(mm, addr, ptep, ~0UL, 0, 0); return __pte(old); } #define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, unsigned long addr, pte_t *ptep, int full) { if (full && radix_enabled()) { /* * We know that this is a full mm pte clear and * hence can be sure there is no parallel set_pte. */ return radix__ptep_get_and_clear_full(mm, addr, ptep, full); } return ptep_get_and_clear(mm, addr, ptep); } static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t * ptep) { pte_update(mm, addr, ptep, ~0UL, 0, 0); } static inline int pte_dirty(pte_t pte) { return !!(pte_raw(pte) & cpu_to_be64(_PAGE_DIRTY)); } static inline int pte_young(pte_t pte) { return !!(pte_raw(pte) & cpu_to_be64(_PAGE_ACCESSED)); } static inline int pte_special(pte_t pte) { return !!(pte_raw(pte) & cpu_to_be64(_PAGE_SPECIAL)); } static inline bool pte_exec(pte_t pte) { return !!(pte_raw(pte) & cpu_to_be64(_PAGE_EXEC)); } #ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY static inline bool pte_soft_dirty(pte_t pte) { return !!(pte_raw(pte) & cpu_to_be64(_PAGE_SOFT_DIRTY)); } static inline pte_t pte_mksoft_dirty(pte_t pte) { return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_SOFT_DIRTY)); } static inline pte_t pte_clear_soft_dirty(pte_t pte) { return __pte_raw(pte_raw(pte) & cpu_to_be64(~_PAGE_SOFT_DIRTY)); } #endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */ #ifdef CONFIG_NUMA_BALANCING static inline int pte_protnone(pte_t pte) { return (pte_raw(pte) & cpu_to_be64(_PAGE_PRESENT | _PAGE_PTE | _PAGE_RWX)) == cpu_to_be64(_PAGE_PRESENT | _PAGE_PTE); } #define pte_mk_savedwrite pte_mk_savedwrite static inline pte_t pte_mk_savedwrite(pte_t pte) { /* * Used by Autonuma subsystem to preserve the write bit * while marking the pte PROT_NONE. Only allow this * on PROT_NONE pte */ VM_BUG_ON((pte_raw(pte) & cpu_to_be64(_PAGE_PRESENT | _PAGE_RWX | _PAGE_PRIVILEGED)) != cpu_to_be64(_PAGE_PRESENT | _PAGE_PRIVILEGED)); return __pte_raw(pte_raw(pte) & cpu_to_be64(~_PAGE_PRIVILEGED)); } #define pte_clear_savedwrite pte_clear_savedwrite static inline pte_t pte_clear_savedwrite(pte_t pte) { /* * Used by KSM subsystem to make a protnone pte readonly. */ VM_BUG_ON(!pte_protnone(pte)); return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_PRIVILEGED)); } #else #define pte_clear_savedwrite pte_clear_savedwrite static inline pte_t pte_clear_savedwrite(pte_t pte) { VM_WARN_ON(1); return __pte_raw(pte_raw(pte) & cpu_to_be64(~_PAGE_WRITE)); } #endif /* CONFIG_NUMA_BALANCING */ static inline bool pte_hw_valid(pte_t pte) { return (pte_raw(pte) & cpu_to_be64(_PAGE_PRESENT | _PAGE_PTE)) == cpu_to_be64(_PAGE_PRESENT | _PAGE_PTE); } static inline int pte_present(pte_t pte) { /* * A pte is considerent present if _PAGE_PRESENT is set. * We also need to consider the pte present which is marked * invalid during ptep_set_access_flags. Hence we look for _PAGE_INVALID * if we find _PAGE_PRESENT cleared. */ if (pte_hw_valid(pte)) return true; return (pte_raw(pte) & cpu_to_be64(_PAGE_INVALID | _PAGE_PTE)) == cpu_to_be64(_PAGE_INVALID | _PAGE_PTE); } #ifdef CONFIG_PPC_MEM_KEYS extern bool arch_pte_access_permitted(u64 pte, bool write, bool execute); #else static inline bool arch_pte_access_permitted(u64 pte, bool write, bool execute) { return true; } #endif /* CONFIG_PPC_MEM_KEYS */ static inline bool pte_user(pte_t pte) { return !(pte_raw(pte) & cpu_to_be64(_PAGE_PRIVILEGED)); } #define pte_access_permitted pte_access_permitted static inline bool pte_access_permitted(pte_t pte, bool write) { /* * _PAGE_READ is needed for any access and will be * cleared for PROT_NONE */ if (!pte_present(pte) || !pte_user(pte) || !pte_read(pte)) return false; if (write && !pte_write(pte)) return false; return arch_pte_access_permitted(pte_val(pte), write, 0); } /* * Conversion functions: convert a page and protection to a page entry, * and a page entry and page directory to the page they refer to. * * Even if PTEs can be unsigned long long, a PFN is always an unsigned * long for now. */ static inline pte_t pfn_pte(unsigned long pfn, pgprot_t pgprot) { VM_BUG_ON(pfn >> (64 - PAGE_SHIFT)); VM_BUG_ON((pfn << PAGE_SHIFT) & ~PTE_RPN_MASK); return __pte(((pte_basic_t)pfn << PAGE_SHIFT) | pgprot_val(pgprot) | _PAGE_PTE); } static inline unsigned long pte_pfn(pte_t pte) { return (pte_val(pte) & PTE_RPN_MASK) >> PAGE_SHIFT; } /* Generic modifiers for PTE bits */ static inline pte_t pte_wrprotect(pte_t pte) { if (unlikely(pte_savedwrite(pte))) return pte_clear_savedwrite(pte); return __pte_raw(pte_raw(pte) & cpu_to_be64(~_PAGE_WRITE)); } static inline pte_t pte_exprotect(pte_t pte) { return __pte_raw(pte_raw(pte) & cpu_to_be64(~_PAGE_EXEC)); } static inline pte_t pte_mkclean(pte_t pte) { return __pte_raw(pte_raw(pte) & cpu_to_be64(~_PAGE_DIRTY)); } static inline pte_t pte_mkold(pte_t pte) { return __pte_raw(pte_raw(pte) & cpu_to_be64(~_PAGE_ACCESSED)); } static inline pte_t pte_mkexec(pte_t pte) { return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_EXEC)); } static inline pte_t pte_mkwrite(pte_t pte) { /* * write implies read, hence set both */ return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_RW)); } static inline pte_t pte_mkdirty(pte_t pte) { return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_DIRTY | _PAGE_SOFT_DIRTY)); } static inline pte_t pte_mkyoung(pte_t pte) { return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_ACCESSED)); } static inline pte_t pte_mkspecial(pte_t pte) { return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_SPECIAL)); } static inline pte_t pte_mkhuge(pte_t pte) { return pte; } static inline pte_t pte_mkdevmap(pte_t pte) { return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_SPECIAL | _PAGE_DEVMAP)); } static inline pte_t pte_mkprivileged(pte_t pte) { return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_PRIVILEGED)); } static inline pte_t pte_mkuser(pte_t pte) { return __pte_raw(pte_raw(pte) & cpu_to_be64(~_PAGE_PRIVILEGED)); } /* * This is potentially called with a pmd as the argument, in which case it's not * safe to check _PAGE_DEVMAP unless we also confirm that _PAGE_PTE is set. * That's because the bit we use for _PAGE_DEVMAP is not reserved for software * use in page directory entries (ie. non-ptes). */ static inline int pte_devmap(pte_t pte) { u64 mask = cpu_to_be64(_PAGE_DEVMAP | _PAGE_PTE); return (pte_raw(pte) & mask) == mask; } static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) { /* FIXME!! check whether this need to be a conditional */ return __pte_raw((pte_raw(pte) & cpu_to_be64(_PAGE_CHG_MASK)) | cpu_to_be64(pgprot_val(newprot))); } /* Encode and de-code a swap entry */ #define MAX_SWAPFILES_CHECK() do { \ BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > SWP_TYPE_BITS); \ /* \ * Don't have overlapping bits with _PAGE_HPTEFLAGS \ * We filter HPTEFLAGS on set_pte. \ */ \ BUILD_BUG_ON(_PAGE_HPTEFLAGS & (0x1f << _PAGE_BIT_SWAP_TYPE)); \ BUILD_BUG_ON(_PAGE_HPTEFLAGS & _PAGE_SWP_SOFT_DIRTY); \ } while (0) #define SWP_TYPE_BITS 5 #define __swp_type(x) (((x).val >> _PAGE_BIT_SWAP_TYPE) \ & ((1UL << SWP_TYPE_BITS) - 1)) #define __swp_offset(x) (((x).val & PTE_RPN_MASK) >> PAGE_SHIFT) #define __swp_entry(type, offset) ((swp_entry_t) { \ ((type) << _PAGE_BIT_SWAP_TYPE) \ | (((offset) << PAGE_SHIFT) & PTE_RPN_MASK)}) /* * swp_entry_t must be independent of pte bits. We build a swp_entry_t from * swap type and offset we get from swap and convert that to pte to find a * matching pte in linux page table. * Clear bits not found in swap entries here. */ #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val((pte)) & ~_PAGE_PTE }) #define __swp_entry_to_pte(x) __pte((x).val | _PAGE_PTE) #define __pmd_to_swp_entry(pmd) (__pte_to_swp_entry(pmd_pte(pmd))) #define __swp_entry_to_pmd(x) (pte_pmd(__swp_entry_to_pte(x))) #ifdef CONFIG_MEM_SOFT_DIRTY #define _PAGE_SWP_SOFT_DIRTY (1UL << (SWP_TYPE_BITS + _PAGE_BIT_SWAP_TYPE)) #else #define _PAGE_SWP_SOFT_DIRTY 0UL #endif /* CONFIG_MEM_SOFT_DIRTY */ #ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY static inline pte_t pte_swp_mksoft_dirty(pte_t pte) { return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_SWP_SOFT_DIRTY)); } static inline bool pte_swp_soft_dirty(pte_t pte) { return !!(pte_raw(pte) & cpu_to_be64(_PAGE_SWP_SOFT_DIRTY)); } static inline pte_t pte_swp_clear_soft_dirty(pte_t pte) { return __pte_raw(pte_raw(pte) & cpu_to_be64(~_PAGE_SWP_SOFT_DIRTY)); } #endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */ static inline bool check_pte_access(unsigned long access, unsigned long ptev) { /* * This check for _PAGE_RWX and _PAGE_PRESENT bits */ if (access & ~ptev) return false; /* * This check for access to privilege space */ if ((access & _PAGE_PRIVILEGED) != (ptev & _PAGE_PRIVILEGED)) return false; return true; } /* * Generic functions with hash/radix callbacks */ static inline void __ptep_set_access_flags(struct vm_area_struct *vma, pte_t *ptep, pte_t entry, unsigned long address, int psize) { if (radix_enabled()) return radix__ptep_set_access_flags(vma, ptep, entry, address, psize); return hash__ptep_set_access_flags(ptep, entry); } #define __HAVE_ARCH_PTE_SAME static inline int pte_same(pte_t pte_a, pte_t pte_b) { if (radix_enabled()) return radix__pte_same(pte_a, pte_b); return hash__pte_same(pte_a, pte_b); } static inline int pte_none(pte_t pte) { if (radix_enabled()) return radix__pte_none(pte); return hash__pte_none(pte); } static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte, int percpu) { VM_WARN_ON(!(pte_raw(pte) & cpu_to_be64(_PAGE_PTE))); /* * Keep the _PAGE_PTE added till we are sure we handle _PAGE_PTE * in all the callers. */ pte = __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_PTE)); if (radix_enabled()) return radix__set_pte_at(mm, addr, ptep, pte, percpu); return hash__set_pte_at(mm, addr, ptep, pte, percpu); } #define _PAGE_CACHE_CTL (_PAGE_SAO | _PAGE_NON_IDEMPOTENT | _PAGE_TOLERANT) #define pgprot_noncached pgprot_noncached static inline pgprot_t pgprot_noncached(pgprot_t prot) { return __pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | _PAGE_NON_IDEMPOTENT); } #define pgprot_noncached_wc pgprot_noncached_wc static inline pgprot_t pgprot_noncached_wc(pgprot_t prot) { return __pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | _PAGE_TOLERANT); } #define pgprot_cached pgprot_cached static inline pgprot_t pgprot_cached(pgprot_t prot) { return __pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL)); } #define pgprot_writecombine pgprot_writecombine static inline pgprot_t pgprot_writecombine(pgprot_t prot) { return pgprot_noncached_wc(prot); } /* * check a pte mapping have cache inhibited property */ static inline bool pte_ci(pte_t pte) { __be64 pte_v = pte_raw(pte); if (((pte_v & cpu_to_be64(_PAGE_CACHE_CTL)) == cpu_to_be64(_PAGE_TOLERANT)) || ((pte_v & cpu_to_be64(_PAGE_CACHE_CTL)) == cpu_to_be64(_PAGE_NON_IDEMPOTENT))) return true; return false; } static inline void pmd_clear(pmd_t *pmdp) { if (IS_ENABLED(CONFIG_DEBUG_VM) && !radix_enabled()) { /* * Don't use this if we can possibly have a hash page table * entry mapping this. */ WARN_ON((pmd_val(*pmdp) & (H_PAGE_HASHPTE | _PAGE_PTE)) == (H_PAGE_HASHPTE | _PAGE_PTE)); } *pmdp = __pmd(0); } static inline int pmd_none(pmd_t pmd) { return !pmd_raw(pmd); } static inline int pmd_present(pmd_t pmd) { /* * A pmd is considerent present if _PAGE_PRESENT is set. * We also need to consider the pmd present which is marked * invalid during a split. Hence we look for _PAGE_INVALID * if we find _PAGE_PRESENT cleared. */ if (pmd_raw(pmd) & cpu_to_be64(_PAGE_PRESENT | _PAGE_INVALID)) return true; return false; } static inline int pmd_is_serializing(pmd_t pmd) { /* * If the pmd is undergoing a split, the _PAGE_PRESENT bit is clear * and _PAGE_INVALID is set (see pmd_present, pmdp_invalidate). * * This condition may also occur when flushing a pmd while flushing * it (see ptep_modify_prot_start), so callers must ensure this * case is fine as well. */ if ((pmd_raw(pmd) & cpu_to_be64(_PAGE_PRESENT | _PAGE_INVALID)) == cpu_to_be64(_PAGE_INVALID)) return true; return false; } static inline int pmd_bad(pmd_t pmd) { if (radix_enabled()) return radix__pmd_bad(pmd); return hash__pmd_bad(pmd); } static inline void pud_clear(pud_t *pudp) { if (IS_ENABLED(CONFIG_DEBUG_VM) && !radix_enabled()) { /* * Don't use this if we can possibly have a hash page table * entry mapping this. */ WARN_ON((pud_val(*pudp) & (H_PAGE_HASHPTE | _PAGE_PTE)) == (H_PAGE_HASHPTE | _PAGE_PTE)); } *pudp = __pud(0); } static inline int pud_none(pud_t pud) { return !pud_raw(pud); } static inline int pud_present(pud_t pud) { return !!(pud_raw(pud) & cpu_to_be64(_PAGE_PRESENT)); } extern struct page *pud_page(pud_t pud); extern struct page *pmd_page(pmd_t pmd); static inline pte_t pud_pte(pud_t pud) { return __pte_raw(pud_raw(pud)); } static inline pud_t pte_pud(pte_t pte) { return __pud_raw(pte_raw(pte)); } #define pud_write(pud) pte_write(pud_pte(pud)) static inline int pud_bad(pud_t pud) { if (radix_enabled()) return radix__pud_bad(pud); return hash__pud_bad(pud); } #define pud_access_permitted pud_access_permitted static inline bool pud_access_permitted(pud_t pud, bool write) { return pte_access_permitted(pud_pte(pud), write); } #define __p4d_raw(x) ((p4d_t) { __pgd_raw(x) }) static inline __be64 p4d_raw(p4d_t x) { return pgd_raw(x.pgd); } #define p4d_write(p4d) pte_write(p4d_pte(p4d)) static inline void p4d_clear(p4d_t *p4dp) { *p4dp = __p4d(0); } static inline int p4d_none(p4d_t p4d) { return !p4d_raw(p4d); } static inline int p4d_present(p4d_t p4d) { return !!(p4d_raw(p4d) & cpu_to_be64(_PAGE_PRESENT)); } static inline pte_t p4d_pte(p4d_t p4d) { return __pte_raw(p4d_raw(p4d)); } static inline p4d_t pte_p4d(pte_t pte) { return __p4d_raw(pte_raw(pte)); } static inline int p4d_bad(p4d_t p4d) { if (radix_enabled()) return radix__p4d_bad(p4d); return hash__p4d_bad(p4d); } #define p4d_access_permitted p4d_access_permitted static inline bool p4d_access_permitted(p4d_t p4d, bool write) { return pte_access_permitted(p4d_pte(p4d), write); } extern struct page *p4d_page(p4d_t p4d); /* Pointers in the page table tree are physical addresses */ #define __pgtable_ptr_val(ptr) __pa(ptr) static inline pud_t *p4d_pgtable(p4d_t p4d) { return (pud_t *)__va(p4d_val(p4d) & ~P4D_MASKED_BITS); } static inline pmd_t *pud_pgtable(pud_t pud) { return (pmd_t *)__va(pud_val(pud) & ~PUD_MASKED_BITS); } #define pte_ERROR(e) \ pr_err("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e)) #define pmd_ERROR(e) \ pr_err("%s:%d: bad pmd %08lx.\n", __FILE__, __LINE__, pmd_val(e)) #define pud_ERROR(e) \ pr_err("%s:%d: bad pud %08lx.\n", __FILE__, __LINE__, pud_val(e)) #define pgd_ERROR(e) \ pr_err("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e)) static inline int map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot) { if (radix_enabled()) { #if defined(CONFIG_PPC_RADIX_MMU) && defined(DEBUG_VM) unsigned long page_size = 1 << mmu_psize_defs[mmu_io_psize].shift; WARN((page_size != PAGE_SIZE), "I/O page size != PAGE_SIZE"); #endif return radix__map_kernel_page(ea, pa, prot, PAGE_SIZE); } return hash__map_kernel_page(ea, pa, prot); } void unmap_kernel_page(unsigned long va); static inline int __meminit vmemmap_create_mapping(unsigned long start, unsigned long page_size, unsigned long phys) { if (radix_enabled()) return radix__vmemmap_create_mapping(start, page_size, phys); return hash__vmemmap_create_mapping(start, page_size, phys); } #ifdef CONFIG_MEMORY_HOTPLUG static inline void vmemmap_remove_mapping(unsigned long start, unsigned long page_size) { if (radix_enabled()) return radix__vmemmap_remove_mapping(start, page_size); return hash__vmemmap_remove_mapping(start, page_size); } #endif static inline pte_t pmd_pte(pmd_t pmd) { return __pte_raw(pmd_raw(pmd)); } static inline pmd_t pte_pmd(pte_t pte) { return __pmd_raw(pte_raw(pte)); } static inline pte_t *pmdp_ptep(pmd_t *pmd) { return (pte_t *)pmd; } #define pmd_pfn(pmd) pte_pfn(pmd_pte(pmd)) #define pmd_dirty(pmd) pte_dirty(pmd_pte(pmd)) #define pmd_young(pmd) pte_young(pmd_pte(pmd)) #define pmd_mkold(pmd) pte_pmd(pte_mkold(pmd_pte(pmd))) #define pmd_wrprotect(pmd) pte_pmd(pte_wrprotect(pmd_pte(pmd))) #define pmd_mkdirty(pmd) pte_pmd(pte_mkdirty(pmd_pte(pmd))) #define pmd_mkclean(pmd) pte_pmd(pte_mkclean(pmd_pte(pmd))) #define pmd_mkyoung(pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd))) #define pmd_mkwrite(pmd) pte_pmd(pte_mkwrite(pmd_pte(pmd))) #define pmd_mk_savedwrite(pmd) pte_pmd(pte_mk_savedwrite(pmd_pte(pmd))) #define pmd_clear_savedwrite(pmd) pte_pmd(pte_clear_savedwrite(pmd_pte(pmd))) #ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY #define pmd_soft_dirty(pmd) pte_soft_dirty(pmd_pte(pmd)) #define pmd_mksoft_dirty(pmd) pte_pmd(pte_mksoft_dirty(pmd_pte(pmd))) #define pmd_clear_soft_dirty(pmd) pte_pmd(pte_clear_soft_dirty(pmd_pte(pmd))) #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION #define pmd_swp_mksoft_dirty(pmd) pte_pmd(pte_swp_mksoft_dirty(pmd_pte(pmd))) #define pmd_swp_soft_dirty(pmd) pte_swp_soft_dirty(pmd_pte(pmd)) #define pmd_swp_clear_soft_dirty(pmd) pte_pmd(pte_swp_clear_soft_dirty(pmd_pte(pmd))) #endif #endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */ #ifdef CONFIG_NUMA_BALANCING static inline int pmd_protnone(pmd_t pmd) { return pte_protnone(pmd_pte(pmd)); } #endif /* CONFIG_NUMA_BALANCING */ #define pmd_write(pmd) pte_write(pmd_pte(pmd)) #define __pmd_write(pmd) __pte_write(pmd_pte(pmd)) #define pmd_savedwrite(pmd) pte_savedwrite(pmd_pte(pmd)) #define pmd_access_permitted pmd_access_permitted static inline bool pmd_access_permitted(pmd_t pmd, bool write) { /* * pmdp_invalidate sets this combination (which is not caught by * !pte_present() check in pte_access_permitted), to prevent * lock-free lookups, as part of the serialize_against_pte_lookup() * synchronisation. * * This also catches the case where the PTE's hardware PRESENT bit is * cleared while TLB is flushed, which is suboptimal but should not * be frequent. */ if (pmd_is_serializing(pmd)) return false; return pte_access_permitted(pmd_pte(pmd), write); } #ifdef CONFIG_TRANSPARENT_HUGEPAGE extern pmd_t pfn_pmd(unsigned long pfn, pgprot_t pgprot); extern pmd_t mk_pmd(struct page *page, pgprot_t pgprot); extern pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot); extern void set_pmd_at(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, pmd_t pmd); static inline void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmd) { } extern int hash__has_transparent_hugepage(void); static inline int has_transparent_hugepage(void) { if (radix_enabled()) return radix__has_transparent_hugepage(); return hash__has_transparent_hugepage(); } #define has_transparent_hugepage has_transparent_hugepage static inline unsigned long pmd_hugepage_update(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, unsigned long clr, unsigned long set) { if (radix_enabled()) return radix__pmd_hugepage_update(mm, addr, pmdp, clr, set); return hash__pmd_hugepage_update(mm, addr, pmdp, clr, set); } /* * returns true for pmd migration entries, THP, devmap, hugetlb * But compile time dependent on THP config */ static inline int pmd_large(pmd_t pmd) { return !!(pmd_raw(pmd) & cpu_to_be64(_PAGE_PTE)); } /* * For radix we should always find H_PAGE_HASHPTE zero. Hence * the below will work for radix too */ static inline int __pmdp_test_and_clear_young(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp) { unsigned long old; if ((pmd_raw(*pmdp) & cpu_to_be64(_PAGE_ACCESSED | H_PAGE_HASHPTE)) == 0) return 0; old = pmd_hugepage_update(mm, addr, pmdp, _PAGE_ACCESSED, 0); return ((old & _PAGE_ACCESSED) != 0); } #define __HAVE_ARCH_PMDP_SET_WRPROTECT static inline void pmdp_set_wrprotect(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp) { if (__pmd_write((*pmdp))) pmd_hugepage_update(mm, addr, pmdp, _PAGE_WRITE, 0); else if (unlikely(pmd_savedwrite(*pmdp))) pmd_hugepage_update(mm, addr, pmdp, 0, _PAGE_PRIVILEGED); } /* * Only returns true for a THP. False for pmd migration entry. * We also need to return true when we come across a pte that * in between a thp split. While splitting THP, we mark the pmd * invalid (pmdp_invalidate()) before we set it with pte page * address. A pmd_trans_huge() check against a pmd entry during that time * should return true. * We should not call this on a hugetlb entry. We should check for HugeTLB * entry using vma->vm_flags * The page table walk rule is explained in Documentation/vm/transhuge.rst */ static inline int pmd_trans_huge(pmd_t pmd) { if (!pmd_present(pmd)) return false; if (radix_enabled()) return radix__pmd_trans_huge(pmd); return hash__pmd_trans_huge(pmd); } #define __HAVE_ARCH_PMD_SAME static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b) { if (radix_enabled()) return radix__pmd_same(pmd_a, pmd_b); return hash__pmd_same(pmd_a, pmd_b); } static inline pmd_t __pmd_mkhuge(pmd_t pmd) { if (radix_enabled()) return radix__pmd_mkhuge(pmd); return hash__pmd_mkhuge(pmd); } /* * pfn_pmd return a pmd_t that can be used as pmd pte entry. */ static inline pmd_t pmd_mkhuge(pmd_t pmd) { #ifdef CONFIG_DEBUG_VM if (radix_enabled()) WARN_ON((pmd_raw(pmd) & cpu_to_be64(_PAGE_PTE)) == 0); else WARN_ON((pmd_raw(pmd) & cpu_to_be64(_PAGE_PTE | H_PAGE_THP_HUGE)) != cpu_to_be64(_PAGE_PTE | H_PAGE_THP_HUGE)); #endif return pmd; } #define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS extern int pmdp_set_access_flags(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp, pmd_t entry, int dirty); #define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG extern int pmdp_test_and_clear_young(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp); #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp) { if (radix_enabled()) return radix__pmdp_huge_get_and_clear(mm, addr, pmdp); return hash__pmdp_huge_get_and_clear(mm, addr, pmdp); } static inline pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp) { if (radix_enabled()) return radix__pmdp_collapse_flush(vma, address, pmdp); return hash__pmdp_collapse_flush(vma, address, pmdp); } #define pmdp_collapse_flush pmdp_collapse_flush #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR_FULL pmd_t pmdp_huge_get_and_clear_full(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmdp, int full); #define __HAVE_ARCH_PGTABLE_DEPOSIT static inline void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, pgtable_t pgtable) { if (radix_enabled()) return radix__pgtable_trans_huge_deposit(mm, pmdp, pgtable); return hash__pgtable_trans_huge_deposit(mm, pmdp, pgtable); } #define __HAVE_ARCH_PGTABLE_WITHDRAW static inline pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) { if (radix_enabled()) return radix__pgtable_trans_huge_withdraw(mm, pmdp); return hash__pgtable_trans_huge_withdraw(mm, pmdp); } #define __HAVE_ARCH_PMDP_INVALIDATE extern pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp); #define pmd_move_must_withdraw pmd_move_must_withdraw struct spinlock; extern int pmd_move_must_withdraw(struct spinlock *new_pmd_ptl, struct spinlock *old_pmd_ptl, struct vm_area_struct *vma); /* * Hash translation mode use the deposited table to store hash pte * slot information. */ #define arch_needs_pgtable_deposit arch_needs_pgtable_deposit static inline bool arch_needs_pgtable_deposit(void) { if (radix_enabled()) return false; return true; } extern void serialize_against_pte_lookup(struct mm_struct *mm); static inline pmd_t pmd_mkdevmap(pmd_t pmd) { if (radix_enabled()) return radix__pmd_mkdevmap(pmd); return hash__pmd_mkdevmap(pmd); } static inline int pmd_devmap(pmd_t pmd) { return pte_devmap(pmd_pte(pmd)); } static inline int pud_devmap(pud_t pud) { return 0; } static inline int pgd_devmap(pgd_t pgd) { return 0; } #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ static inline int pud_pfn(pud_t pud) { /* * Currently all calls to pud_pfn() are gated around a pud_devmap() * check so this should never be used. If it grows another user we * want to know about it. */ BUILD_BUG(); return 0; } #define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION pte_t ptep_modify_prot_start(struct vm_area_struct *, unsigned long, pte_t *); void ptep_modify_prot_commit(struct vm_area_struct *, unsigned long, pte_t *, pte_t, pte_t); /* * Returns true for a R -> RW upgrade of pte */ static inline bool is_pte_rw_upgrade(unsigned long old_val, unsigned long new_val) { if (!(old_val & _PAGE_READ)) return false; if ((!(old_val & _PAGE_WRITE)) && (new_val & _PAGE_WRITE)) return true; return false; } /* * Like pmd_huge() and pmd_large(), but works regardless of config options */ #define pmd_is_leaf pmd_is_leaf #define pmd_leaf pmd_is_leaf static inline bool pmd_is_leaf(pmd_t pmd) { return !!(pmd_raw(pmd) & cpu_to_be64(_PAGE_PTE)); } #define pud_is_leaf pud_is_leaf #define pud_leaf pud_is_leaf static inline bool pud_is_leaf(pud_t pud) { return !!(pud_raw(pud) & cpu_to_be64(_PAGE_PTE)); } #define p4d_is_leaf p4d_is_leaf #define p4d_leaf p4d_is_leaf static inline bool p4d_is_leaf(p4d_t p4d) { return !!(p4d_raw(p4d) & cpu_to_be64(_PAGE_PTE)); } #endif /* __ASSEMBLY__ */ #endif /* _ASM_POWERPC_BOOK3S_64_PGTABLE_H_ */ PK ! I�OP:( :( kup.hnu �[��� /* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_BOOK3S_64_KUP_H #define _ASM_POWERPC_BOOK3S_64_KUP_H #include <linux/const.h> #include <asm/reg.h> #define AMR_KUAP_BLOCK_READ UL(0x5455555555555555) #define AMR_KUAP_BLOCK_WRITE UL(0xa8aaaaaaaaaaaaaa) #define AMR_KUEP_BLOCKED UL(0x5455555555555555) #define AMR_KUAP_BLOCKED (AMR_KUAP_BLOCK_READ | AMR_KUAP_BLOCK_WRITE) #ifdef __ASSEMBLY__ .macro kuap_user_restore gpr1, gpr2 #if defined(CONFIG_PPC_PKEY) BEGIN_MMU_FTR_SECTION_NESTED(67) b 100f // skip_restore_amr END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_PKEY, 67) /* * AMR and IAMR are going to be different when * returning to userspace. */ ld \gpr1, STACK_REGS_AMR(r1) /* * If kuap feature is not enabled, do the mtspr * only if AMR value is different. */ BEGIN_MMU_FTR_SECTION_NESTED(68) mfspr \gpr2, SPRN_AMR cmpd \gpr1, \gpr2 beq 99f END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_BOOK3S_KUAP, 68) isync mtspr SPRN_AMR, \gpr1 99: /* * Restore IAMR only when returning to userspace */ ld \gpr1, STACK_REGS_IAMR(r1) /* * If kuep feature is not enabled, do the mtspr * only if IAMR value is different. */ BEGIN_MMU_FTR_SECTION_NESTED(69) mfspr \gpr2, SPRN_IAMR cmpd \gpr1, \gpr2 beq 100f END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_BOOK3S_KUEP, 69) isync mtspr SPRN_IAMR, \gpr1 100: //skip_restore_amr /* No isync required, see kuap_user_restore() */ #endif .endm .macro kuap_kernel_restore gpr1, gpr2 #if defined(CONFIG_PPC_PKEY) BEGIN_MMU_FTR_SECTION_NESTED(67) /* * AMR is going to be mostly the same since we are * returning to the kernel. Compare and do a mtspr. */ ld \gpr2, STACK_REGS_AMR(r1) mfspr \gpr1, SPRN_AMR cmpd \gpr1, \gpr2 beq 100f isync mtspr SPRN_AMR, \gpr2 /* * No isync required, see kuap_restore_amr() * No need to restore IAMR when returning to kernel space. */ 100: END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_BOOK3S_KUAP, 67) #endif .endm #ifdef CONFIG_PPC_KUAP .macro kuap_check_amr gpr1, gpr2 #ifdef CONFIG_PPC_KUAP_DEBUG BEGIN_MMU_FTR_SECTION_NESTED(67) mfspr \gpr1, SPRN_AMR /* Prevent access to userspace using any key values */ LOAD_REG_IMMEDIATE(\gpr2, AMR_KUAP_BLOCKED) 999: tdne \gpr1, \gpr2 EMIT_WARN_ENTRY 999b, __FILE__, __LINE__, (BUGFLAG_WARNING | BUGFLAG_ONCE) END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_BOOK3S_KUAP, 67) #endif .endm #endif /* * if (pkey) { * * save AMR -> stack; * if (kuap) { * if (AMR != BLOCKED) * KUAP_BLOCKED -> AMR; * } * if (from_user) { * save IAMR -> stack; * if (kuep) { * KUEP_BLOCKED ->IAMR * } * } * return; * } * * if (kuap) { * if (from_kernel) { * save AMR -> stack; * if (AMR != BLOCKED) * KUAP_BLOCKED -> AMR; * } * * } */ .macro kuap_save_amr_and_lock gpr1, gpr2, use_cr, msr_pr_cr #if defined(CONFIG_PPC_PKEY) /* * if both pkey and kuap is disabled, nothing to do */ BEGIN_MMU_FTR_SECTION_NESTED(68) b 100f // skip_save_amr END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_PKEY | MMU_FTR_BOOK3S_KUAP, 68) /* * if pkey is disabled and we are entering from userspace * don't do anything. */ BEGIN_MMU_FTR_SECTION_NESTED(67) .ifnb \msr_pr_cr /* * Without pkey we are not changing AMR outside the kernel * hence skip this completely. */ bne \msr_pr_cr, 100f // from userspace .endif END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_PKEY, 67) /* * pkey is enabled or pkey is disabled but entering from kernel */ mfspr \gpr1, SPRN_AMR std \gpr1, STACK_REGS_AMR(r1) /* * update kernel AMR with AMR_KUAP_BLOCKED only * if KUAP feature is enabled */ BEGIN_MMU_FTR_SECTION_NESTED(69) LOAD_REG_IMMEDIATE(\gpr2, AMR_KUAP_BLOCKED) cmpd \use_cr, \gpr1, \gpr2 beq \use_cr, 102f /* * We don't isync here because we very recently entered via an interrupt */ mtspr SPRN_AMR, \gpr2 isync 102: END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_BOOK3S_KUAP, 69) /* * if entering from kernel we don't need save IAMR */ .ifnb \msr_pr_cr beq \msr_pr_cr, 100f // from kernel space mfspr \gpr1, SPRN_IAMR std \gpr1, STACK_REGS_IAMR(r1) /* * update kernel IAMR with AMR_KUEP_BLOCKED only * if KUEP feature is enabled */ BEGIN_MMU_FTR_SECTION_NESTED(70) LOAD_REG_IMMEDIATE(\gpr2, AMR_KUEP_BLOCKED) mtspr SPRN_IAMR, \gpr2 isync END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_BOOK3S_KUEP, 70) .endif 100: // skip_save_amr #endif .endm #else /* !__ASSEMBLY__ */ #include <linux/jump_label.h> DECLARE_STATIC_KEY_FALSE(uaccess_flush_key); #ifdef CONFIG_PPC_PKEY extern u64 __ro_after_init default_uamor; extern u64 __ro_after_init default_amr; extern u64 __ro_after_init default_iamr; #include <asm/mmu.h> #include <asm/ptrace.h> /* usage of kthread_use_mm() should inherit the * AMR value of the operating address space. But, the AMR value is * thread-specific and we inherit the address space and not thread * access restrictions. Because of this ignore AMR value when accessing * userspace via kernel thread. */ static inline u64 current_thread_amr(void) { if (current->thread.regs) return current->thread.regs->amr; return default_amr; } static inline u64 current_thread_iamr(void) { if (current->thread.regs) return current->thread.regs->iamr; return default_iamr; } #endif /* CONFIG_PPC_PKEY */ #ifdef CONFIG_PPC_KUAP static inline void kuap_user_restore(struct pt_regs *regs) { bool restore_amr = false, restore_iamr = false; unsigned long amr, iamr; if (!mmu_has_feature(MMU_FTR_PKEY)) return; if (!mmu_has_feature(MMU_FTR_BOOK3S_KUAP)) { amr = mfspr(SPRN_AMR); if (amr != regs->amr) restore_amr = true; } else { restore_amr = true; } if (!mmu_has_feature(MMU_FTR_BOOK3S_KUEP)) { iamr = mfspr(SPRN_IAMR); if (iamr != regs->iamr) restore_iamr = true; } else { restore_iamr = true; } if (restore_amr || restore_iamr) { isync(); if (restore_amr) mtspr(SPRN_AMR, regs->amr); if (restore_iamr) mtspr(SPRN_IAMR, regs->iamr); } /* * No isync required here because we are about to rfi * back to previous context before any user accesses * would be made, which is a CSI. */ } static inline void kuap_kernel_restore(struct pt_regs *regs, unsigned long amr) { if (mmu_has_feature(MMU_FTR_BOOK3S_KUAP)) { if (unlikely(regs->amr != amr)) { isync(); mtspr(SPRN_AMR, regs->amr); /* * No isync required here because we are about to rfi * back to previous context before any user accesses * would be made, which is a CSI. */ } } /* * No need to restore IAMR when returning to kernel space. */ } static inline unsigned long kuap_get_and_assert_locked(void) { if (mmu_has_feature(MMU_FTR_BOOK3S_KUAP)) { unsigned long amr = mfspr(SPRN_AMR); if (IS_ENABLED(CONFIG_PPC_KUAP_DEBUG)) /* kuap_check_amr() */ WARN_ON_ONCE(amr != AMR_KUAP_BLOCKED); return amr; } return 0; } static inline void kuap_assert_locked(void) { if (IS_ENABLED(CONFIG_PPC_KUAP_DEBUG) && mmu_has_feature(MMU_FTR_BOOK3S_KUAP)) WARN_ON_ONCE(mfspr(SPRN_AMR) != AMR_KUAP_BLOCKED); } /* * We support individually allowing read or write, but we don't support nesting * because that would require an expensive read/modify write of the AMR. */ static inline unsigned long get_kuap(void) { /* * We return AMR_KUAP_BLOCKED when we don't support KUAP because * prevent_user_access_return needs to return AMR_KUAP_BLOCKED to * cause restore_user_access to do a flush. * * This has no effect in terms of actually blocking things on hash, * so it doesn't break anything. */ if (!mmu_has_feature(MMU_FTR_BOOK3S_KUAP)) return AMR_KUAP_BLOCKED; return mfspr(SPRN_AMR); } static inline void set_kuap(unsigned long value) { if (!mmu_has_feature(MMU_FTR_BOOK3S_KUAP)) return; /* * ISA v3.0B says we need a CSI (Context Synchronising Instruction) both * before and after the move to AMR. See table 6 on page 1134. */ isync(); mtspr(SPRN_AMR, value); isync(); } static inline bool bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write) { if (!mmu_has_feature(MMU_FTR_BOOK3S_KUAP)) return false; /* * For radix this will be a storage protection fault (DSISR_PROTFAULT). * For hash this will be a key fault (DSISR_KEYFAULT) */ /* * We do have exception table entry, but accessing the * userspace results in fault. This could be because we * didn't unlock the AMR or access is denied by userspace * using a key value that blocks access. We are only interested * in catching the use case of accessing without unlocking * the AMR. Hence check for BLOCK_WRITE/READ against AMR. */ if (is_write) { return (regs->amr & AMR_KUAP_BLOCK_WRITE) == AMR_KUAP_BLOCK_WRITE; } return (regs->amr & AMR_KUAP_BLOCK_READ) == AMR_KUAP_BLOCK_READ; } static __always_inline void allow_user_access(void __user *to, const void __user *from, unsigned long size, unsigned long dir) { unsigned long thread_amr = 0; // This is written so we can resolve to a single case at build time BUILD_BUG_ON(!__builtin_constant_p(dir)); if (mmu_has_feature(MMU_FTR_PKEY)) thread_amr = current_thread_amr(); if (dir == KUAP_READ) set_kuap(thread_amr | AMR_KUAP_BLOCK_WRITE); else if (dir == KUAP_WRITE) set_kuap(thread_amr | AMR_KUAP_BLOCK_READ); else if (dir == KUAP_READ_WRITE) set_kuap(thread_amr); else BUILD_BUG(); } #else /* CONFIG_PPC_KUAP */ static inline unsigned long get_kuap(void) { return AMR_KUAP_BLOCKED; } static inline void set_kuap(unsigned long value) { } static __always_inline void allow_user_access(void __user *to, const void __user *from, unsigned long size, unsigned long dir) { } #endif /* !CONFIG_PPC_KUAP */ static inline void prevent_user_access(unsigned long dir) { set_kuap(AMR_KUAP_BLOCKED); if (static_branch_unlikely(&uaccess_flush_key)) do_uaccess_flush(); } static inline unsigned long prevent_user_access_return(void) { unsigned long flags = get_kuap(); set_kuap(AMR_KUAP_BLOCKED); if (static_branch_unlikely(&uaccess_flush_key)) do_uaccess_flush(); return flags; } static inline void restore_user_access(unsigned long flags) { set_kuap(flags); if (static_branch_unlikely(&uaccess_flush_key) && flags == AMR_KUAP_BLOCKED) do_uaccess_flush(); } #endif /* __ASSEMBLY__ */ #endif /* _ASM_POWERPC_BOOK3S_64_KUP_H */ PK ! y ��[ [ pgalloc.hnu �[��� /* SPDX-License-Identifier: GPL-2.0-or-later */ #ifndef _ASM_POWERPC_BOOK3S_64_PGALLOC_H #define _ASM_POWERPC_BOOK3S_64_PGALLOC_H /* */ #include <linux/slab.h> #include <linux/cpumask.h> #include <linux/kmemleak.h> #include <linux/percpu.h> struct vmemmap_backing { struct vmemmap_backing *list; unsigned long phys; unsigned long virt_addr; }; extern struct vmemmap_backing *vmemmap_list; extern pmd_t *pmd_fragment_alloc(struct mm_struct *, unsigned long); extern void pmd_fragment_free(unsigned long *); extern void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift); extern void __tlb_remove_table(void *_table); void pte_frag_destroy(void *pte_frag); static inline pgd_t *radix__pgd_alloc(struct mm_struct *mm) { #ifdef CONFIG_PPC_64K_PAGES return (pgd_t *)__get_free_page(pgtable_gfp_flags(mm, PGALLOC_GFP)); #else struct page *page; page = alloc_pages(pgtable_gfp_flags(mm, PGALLOC_GFP | __GFP_RETRY_MAYFAIL), 4); if (!page) return NULL; return (pgd_t *) page_address(page); #endif } static inline void radix__pgd_free(struct mm_struct *mm, pgd_t *pgd) { #ifdef CONFIG_PPC_64K_PAGES free_page((unsigned long)pgd); #else free_pages((unsigned long)pgd, 4); #endif } static inline pgd_t *pgd_alloc(struct mm_struct *mm) { pgd_t *pgd; if (radix_enabled()) return radix__pgd_alloc(mm); pgd = kmem_cache_alloc(PGT_CACHE(PGD_INDEX_SIZE), pgtable_gfp_flags(mm, GFP_KERNEL)); if (unlikely(!pgd)) return pgd; /* * Don't scan the PGD for pointers, it contains references to PUDs but * those references are not full pointers and so can't be recognised by * kmemleak. */ kmemleak_no_scan(pgd); /* * With hugetlb, we don't clear the second half of the page table. * If we share the same slab cache with the pmd or pud level table, * we need to make sure we zero out the full table on alloc. * With 4K we don't store slot in the second half. Hence we don't * need to do this for 4k. */ #if defined(CONFIG_HUGETLB_PAGE) && defined(CONFIG_PPC_64K_PAGES) && \ (H_PGD_INDEX_SIZE == H_PUD_CACHE_INDEX) memset(pgd, 0, PGD_TABLE_SIZE); #endif return pgd; } static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) { if (radix_enabled()) return radix__pgd_free(mm, pgd); kmem_cache_free(PGT_CACHE(PGD_INDEX_SIZE), pgd); } static inline void p4d_populate(struct mm_struct *mm, p4d_t *pgd, pud_t *pud) { *pgd = __p4d(__pgtable_ptr_val(pud) | PGD_VAL_BITS); } static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr) { pud_t *pud; pud = kmem_cache_alloc(PGT_CACHE(PUD_CACHE_INDEX), pgtable_gfp_flags(mm, GFP_KERNEL)); /* * Tell kmemleak to ignore the PUD, that means don't scan it for * pointers and don't consider it a leak. PUDs are typically only * referred to by their PGD, but kmemleak is not able to recognise those * as pointers, leading to false leak reports. */ kmemleak_ignore(pud); return pud; } static inline void __pud_free(pud_t *pud) { struct page *page = virt_to_page(pud); /* * Early pud pages allocated via memblock allocator * can't be directly freed to slab */ if (PageReserved(page)) free_reserved_page(page); else kmem_cache_free(PGT_CACHE(PUD_CACHE_INDEX), pud); } static inline void pud_free(struct mm_struct *mm, pud_t *pud) { return __pud_free(pud); } static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) { *pud = __pud(__pgtable_ptr_val(pmd) | PUD_VAL_BITS); } static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud, unsigned long address) { pgtable_free_tlb(tlb, pud, PUD_INDEX); } static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) { return pmd_fragment_alloc(mm, addr); } static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) { pmd_fragment_free((unsigned long *)pmd); } static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd, unsigned long address) { return pgtable_free_tlb(tlb, pmd, PMD_INDEX); } static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, pte_t *pte) { *pmd = __pmd(__pgtable_ptr_val(pte) | PMD_VAL_BITS); } static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, pgtable_t pte_page) { *pmd = __pmd(__pgtable_ptr_val(pte_page) | PMD_VAL_BITS); } static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table, unsigned long address) { pgtable_free_tlb(tlb, table, PTE_INDEX); } extern atomic_long_t direct_pages_count[MMU_PAGE_COUNT]; static inline void update_page_count(int psize, long count) { if (IS_ENABLED(CONFIG_PROC_FS)) atomic_long_add(count, &direct_pages_count[psize]); } #endif /* _ASM_POWERPC_BOOK3S_64_PGALLOC_H */ PK ! ��� � hugetlb.hnu �[��� /* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_BOOK3S_64_HUGETLB_H #define _ASM_POWERPC_BOOK3S_64_HUGETLB_H /* * For radix we want generic code to handle hugetlb. But then if we want * both hash and radix to be enabled together we need to workaround the * limitations. */ void radix__flush_hugetlb_page(struct vm_area_struct *vma, unsigned long vmaddr); void radix__local_flush_hugetlb_page(struct vm_area_struct *vma, unsigned long vmaddr); extern unsigned long radix__hugetlb_get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags); extern void radix__huge_ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, pte_t old_pte, pte_t pte); static inline int hstate_get_psize(struct hstate *hstate) { unsigned long shift; shift = huge_page_shift(hstate); if (shift == mmu_psize_defs[MMU_PAGE_2M].shift) return MMU_PAGE_2M; else if (shift == mmu_psize_defs[MMU_PAGE_1G].shift) return MMU_PAGE_1G; else if (shift == mmu_psize_defs[MMU_PAGE_16M].shift) return MMU_PAGE_16M; else if (shift == mmu_psize_defs[MMU_PAGE_16G].shift) return MMU_PAGE_16G; else { WARN(1, "Wrong huge page shift\n"); return mmu_virtual_psize; } } #define __HAVE_ARCH_GIGANTIC_PAGE_RUNTIME_SUPPORTED static inline bool gigantic_page_runtime_supported(void) { /* * We used gigantic page reservation with hypervisor assist in some case. * We cannot use runtime allocation of gigantic pages in those platforms * This is hash translation mode LPARs. */ if (firmware_has_feature(FW_FEATURE_LPAR) && !radix_enabled()) return false; return true; } /* hugepd entry valid bit */ #define HUGEPD_VAL_BITS (0x8000000000000000UL) #define huge_ptep_modify_prot_start huge_ptep_modify_prot_start extern pte_t huge_ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep); #define huge_ptep_modify_prot_commit huge_ptep_modify_prot_commit extern void huge_ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, pte_t old_pte, pte_t new_pte); /* * This should work for other subarchs too. But right now we use the * new format only for 64bit book3s */ static inline pte_t *hugepd_page(hugepd_t hpd) { BUG_ON(!hugepd_ok(hpd)); /* * We have only four bits to encode, MMU page size */ BUILD_BUG_ON((MMU_PAGE_COUNT - 1) > 0xf); return __va(hpd_val(hpd) & HUGEPD_ADDR_MASK); } static inline unsigned int hugepd_mmu_psize(hugepd_t hpd) { return (hpd_val(hpd) & HUGEPD_SHIFT_MASK) >> 2; } static inline unsigned int hugepd_shift(hugepd_t hpd) { return mmu_psize_to_shift(hugepd_mmu_psize(hpd)); } static inline void flush_hugetlb_page(struct vm_area_struct *vma, unsigned long vmaddr) { if (radix_enabled()) return radix__flush_hugetlb_page(vma, vmaddr); } static inline pte_t *hugepte_offset(hugepd_t hpd, unsigned long addr, unsigned int pdshift) { unsigned long idx = (addr & ((1UL << pdshift) - 1)) >> hugepd_shift(hpd); return hugepd_page(hpd) + idx; } static inline void hugepd_populate(hugepd_t *hpdp, pte_t *new, unsigned int pshift) { *hpdp = __hugepd(__pa(new) | HUGEPD_VAL_BITS | (shift_to_mmu_psize(pshift) << 2)); } void flush_hugetlb_page(struct vm_area_struct *vma, unsigned long vmaddr); static inline int check_and_get_huge_psize(int shift) { int mmu_psize; if (shift > SLICE_HIGH_SHIFT) return -EINVAL; mmu_psize = shift_to_mmu_psize(shift); /* * We need to make sure that for different page sizes reported by * firmware we only add hugetlb support for page sizes that can be * supported by linux page table layout. * For now we have * Radix: 2M and 1G * Hash: 16M and 16G */ if (radix_enabled()) { if (mmu_psize != MMU_PAGE_2M && mmu_psize != MMU_PAGE_1G) return -EINVAL; } else { if (mmu_psize != MMU_PAGE_16M && mmu_psize != MMU_PAGE_16G) return -EINVAL; } return mmu_psize; } #endif PK ! `��� � mmu.hnu �[��� PK ! � �}p }p � mmu-hash.hnu �[��� PK ! ���߉ � �� pgtable-64k.hnu �[��� PK ! ���� � O� tlbflush-hash.hnu �[��� PK ! ;��' �' �� radix.hnu �[��� PK ! �ɼ* * `� tlbflush-radix.hnu �[��� PK ! ^�y �� tlbflush.hnu �[��� PK ! �PY Y � radix-4k.hnu �[��� PK ! ��f�3 3 �� slice.hnu �[��� PK ! ~O�t � pkeys.hnu �[��� PK ! �� }* * 9� hash.hnu �[��� PK ! O0I� � � radix-64k.hnu �[��� PK ! �|C� � g pgtable-4k.hnu �[��� PK ! 4���| | � hash-4k.hnu �[��� PK ! �f>�( �( 8. hash-64k.hnu �[��� PK ! �3P� 'W hash-pkey.hnu �[��� PK ! uI4�- - r] kexec.hnu �[��� PK ! r*w�W� W� �_ pgtable.hnu �[��� PK ! I�OP:( :( f kup.hnu �[��� PK ! y ��[ [ �) pgalloc.hnu �[��� PK ! ��� � i< hugetlb.hnu �[��� PK .L
| ver. 1.1 | |
.
| PHP 8.4.18 | Ð“ÐµÐ½ÐµÑ€Ð°Ñ†Ð¸Ñ Ñтраницы: 0.02 |
proxy
|
phpinfo
|
ÐаÑтройка