diff --git a/VERSION b/VERSION index 09b254e90c6..cc94f6b803e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -6.0.0 +6.0.50 diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c index 78da27ffaf9..ca7a1d41dca 100644 --- a/accel/tcg/cpu-exec.c +++ b/accel/tcg/cpu-exec.c @@ -166,10 +166,10 @@ cpu_tb_exec(CPUState *cpu, TranslationBlock *itb, int *tb_exit) qemu_log_mask_and_addr(CPU_LOG_EXEC, itb->pc, "Trace %d: %p [" TARGET_FMT_lx "/" TARGET_FMT_lx - "/" TARGET_FMT_lx "/%#x/%#x] %s\n", + "/" TARGET_FMT_lx "-" TARGET_FMT_lx "/%#x/%#x] %s\n", cpu->cpu_index, itb->tc.ptr, itb->cs_base, itb->pc, - itb->cs_top, itb->cheri_flags, itb->flags, - lookup_symbol(itb->pc)); + itb->pcc_base, itb->pcc_top, itb->cheri_flags, + itb->flags, lookup_symbol(itb->pc)); #if defined(DEBUG_DISAS) if (qemu_loglevel_mask(CPU_LOG_TB_CPU) @@ -247,7 +247,7 @@ void cpu_exec_step_atomic(CPUState *cpu) { CPUArchState *env = (CPUArchState *)cpu->env_ptr; TranslationBlock *tb; - target_ulong cs_base, cs_top = 0, pc; + target_ulong cs_base, pcc_base = 0, pcc_top = 0, pc; uint32_t cheri_flags = 0; uint32_t flags; uint32_t cflags = (curr_cflags(cpu) & ~CF_PARALLEL) | 1; @@ -259,13 +259,15 @@ void cpu_exec_step_atomic(CPUState *cpu) g_assert(!cpu->running); cpu->running = true; - cpu_get_tb_cpu_state_6(env, &pc, &cs_base, &cs_top, &cheri_flags, &flags); - tb = tb_lookup(cpu, pc, cs_base, cs_top, cheri_flags, flags, cflags); + cpu_get_tb_cpu_state_ext(env, &pc, &cs_base, &pcc_base, &pcc_top, + &cheri_flags, &flags); + tb = tb_lookup(cpu, pc, cs_base, pcc_base, pcc_top, cheri_flags, flags, + cflags); if (tb == NULL) { mmap_lock(); - tb = tb_gen_code(cpu, pc, cs_base, cs_top, cheri_flags, flags, - cflags); + tb = tb_gen_code(cpu, pc, cs_base, pcc_base, pcc_top, cheri_flags, + flags, cflags); mmap_unlock(); } @@ -303,7 +305,8 @@ void cpu_exec_step_atomic(CPUState *cpu) struct tb_desc { target_ulong pc; target_ulong cs_base; - target_ulong cs_top; + target_ulong pcc_base; + target_ulong pcc_top; CPUArchState *env; tb_page_addr_t phys_page1; uint32_t cheri_flags; @@ -318,8 +321,9 @@ static bool tb_lookup_cmp(const void *p, const void *d) const struct tb_desc *desc = d; if (tb->pc == desc->pc && tb->page_addr[0] == desc->phys_page1 && - tb->cs_base == desc->cs_base && tb->cs_top == desc->cs_top && - tb->cheri_flags == desc->cheri_flags && tb->flags == desc->flags && + tb->cs_base == desc->cs_base && tb->pcc_base == desc->pcc_base && + tb->pcc_top == desc->pcc_top && tb->cheri_flags == desc->cheri_flags && + tb->flags == desc->flags && tb->trace_vcpu_dstate == desc->trace_vcpu_dstate && tb_cflags(tb) == desc->cflags) { /* check next page if needed */ @@ -340,9 +344,9 @@ static bool tb_lookup_cmp(const void *p, const void *d) } TranslationBlock *tb_htable_lookup(CPUState *cpu, target_ulong pc, - target_ulong cs_base, target_ulong cs_top, - uint32_t cheri_flags, uint32_t flags, - uint32_t cflags) + target_ulong cs_base, target_ulong pcc_base, + target_ulong pcc_top, uint32_t cheri_flags, + uint32_t flags, uint32_t cflags) { tb_page_addr_t phys_pc; struct tb_desc desc; @@ -350,7 +354,8 @@ TranslationBlock *tb_htable_lookup(CPUState *cpu, target_ulong pc, desc.env = (CPUArchState *)cpu->env_ptr; desc.cs_base = cs_base; - desc.cs_top = cs_top; + desc.pcc_base = pcc_base; + desc.pcc_top = pcc_top; desc.cheri_flags = cheri_flags; desc.flags = flags; desc.cflags = cflags; @@ -425,16 +430,19 @@ static inline TranslationBlock *tb_find(CPUState *cpu, { CPUArchState *env = (CPUArchState *)cpu->env_ptr; TranslationBlock *tb; - target_ulong cs_base, cs_top = 0, pc; + target_ulong cs_base, pcc_base = 0, pcc_top = 0, pc; uint32_t cheri_flags = 0; uint32_t flags; - cpu_get_tb_cpu_state_6(env, &pc, &cs_base, &cs_top, &cheri_flags, &flags); + cpu_get_tb_cpu_state_ext(env, &pc, &cs_base, &pcc_base, &pcc_top, + &cheri_flags, &flags); - tb = tb_lookup(cpu, pc, cs_base, cs_top, cheri_flags, flags, cflags); + tb = tb_lookup(cpu, pc, cs_base, pcc_base, pcc_top, cheri_flags, flags, + cflags); if (tb == NULL) { mmap_lock(); - tb = tb_gen_code(cpu, pc, cs_base, cs_top, cheri_flags, flags, cflags); + tb = tb_gen_code(cpu, pc, cs_base, pcc_base, pcc_top, cheri_flags, + flags, cflags); mmap_unlock(); /* We add the TB in the virtual pc hash table for the fast lookup */ qatomic_set(&cpu->tb_jmp_cache[tb_jmp_cache_hash_func(pc)], tb); diff --git a/accel/tcg/internal.h b/accel/tcg/internal.h index 301d9682ee3..b77cdea447e 100644 --- a/accel/tcg/internal.h +++ b/accel/tcg/internal.h @@ -12,8 +12,9 @@ #include "exec/exec-all.h" TranslationBlock *tb_gen_code(CPUState *cpu, target_ulong pc, - target_ulong cs_base, target_ulong cs_top, - uint32_t cheri_flags, uint32_t flags, int cflags); + target_ulong cs_base, target_ulong pcc_base, + target_ulong pcc_top, uint32_t cheri_flags, + uint32_t flags, int cflags); void QEMU_NORETURN cpu_io_recompile(CPUState *cpu, uintptr_t retaddr); diff --git a/accel/tcg/tcg-runtime.c b/accel/tcg/tcg-runtime.c index 12ba4585619..8bad0f9b92d 100644 --- a/accel/tcg/tcg-runtime.c +++ b/accel/tcg/tcg-runtime.c @@ -154,21 +154,23 @@ const void *HELPER(lookup_tb_ptr)(CPUArchState *env) { CPUState *cpu = env_cpu(env); TranslationBlock *tb; - target_ulong cs_base, cs_top = 0, pc; + target_ulong cs_base, pcc_base = 0, pcc_top = 0, pc; uint32_t cheri_flags = 0; uint32_t flags; - cpu_get_tb_cpu_state_6(env, &pc, &cs_base, &cs_top, &cheri_flags, &flags); + cpu_get_tb_cpu_state_ext(env, &pc, &cs_base, &pcc_base, &pcc_top, + &cheri_flags, &flags); - tb = tb_lookup(cpu, pc, cs_base, cs_top, cheri_flags, flags, curr_cflags(cpu)); + tb = tb_lookup(cpu, pc, cs_base, pcc_base, pcc_top, cheri_flags, flags, + curr_cflags(cpu)); if (tb == NULL) { return tcg_code_gen_epilogue; } qemu_log_mask_and_addr(CPU_LOG_EXEC, pc, "Chain %d: %p [" TARGET_FMT_lx "/" TARGET_FMT_lx - "/" TARGET_FMT_lx "/%#x/%#x] %s\n", - cpu->cpu_index, tb->tc.ptr, cs_base, pc, cs_top, - cheri_flags, flags, lookup_symbol(pc)); + "/" TARGET_FMT_lx "-" TARGET_FMT_lx "/%#x/%#x] %s\n", + cpu->cpu_index, tb->tc.ptr, cs_base, pc, pcc_base, + pcc_top, cheri_flags, flags, lookup_symbol(pc)); return tb->tc.ptr; } diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index 2a254563802..6df58673715 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -1309,7 +1309,8 @@ static bool tb_cmp(const void *ap, const void *bp) return a->pc == b->pc && a->cs_base == b->cs_base && - a->cs_top == b->cs_top && + a->pcc_base == b->pcc_base && + a->pcc_top == b->pcc_top && a->cheri_flags == b->cheri_flags && a->flags == b->flags && (tb_cflags(a) & ~CF_INVALID) == (tb_cflags(b) & ~CF_INVALID) && @@ -1844,8 +1845,9 @@ tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc, /* Called with mmap_lock held for user mode emulation. */ TranslationBlock *tb_gen_code(CPUState *cpu, target_ulong pc, - target_ulong cs_base, target_ulong cs_top, - uint32_t cheri_flags, uint32_t flags, int cflags) + target_ulong cs_base, target_ulong pcc_base, + target_ulong pcc_top, uint32_t cheri_flags, + uint32_t flags, int cflags) { CPUArchState *env = cpu->env_ptr; TranslationBlock *tb, *existing_tb; @@ -1894,7 +1896,8 @@ TranslationBlock *tb_gen_code(CPUState *cpu, target_ulong pc, tb->tc.ptr = tcg_splitwx_to_rx(gen_code_buf); tb->pc = pc; tb->cs_base = cs_base; - tb->cs_top = cs_top; + tb->pcc_base = pcc_base; + tb->pcc_top = pcc_top; tb->cheri_flags = cheri_flags; tb->flags = flags; tb->cflags = cflags; @@ -2134,7 +2137,8 @@ tb_invalidate_phys_page_range__locked(struct page_collection *pages, TranslationBlock *current_tb = NULL; target_ulong current_pc = 0; target_ulong current_cs_base = 0; - target_ulong current_cs_top = 0; + target_ulong current_pcc_base = 0; + target_ulong current_pcc_top = 0; uint32_t current_cheri_flags = 0; uint32_t current_flags = 0; #endif /* TARGET_HAS_PRECISE_SMC */ @@ -2180,9 +2184,9 @@ tb_invalidate_phys_page_range__locked(struct page_collection *pages, */ current_tb_modified = true; cpu_restore_state_from_tb(cpu, current_tb, retaddr, true); - cpu_get_tb_cpu_state_6(env, ¤t_pc, ¤t_cs_base, - ¤t_cs_top, ¤t_cheri_flags, - ¤t_flags); + cpu_get_tb_cpu_state_ext(env, ¤t_pc, ¤t_cs_base, + ¤t_pcc_base, ¤t_pcc_top, + ¤t_cheri_flags, ¤t_flags); } #endif /* TARGET_HAS_PRECISE_SMC */ tb_phys_invalidate__locked(tb); @@ -2325,8 +2329,8 @@ static bool tb_invalidate_phys_page(tb_page_addr_t addr, uintptr_t pc) int current_tb_modified = 0; target_ulong current_pc = 0; target_ulong current_cs_base = 0; - target_ulong current_cs_top = 0; - target_ulong current_ds_base = 0; + target_ulong current_pcc_base = 0; + target_ulong current_pcc_top = 0; uint32_t current_cheri_flags = 0; uint32_t current_flags = 0; #endif @@ -2360,9 +2364,9 @@ static bool tb_invalidate_phys_page(tb_page_addr_t addr, uintptr_t pc) current_tb_modified = 1; cpu_restore_state_from_tb(cpu, current_tb, pc, true); - cpu_get_tb_cpu_state_6(env, ¤t_pc, ¤t_cs_base, - ¤t_cs_top, ¤t_cheri_flags, - ¤t_flags); + cpu_get_tb_cpu_state_ext(env, ¤t_pc, ¤t_cs_base, + ¤t_pcc_base, ¤t_pcc_top, + ¤t_cheri_flags, ¤t_flags); } #endif /* TARGET_HAS_PRECISE_SMC */ tb_phys_invalidate(tb, addr); @@ -2396,12 +2400,13 @@ void tb_check_watchpoint(CPUState *cpu, uintptr_t retaddr) /* The exception probably happened in a helper. The CPU state should have been saved before calling it. Fetch the PC from there. */ CPUArchState *env = cpu->env_ptr; - target_ulong pc, cs_base, cs_top = 0; + target_ulong pc, cs_base, pcc_base = 0, pcc_top = 0; uint32_t cheri_flags = 0; tb_page_addr_t addr; uint32_t flags; - cpu_get_tb_cpu_state_6(env, &pc, &cs_base, &cs_top, &cheri_flags, &flags); + cpu_get_tb_cpu_state_ext(env, &pc, &cs_base, &pcc_base, &pcc_top, + &cheri_flags, &flags); addr = get_page_addr_code(env, pc); if (addr != -1) { tb_invalidate_phys_range(addr, addr + 1); diff --git a/accel/tcg/translator.c b/accel/tcg/translator.c index 4fd503b9ff2..e27d422f74e 100644 --- a/accel/tcg/translator.c +++ b/accel/tcg/translator.c @@ -58,8 +58,8 @@ void translator_loop(const TranslatorOps *ops, DisasContextBase *db, db->max_insns = max_insns; db->singlestep_enabled = cpu->singlestep_enabled; #ifdef TARGET_CHERI - db->pcc_base = tb->cs_base; - db->pcc_top = tb->cs_top; + db->pcc_base = tb->pcc_base; + db->pcc_top = tb->pcc_top; cheri_debug_assert(db->pcc_base == cap_get_base(cheri_get_recent_pcc(cpu->env_ptr))); cheri_debug_assert(db->pcc_top == diff --git a/hw/arm/smmuv3.c b/hw/arm/smmuv3.c index 87056125357..228dc54b0bc 100644 --- a/hw/arm/smmuv3.c +++ b/hw/arm/smmuv3.c @@ -259,8 +259,9 @@ static void smmuv3_init_regs(SMMUv3State *s) s->idr[3] = FIELD_DP32(s->idr[3], IDR3, RIL, 1); s->idr[3] = FIELD_DP32(s->idr[3], IDR3, HAD, 1); - /* 4K and 64K granule support */ + /* 4K, 16K and 64K granule support */ s->idr[5] = FIELD_DP32(s->idr[5], IDR5, GRAN4K, 1); + s->idr[5] = FIELD_DP32(s->idr[5], IDR5, GRAN16K, 1); s->idr[5] = FIELD_DP32(s->idr[5], IDR5, GRAN64K, 1); s->idr[5] = FIELD_DP32(s->idr[5], IDR5, OAS, SMMU_IDR5_OAS); /* 44 bits */ @@ -503,7 +504,8 @@ static int decode_cd(SMMUTransCfg *cfg, CD *cd, SMMUEventInfo *event) tg = CD_TG(cd, i); tt->granule_sz = tg2granule(tg, i); - if ((tt->granule_sz != 12 && tt->granule_sz != 16) || CD_ENDI(cd)) { + if ((tt->granule_sz != 12 && tt->granule_sz != 14 && + tt->granule_sz != 16) || CD_ENDI(cd)) { goto bad_cd; } diff --git a/hw/arm/virt.c b/hw/arm/virt.c index a30a254600c..13c3102c30c 100644 --- a/hw/arm/virt.c +++ b/hw/arm/virt.c @@ -2772,10 +2772,15 @@ static void machvirt_machine_init(void) } type_init(machvirt_machine_init); +static void virt_machine_6_1_options(MachineClass *mc) +{ +} +DEFINE_VIRT_MACHINE_AS_LATEST(6, 1) + static void virt_machine_6_0_options(MachineClass *mc) { } -DEFINE_VIRT_MACHINE_AS_LATEST(6, 0) +DEFINE_VIRT_MACHINE(6, 0) static void virt_machine_5_2_options(MachineClass *mc) { diff --git a/hw/core/machine.c b/hw/core/machine.c index 40def78183a..0f5ce43d0c2 100644 --- a/hw/core/machine.c +++ b/hw/core/machine.c @@ -36,6 +36,11 @@ #include "hw/virtio/virtio.h" #include "hw/virtio/virtio-pci.h" +GlobalProperty hw_compat_6_0[] = { + { "gpex-pcihost", "allow-unmapped-accesses", "false" }, +}; +const size_t hw_compat_6_0_len = G_N_ELEMENTS(hw_compat_6_0); + GlobalProperty hw_compat_5_2[] = { { "ICH9-LPC", "smm-compat", "on"}, { "PIIX4_PM", "smm-compat", "on"}, diff --git a/hw/i386/pc.c b/hw/i386/pc.c index 8a84b25a031..364816efc9d 100644 --- a/hw/i386/pc.c +++ b/hw/i386/pc.c @@ -96,6 +96,9 @@ #include "trace.h" #include CONFIG_DEVICES +GlobalProperty pc_compat_6_0[] = {}; +const size_t pc_compat_6_0_len = G_N_ELEMENTS(pc_compat_6_0); + GlobalProperty pc_compat_5_2[] = { { "ICH9-LPC", "x-smi-cpu-hotunplug", "off" }, }; diff --git a/hw/i386/pc_piix.c b/hw/i386/pc_piix.c index 46cc951073b..4e8edffeaf6 100644 --- a/hw/i386/pc_piix.c +++ b/hw/i386/pc_piix.c @@ -415,7 +415,7 @@ static void pc_i440fx_machine_options(MachineClass *m) machine_class_allow_dynamic_sysbus_dev(m, TYPE_VMBUS_BRIDGE); } -static void pc_i440fx_6_0_machine_options(MachineClass *m) +static void pc_i440fx_6_1_machine_options(MachineClass *m) { PCMachineClass *pcmc = PC_MACHINE_CLASS(m); pc_i440fx_machine_options(m); @@ -424,6 +424,18 @@ static void pc_i440fx_6_0_machine_options(MachineClass *m) pcmc->default_cpu_version = 1; } +DEFINE_I440FX_MACHINE(v6_1, "pc-i440fx-6.1", NULL, + pc_i440fx_6_1_machine_options); + +static void pc_i440fx_6_0_machine_options(MachineClass *m) +{ + pc_i440fx_6_1_machine_options(m); + m->alias = NULL; + m->is_default = false; + compat_props_add(m->compat_props, hw_compat_6_0, hw_compat_6_0_len); + compat_props_add(m->compat_props, pc_compat_6_0, pc_compat_6_0_len); +} + DEFINE_I440FX_MACHINE(v6_0, "pc-i440fx-6.0", NULL, pc_i440fx_6_0_machine_options); diff --git a/hw/i386/pc_q35.c b/hw/i386/pc_q35.c index 53450190f54..458ed41c65d 100644 --- a/hw/i386/pc_q35.c +++ b/hw/i386/pc_q35.c @@ -345,7 +345,7 @@ static void pc_q35_machine_options(MachineClass *m) m->max_cpus = 288; } -static void pc_q35_6_0_machine_options(MachineClass *m) +static void pc_q35_6_1_machine_options(MachineClass *m) { PCMachineClass *pcmc = PC_MACHINE_CLASS(m); pc_q35_machine_options(m); @@ -353,6 +353,17 @@ static void pc_q35_6_0_machine_options(MachineClass *m) pcmc->default_cpu_version = 1; } +DEFINE_Q35_MACHINE(v6_1, "pc-q35-6.1", NULL, + pc_q35_6_1_machine_options); + +static void pc_q35_6_0_machine_options(MachineClass *m) +{ + pc_q35_6_1_machine_options(m); + m->alias = NULL; + compat_props_add(m->compat_props, hw_compat_6_0, hw_compat_6_0_len); + compat_props_add(m->compat_props, pc_compat_6_0, pc_compat_6_0_len); +} + DEFINE_Q35_MACHINE(v6_0, "pc-q35-6.0", NULL, pc_q35_6_0_machine_options); diff --git a/hw/pci-host/gpex.c b/hw/pci-host/gpex.c index 2bdbe7b4561..a6752fac5e8 100644 --- a/hw/pci-host/gpex.c +++ b/hw/pci-host/gpex.c @@ -83,12 +83,51 @@ static void gpex_host_realize(DeviceState *dev, Error **errp) int i; pcie_host_mmcfg_init(pex, PCIE_MMCFG_SIZE_MAX); + sysbus_init_mmio(sbd, &pex->mmio); + + /* + * Note that the MemoryRegions io_mmio and io_ioport that we pass + * to pci_register_root_bus() are not the same as the + * MemoryRegions io_mmio_window and io_ioport_window that we + * expose as SysBus MRs. The difference is in the behaviour of + * accesses to addresses where no PCI device has been mapped. + * + * io_mmio and io_ioport are the underlying PCI view of the PCI + * address space, and when a PCI device does a bus master access + * to a bad address this is reported back to it as a transaction + * failure. + * + * io_mmio_window and io_ioport_window implement "unmapped + * addresses read as -1 and ignore writes"; this is traditional + * x86 PC behaviour, which is not mandated by the PCI spec proper + * but expected by much PCI-using guest software, including Linux. + * + * In the interests of not being unnecessarily surprising, we + * implement it in the gpex PCI host controller, by providing the + * _window MRs, which are containers with io ops that implement + * the 'background' behaviour and which hold the real PCI MRs as + * subregions. + */ memory_region_init(&s->io_mmio, OBJECT(s), "gpex_mmio", UINT64_MAX); memory_region_init(&s->io_ioport, OBJECT(s), "gpex_ioport", 64 * 1024); - sysbus_init_mmio(sbd, &pex->mmio); - sysbus_init_mmio(sbd, &s->io_mmio); - sysbus_init_mmio(sbd, &s->io_ioport); + if (s->allow_unmapped_accesses) { + memory_region_init_io(&s->io_mmio_window, OBJECT(s), + &unassigned_io_ops, OBJECT(s), + "gpex_mmio_window", UINT64_MAX); + memory_region_init_io(&s->io_ioport_window, OBJECT(s), + &unassigned_io_ops, OBJECT(s), + "gpex_ioport_window", 64 * 1024); + + memory_region_add_subregion(&s->io_mmio_window, 0, &s->io_mmio); + memory_region_add_subregion(&s->io_ioport_window, 0, &s->io_ioport); + sysbus_init_mmio(sbd, &s->io_mmio_window); + sysbus_init_mmio(sbd, &s->io_ioport_window); + } else { + sysbus_init_mmio(sbd, &s->io_mmio); + sysbus_init_mmio(sbd, &s->io_ioport); + } + for (i = 0; i < GPEX_NUM_IRQS; i++) { sysbus_init_irq(sbd, &s->irq[i]); s->irq_num[i] = -1; @@ -108,6 +147,16 @@ static const char *gpex_host_root_bus_path(PCIHostState *host_bridge, return "0000:00"; } +static Property gpex_host_properties[] = { + /* + * Permit CPU accesses to unmapped areas of the PIO and MMIO windows + * (discarding writes and returning -1 for reads) rather than aborting. + */ + DEFINE_PROP_BOOL("allow-unmapped-accesses", GPEXHost, + allow_unmapped_accesses, true), + DEFINE_PROP_END_OF_LIST(), +}; + static void gpex_host_class_init(ObjectClass *klass, void *data) { DeviceClass *dc = DEVICE_CLASS(klass); @@ -117,6 +166,7 @@ static void gpex_host_class_init(ObjectClass *klass, void *data) dc->realize = gpex_host_realize; set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories); dc->fw_name = "pci"; + device_class_set_props(dc, gpex_host_properties); } static void gpex_host_initfn(Object *obj) diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c index e4be00b732a..529ff056dd2 100644 --- a/hw/ppc/spapr.c +++ b/hw/ppc/spapr.c @@ -4604,15 +4604,26 @@ static void spapr_machine_latest_class_options(MachineClass *mc) } \ type_init(spapr_machine_register_##suffix) +/* + * pseries-6.1 + */ +static void spapr_machine_6_1_class_options(MachineClass *mc) +{ + /* Defaults for the latest behaviour inherited from the base class */ +} + +DEFINE_SPAPR_MACHINE(6_1, "6.1", true); + /* * pseries-6.0 */ static void spapr_machine_6_0_class_options(MachineClass *mc) { - /* Defaults for the latest behaviour inherited from the base class */ + spapr_machine_6_1_class_options(mc); + compat_props_add(mc->compat_props, hw_compat_6_0, hw_compat_6_0_len); } -DEFINE_SPAPR_MACHINE(6_0, "6.0", true); +DEFINE_SPAPR_MACHINE(6_0, "6.0", false); /* * pseries-5.2 diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c index 2972b607f36..56b52d2d309 100644 --- a/hw/s390x/s390-virtio-ccw.c +++ b/hw/s390x/s390-virtio-ccw.c @@ -795,14 +795,26 @@ bool css_migration_enabled(void) } \ type_init(ccw_machine_register_##suffix) +static void ccw_machine_6_1_instance_options(MachineState *machine) +{ +} + +static void ccw_machine_6_1_class_options(MachineClass *mc) +{ +} +DEFINE_CCW_MACHINE(6_1, "6.1", true); + static void ccw_machine_6_0_instance_options(MachineState *machine) { + ccw_machine_6_1_instance_options(machine); } static void ccw_machine_6_0_class_options(MachineClass *mc) { + ccw_machine_6_1_class_options(mc); + compat_props_add(mc->compat_props, hw_compat_6_0, hw_compat_6_0_len); } -DEFINE_CCW_MACHINE(6_0, "6.0", true); +DEFINE_CCW_MACHINE(6_0, "6.0", false); static void ccw_machine_5_2_instance_options(MachineState *machine) { diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index 4376e654517..8fda2b15188 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -446,7 +446,8 @@ struct tb_tc { struct TranslationBlock { target_ulong pc; /* simulated PC corresponding to this block (EIP + CS base) */ target_ulong cs_base; /* CS base for this block */ - target_ulong cs_top; /* End (exclusive) of code segment for this block */ + target_ulong pcc_base; /* CHERI: Base of program counter for this block */ + target_ulong pcc_top; /* CHERI: End of program counter for this block */ uint32_t cheri_flags; /* Extra flags for CHERI. We need more bits than are available in flags (at least for MIPS) and this will allow us to avoid continuously changing the @@ -518,15 +519,17 @@ struct TranslationBlock { uintptr_t jmp_dest[2]; }; -// Reduce diff to upstream for CHERI (since we addd cs_top/ds_base/ds_top) -#if !defined(cpu_get_tb_cpu_state_6) -static inline void cpu_get_tb_cpu_state_6(CPUArchState *env, target_ulong *pc, +/* Reduce diff to upstream for CHERI since we add pcc_{base,top},cheri_flags. */ +#if !defined(cpu_get_tb_cpu_state_ext) +static inline void cpu_get_tb_cpu_state_ext(CPUArchState *env, target_ulong *pc, target_ulong *cs_base, - target_ulong *cs_top, + target_ulong *pcc_base, + target_ulong *pcc_top, uint32_t *cheri_flags, uint32_t *flags) { - (void)cs_top; + (void)pcc_base; + (void)pcc_top; (void)cheri_flags; cpu_get_tb_cpu_state(env, pc, cs_base, flags); } @@ -554,9 +557,9 @@ void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr, MemTxAttrs attrs); void tb_flush(CPUState *cpu); void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr); TranslationBlock *tb_htable_lookup(CPUState *cpu, target_ulong pc, - target_ulong cs_base, target_ulong cs_top, - uint32_t cheri_flags, uint32_t flags, - uint32_t cflags); + target_ulong cs_base, target_ulong pcc_base, + target_ulong pcc_top, uint32_t cheri_flags, + uint32_t flags, uint32_t cflags); void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr); /* GETPC is the true target of the return instruction that we'll execute. */ diff --git a/include/exec/tb-lookup.h b/include/exec/tb-lookup.h index 2f314eb6eb2..31d9fe7a9d6 100644 --- a/include/exec/tb-lookup.h +++ b/include/exec/tb-lookup.h @@ -19,7 +19,8 @@ /* Might cause an exception, so have a longjmp destination ready */ static inline TranslationBlock *tb_lookup(CPUState *cpu, target_ulong pc, target_ulong cs_base, - target_ulong cs_top, + target_ulong pcc_base, + target_ulong pcc_top, uint32_t cheri_flags, uint32_t flags, uint32_t cflags) { @@ -35,14 +36,16 @@ static inline TranslationBlock *tb_lookup(CPUState *cpu, target_ulong pc, if (likely(tb && tb->pc == pc && tb->cs_base == cs_base && - tb->cs_top == cs_top && + tb->pcc_base == pcc_base && + tb->pcc_top == pcc_top && tb->cheri_flags == cheri_flags && tb->flags == flags && tb->trace_vcpu_dstate == *cpu->trace_dstate && tb_cflags(tb) == cflags)) { return tb; } - tb = tb_htable_lookup(cpu, pc, cs_base, cs_top, cheri_flags, flags, cflags); + tb = tb_htable_lookup(cpu, pc, cs_base, pcc_base, pcc_top, cheri_flags, + flags, cflags); if (tb == NULL) { return NULL; diff --git a/include/hw/boards.h b/include/hw/boards.h index ad6c8fd5376..3d55d2bd62c 100644 --- a/include/hw/boards.h +++ b/include/hw/boards.h @@ -353,6 +353,9 @@ struct MachineState { } \ type_init(machine_initfn##_register_types) +extern GlobalProperty hw_compat_6_0[]; +extern const size_t hw_compat_6_0_len; + extern GlobalProperty hw_compat_5_2[]; extern const size_t hw_compat_5_2_len; diff --git a/include/hw/i386/pc.h b/include/hw/i386/pc.h index dcf060b7918..1522a3359a9 100644 --- a/include/hw/i386/pc.h +++ b/include/hw/i386/pc.h @@ -197,6 +197,9 @@ bool pc_system_ovmf_table_find(const char *entry, uint8_t **data, void pc_madt_cpu_entry(AcpiDeviceIf *adev, int uid, const CPUArchIdList *apic_ids, GArray *entry); +extern GlobalProperty pc_compat_6_0[]; +extern const size_t pc_compat_6_0_len; + extern GlobalProperty pc_compat_5_2[]; extern const size_t pc_compat_5_2_len; diff --git a/include/hw/pci-host/gpex.h b/include/hw/pci-host/gpex.h index d48a020a952..fcf8b638200 100644 --- a/include/hw/pci-host/gpex.h +++ b/include/hw/pci-host/gpex.h @@ -49,8 +49,12 @@ struct GPEXHost { MemoryRegion io_ioport; MemoryRegion io_mmio; + MemoryRegion io_ioport_window; + MemoryRegion io_mmio_window; qemu_irq irq[GPEX_NUM_IRQS]; int irq_num[GPEX_NUM_IRQS]; + + bool allow_unmapped_accesses; }; struct GPEXConfig { diff --git a/target/arm/cpu.h b/target/arm/cpu.h index 4bbc3b6eded..658c47cb851 100644 --- a/target/arm/cpu.h +++ b/target/arm/cpu.h @@ -249,6 +249,12 @@ extern const char * const arm32_regnames[16]; extern const char * const arm64_regnames[32]; #endif +/* See the commentary above the TBFLAG field definitions. */ +typedef struct CPUARMTBFlags { + uint32_t flags; + target_ulong flags2; +} CPUARMTBFlags; + typedef struct CPUARMState { /* Regs for current mode. */ uint32_t regs[16]; @@ -297,7 +303,7 @@ typedef struct CPUARMState { * well */ uint32_t chflags; #endif - uint32_t hflags; + CPUARMTBFlags hflags; /* Frequently accessed CPSR bits are stored separately for efficiency. This contains all the other bits. Use cpsr_{read,write} to access @@ -3657,32 +3663,40 @@ static inline target_ulong get_aarch_reg_as_x(AARCH_REG_TYPE *aarch_reg) } /* - * Bit usage in the TB flags field: bit 31 indicates whether we are - * in 32 or 64 bit mode. The meaning of the other bits depends on that. - * We put flags which are shared between 32 and 64 bit mode at the top - * of the word, and flags which apply to only one mode at the bottom. + * We have more than 32-bits worth of state per TB, so we split the data + * between tb->flags and tb->cs_base, which is otherwise unused for ARM. + * We collect these two parts in CPUARMTBFlags where they are named + * flags and flags2 respectively. + * + * The flags that are shared between all execution modes, TBFLAG_ANY, + * are stored in flags. The flags that are specific to a given mode + * are stores in flags2. Since cs_base is sized on the configured + * address size, flags2 always has 64-bits for A64, and a minimum of + * 32-bits for A32 and M32. * - * 31 20 18 14 9 0 - * +--------------+-----+-----+----------+--------------+ - * | | | TBFLAG_A32 | | - * | | +-----+----------+ TBFLAG_AM32 | - * | TBFLAG_ANY | |TBFLAG_M32| | - * | +-----------+----------+--------------| - * | | TBFLAG_A64 | - * +--------------+-------------------------------------+ - * 31 20 0 + * The bits for 32-bit A-profile and M-profile partially overlap: + * + * 31 23 11 10 0 + * +-------------+----------+----------------+ + * | | | TBFLAG_A32 | + * | TBFLAG_AM32 | +-----+----------+ + * | | |TBFLAG_M32| + * +-------------+----------------+----------+ + * 31 23 5 4 0 * * Unless otherwise noted, these bits are cached in env->hflags. */ -FIELD(TBFLAG_ANY, AARCH64_STATE, 31, 1) -FIELD(TBFLAG_ANY, SS_ACTIVE, 30, 1) -FIELD(TBFLAG_ANY, PSTATE_SS, 29, 1) /* Not cached. */ -FIELD(TBFLAG_ANY, BE_DATA, 28, 1) -FIELD(TBFLAG_ANY, MMUIDX, 24, 4) +FIELD(TBFLAG_ANY, AARCH64_STATE, 0, 1) +FIELD(TBFLAG_ANY, SS_ACTIVE, 1, 1) +FIELD(TBFLAG_ANY, PSTATE__SS, 2, 1) /* Not cached. */ +FIELD(TBFLAG_ANY, BE_DATA, 3, 1) +FIELD(TBFLAG_ANY, MMUIDX, 4, 4) /* Target EL if we take a floating-point-disabled exception */ -FIELD(TBFLAG_ANY, FPEXC_EL, 22, 2) +FIELD(TBFLAG_ANY, FPEXC_EL, 8, 2) /* For A-profile only, target EL for debug exceptions. */ -FIELD(TBFLAG_ANY, DEBUG_TARGET_EL, 20, 2) +FIELD(TBFLAG_ANY, DEBUG_TARGET_EL, 10, 2) +/* Memory operations require alignment: SCTLR_ELx.A or CCR.UNALIGN_TRP */ +FIELD(TBFLAG_ANY, ALIGN_MEM, 12, 1) #ifdef TARGET_CHERI @@ -3703,7 +3717,6 @@ FIELD(TBFLAG_ANY, DEBUG_TARGET_EL, 20, 2) #define CCTLR_SBL (1 << 7) #define CCTLR_DEFINED_START 2 -#define CCTLR_DEFINED_LENGTH 6 FIELD(TBFLAG_CHERI, CCTLR, 0, 6) // The 6 defined bits from CCTLR at all levels FIELD(TBFLAG_CHERI, PSTATE_C64, 6, 1) // The PSTATE.C64 bit @@ -3714,9 +3727,9 @@ FIELD(TBFLAG_CHERI, SETTAG, 9, 1) FIELD(TBFLAG_CHERI, CAP_ENABLED, 10, 1) // These flags really belongs in TBFLAG_ANY, but there is no room. If upstream // wants this feature, then they can move some bits around -FIELD(TBFLAG_CHERI, SCTLRA, 11, 1) -FIELD(TBFLAG_CHERI, SCTLRSA, 12, 1) -#define TBFLAG_CHERI_SIZE (CCTLR_DEFINED_LENGTH + 7) +FIELD(TBFLAG_CHERI, SCTLRSA, 11, 1) +#define TBFLAG_END(flag) (R_##flag##_SHIFT + R_##flag##_LENGTH) +#define TBFLAG_CHERI_SIZE TBFLAG_END(TBFLAG_CHERI_SCTLRSA) _Static_assert(TBFLAG_CHERI_SIZE <= 32, ""); #endif @@ -3724,44 +3737,44 @@ _Static_assert(TBFLAG_CHERI_SIZE <= 32, ""); /* * Bit usage when in AArch32 state, both A- and M-profile. */ -FIELD(TBFLAG_AM32, CONDEXEC, 0, 8) /* Not cached. */ -FIELD(TBFLAG_AM32, THUMB, 8, 1) /* Not cached. */ +FIELD(TBFLAG_AM32, CONDEXEC, 24, 8) /* Not cached. */ +FIELD(TBFLAG_AM32, THUMB, 23, 1) /* Not cached. */ /* * Bit usage when in AArch32 state, for A-profile only. */ -FIELD(TBFLAG_A32, VECLEN, 9, 3) /* Not cached. */ -FIELD(TBFLAG_A32, VECSTRIDE, 12, 2) /* Not cached. */ +FIELD(TBFLAG_A32, VECLEN, 0, 3) /* Not cached. */ +FIELD(TBFLAG_A32, VECSTRIDE, 3, 2) /* Not cached. */ /* * We store the bottom two bits of the CPAR as TB flags and handle * checks on the other bits at runtime. This shares the same bits as * VECSTRIDE, which is OK as no XScale CPU has VFP. * Not cached, because VECLEN+VECSTRIDE are not cached. */ -FIELD(TBFLAG_A32, XSCALE_CPAR, 12, 2) -FIELD(TBFLAG_A32, VFPEN, 14, 1) /* Partially cached, minus FPEXC. */ -FIELD(TBFLAG_A32, SCTLR_B, 15, 1) -FIELD(TBFLAG_A32, HSTR_ACTIVE, 16, 1) +FIELD(TBFLAG_A32, XSCALE_CPAR, 5, 2) +FIELD(TBFLAG_A32, VFPEN, 7, 1) /* Partially cached, minus FPEXC. */ +FIELD(TBFLAG_A32, SCTLR__B, 8, 1) /* Cannot overlap with SCTLR_B */ +FIELD(TBFLAG_A32, HSTR_ACTIVE, 9, 1) /* * Indicates whether cp register reads and writes by guest code should access * the secure or nonsecure bank of banked registers; note that this is not * the same thing as the current security state of the processor! */ -FIELD(TBFLAG_A32, NS, 17, 1) +FIELD(TBFLAG_A32, NS, 10, 1) /* * Bit usage when in AArch32 state, for M-profile only. */ /* Handler (ie not Thread) mode */ -FIELD(TBFLAG_M32, HANDLER, 9, 1) +FIELD(TBFLAG_M32, HANDLER, 0, 1) /* Whether we should generate stack-limit checks */ -FIELD(TBFLAG_M32, STACKCHECK, 10, 1) +FIELD(TBFLAG_M32, STACKCHECK, 1, 1) /* Set if FPCCR.LSPACT is set */ -FIELD(TBFLAG_M32, LSPACT, 11, 1) /* Not cached. */ +FIELD(TBFLAG_M32, LSPACT, 2, 1) /* Not cached. */ /* Set if we must create a new FP context */ -FIELD(TBFLAG_M32, NEW_FP_CTXT_NEEDED, 12, 1) /* Not cached. */ +FIELD(TBFLAG_M32, NEW_FP_CTXT_NEEDED, 3, 1) /* Not cached. */ /* Set if FPCCR.S does not match current security state */ -FIELD(TBFLAG_M32, FPCCR_S_WRONG, 13, 1) /* Not cached. */ +FIELD(TBFLAG_M32, FPCCR_S_WRONG, 4, 1) /* Not cached. */ /* * Bit usage when in AArch64 state @@ -3779,12 +3792,33 @@ FIELD(TBFLAG_A64, TCMA, 16, 2) FIELD(TBFLAG_A64, MTE_ACTIVE, 18, 1) FIELD(TBFLAG_A64, MTE0_ACTIVE, 19, 1) +/* + * Helpers for using the above. + */ +#define DP_TBFLAG_ANY(DST, WHICH, VAL) \ + (DST.flags = FIELD_DP32(DST.flags, TBFLAG_ANY, WHICH, VAL)) +#define DP_TBFLAG_A64(DST, WHICH, VAL) \ + (DST.flags2 = FIELD_DP32(DST.flags2, TBFLAG_A64, WHICH, VAL)) +#define DP_TBFLAG_A32(DST, WHICH, VAL) \ + (DST.flags2 = FIELD_DP32(DST.flags2, TBFLAG_A32, WHICH, VAL)) +#define DP_TBFLAG_M32(DST, WHICH, VAL) \ + (DST.flags2 = FIELD_DP32(DST.flags2, TBFLAG_M32, WHICH, VAL)) +#define DP_TBFLAG_AM32(DST, WHICH, VAL) \ + (DST.flags2 = FIELD_DP32(DST.flags2, TBFLAG_AM32, WHICH, VAL)) + +#define EX_TBFLAG_ANY(IN, WHICH) FIELD_EX32(IN.flags, TBFLAG_ANY, WHICH) +#define EX_TBFLAG_A64(IN, WHICH) FIELD_EX32(IN.flags2, TBFLAG_A64, WHICH) +#define EX_TBFLAG_A32(IN, WHICH) FIELD_EX32(IN.flags2, TBFLAG_A32, WHICH) +#define EX_TBFLAG_M32(IN, WHICH) FIELD_EX32(IN.flags2, TBFLAG_M32, WHICH) +#define EX_TBFLAG_AM32(IN, WHICH) FIELD_EX32(IN.flags2, TBFLAG_AM32, WHICH) + extern void aarch_cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc, target_ulong *cs_base, - target_ulong *cs_top, + target_ulong *pcc_base, + target_ulong *pcc_top, uint32_t *cheri_flags, uint32_t *pflags); // Ugly macro hack to avoid having to modify cpu_get_tb_cpu_state in all targets -#define cpu_get_tb_cpu_state_6 aarch_cpu_get_tb_cpu_state +#define cpu_get_tb_cpu_state_ext aarch_cpu_get_tb_cpu_state /** * cpu_mmu_index: @@ -3796,7 +3830,7 @@ extern void aarch_cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc, */ static inline int cpu_mmu_index(CPUARMState *env, bool ifetch) { - return FIELD_EX32(env->hflags, TBFLAG_ANY, MMUIDX); + return EX_TBFLAG_ANY(env->hflags, MMUIDX); } static inline bool bswap_code(bool sctlr_b) @@ -4735,9 +4769,6 @@ static inline uint32_t arm_rebuild_chflags_el(CPUARMState *env, int el) chflags = FIELD_DP32(chflags, TBFLAG_CHERI, CAP_ENABLED, 1); } uint64_t sctlr = arm_sctlr(env, el); - if (sctlr & SCTLR_A) { - chflags = FIELD_DP32(chflags, TBFLAG_CHERI, SCTLRA, 1); - } if ((el == 0) ? (sctlr & SCTLR_SA0) : (sctlr & SCTLR_SA)) { chflags = FIELD_DP32(chflags, TBFLAG_CHERI, SCTLRSA, 1); } diff --git a/target/arm/helper-a64.c b/target/arm/helper-a64.c index 7dc89e5c89a..9da51529ced 100644 --- a/target/arm/helper-a64.c +++ b/target/arm/helper-a64.c @@ -1052,7 +1052,7 @@ void HELPER(exception_return)(CPUARMState *env, uint64_t new_pc) * the hflags rebuild, since we can pull the composite TBII field * from there. */ - tbii = FIELD_EX32(env->hflags, TBFLAG_A64, TBII); + tbii = EX_TBFLAG_A64(env->hflags, TBII); if ((tbii >> extract64(new_pc, 55, 1)) & 1) { /* TBI is enabled. */ int core_mmu_idx = cpu_mmu_index(env, false); diff --git a/target/arm/helper-a64.h b/target/arm/helper-a64.h index c2f0b0d3ee9..29fcc90a877 100644 --- a/target/arm/helper-a64.h +++ b/target/arm/helper-a64.h @@ -104,8 +104,7 @@ DEF_HELPER_FLAGS_3(autdb, TCG_CALL_NO_WG, i64, env, i64, i64) DEF_HELPER_FLAGS_2(xpaci, TCG_CALL_NO_RWG_SE, i64, env, i64) DEF_HELPER_FLAGS_2(xpacd, TCG_CALL_NO_RWG_SE, i64, env, i64) -DEF_HELPER_FLAGS_3(mte_check1, TCG_CALL_NO_WG, i64, env, i32, i64) -DEF_HELPER_FLAGS_3(mte_checkN, TCG_CALL_NO_WG, i64, env, i32, i64) +DEF_HELPER_FLAGS_3(mte_check, TCG_CALL_NO_WG, i64, env, i32, i64) DEF_HELPER_FLAGS_3(mte_check_zva, TCG_CALL_NO_WG, cap_checked_ptr, env, i32, i64) DEF_HELPER_FLAGS_3(irg, TCG_CALL_NO_RWG, i64, env, i64, i64) diff --git a/target/arm/helper.c b/target/arm/helper.c index 1a9c00370cc..a4e532a63a7 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -13559,42 +13559,49 @@ ARMMMUIdx arm_stage1_mmu_idx(CPUARMState *env) } #endif -static uint32_t rebuild_hflags_common(CPUARMState *env, int fp_el, - ARMMMUIdx mmu_idx, uint32_t flags) +static CPUARMTBFlags rebuild_hflags_common(CPUARMState *env, int fp_el, + ARMMMUIdx mmu_idx, + CPUARMTBFlags flags) { - flags = FIELD_DP32(flags, TBFLAG_ANY, FPEXC_EL, fp_el); - flags = FIELD_DP32(flags, TBFLAG_ANY, MMUIDX, - arm_to_core_mmu_idx(mmu_idx)); + DP_TBFLAG_ANY(flags, FPEXC_EL, fp_el); + DP_TBFLAG_ANY(flags, MMUIDX, arm_to_core_mmu_idx(mmu_idx)); if (arm_singlestep_active(env)) { - flags = FIELD_DP32(flags, TBFLAG_ANY, SS_ACTIVE, 1); + DP_TBFLAG_ANY(flags, SS_ACTIVE, 1); } return flags; } -static uint32_t rebuild_hflags_common_32(CPUARMState *env, int fp_el, - ARMMMUIdx mmu_idx, uint32_t flags) +static CPUARMTBFlags rebuild_hflags_common_32(CPUARMState *env, int fp_el, + ARMMMUIdx mmu_idx, + CPUARMTBFlags flags) { bool sctlr_b = arm_sctlr_b(env); if (sctlr_b) { - flags = FIELD_DP32(flags, TBFLAG_A32, SCTLR_B, 1); + DP_TBFLAG_A32(flags, SCTLR__B, 1); } if (arm_cpu_data_is_big_endian_a32(env, sctlr_b)) { - flags = FIELD_DP32(flags, TBFLAG_ANY, BE_DATA, 1); + DP_TBFLAG_ANY(flags, BE_DATA, 1); } - flags = FIELD_DP32(flags, TBFLAG_A32, NS, !access_secure_reg(env)); + DP_TBFLAG_A32(flags, NS, !access_secure_reg(env)); return rebuild_hflags_common(env, fp_el, mmu_idx, flags); } -static uint32_t rebuild_hflags_m32(CPUARMState *env, int fp_el, - ARMMMUIdx mmu_idx) +static CPUARMTBFlags rebuild_hflags_m32(CPUARMState *env, int fp_el, + ARMMMUIdx mmu_idx) { - uint32_t flags = 0; + CPUARMTBFlags flags = {}; + uint32_t ccr = env->v7m.ccr[env->v7m.secure]; + + /* Without HaveMainExt, CCR.UNALIGN_TRP is RES1. */ + if (ccr & R_V7M_CCR_UNALIGN_TRP_MASK) { + DP_TBFLAG_ANY(flags, ALIGN_MEM, 1); + } if (arm_v7m_is_handler_mode(env)) { - flags = FIELD_DP32(flags, TBFLAG_M32, HANDLER, 1); + DP_TBFLAG_M32(flags, HANDLER, 1); } /* @@ -13604,56 +13611,60 @@ static uint32_t rebuild_hflags_m32(CPUARMState *env, int fp_el, */ if (arm_feature(env, ARM_FEATURE_V8) && !((mmu_idx & ARM_MMU_IDX_M_NEGPRI) && - (env->v7m.ccr[env->v7m.secure] & R_V7M_CCR_STKOFHFNMIGN_MASK))) { - flags = FIELD_DP32(flags, TBFLAG_M32, STACKCHECK, 1); + (ccr & R_V7M_CCR_STKOFHFNMIGN_MASK))) { + DP_TBFLAG_M32(flags, STACKCHECK, 1); } return rebuild_hflags_common_32(env, fp_el, mmu_idx, flags); } -static uint32_t rebuild_hflags_aprofile(CPUARMState *env) +static CPUARMTBFlags rebuild_hflags_aprofile(CPUARMState *env) { - int flags = 0; + CPUARMTBFlags flags = {}; - flags = FIELD_DP32(flags, TBFLAG_ANY, DEBUG_TARGET_EL, - arm_debug_target_el(env)); + DP_TBFLAG_ANY(flags, DEBUG_TARGET_EL, arm_debug_target_el(env)); return flags; } -static uint32_t rebuild_hflags_a32(CPUARMState *env, int fp_el, - ARMMMUIdx mmu_idx) +static CPUARMTBFlags rebuild_hflags_a32(CPUARMState *env, int fp_el, + ARMMMUIdx mmu_idx) { - uint32_t flags = rebuild_hflags_aprofile(env); + CPUARMTBFlags flags = rebuild_hflags_aprofile(env); + int el = arm_current_el(env); + + if (arm_sctlr(env, el) & SCTLR_A) { + DP_TBFLAG_ANY(flags, ALIGN_MEM, 1); + } if (arm_el_is_aa64(env, 1)) { - flags = FIELD_DP32(flags, TBFLAG_A32, VFPEN, 1); + DP_TBFLAG_A32(flags, VFPEN, 1); } - if (arm_current_el(env) < 2 && env->cp15.hstr_el2 && + if (el < 2 && env->cp15.hstr_el2 && (arm_hcr_el2_eff(env) & (HCR_E2H | HCR_TGE)) != (HCR_E2H | HCR_TGE)) { - flags = FIELD_DP32(flags, TBFLAG_A32, HSTR_ACTIVE, 1); + DP_TBFLAG_A32(flags, HSTR_ACTIVE, 1); } return rebuild_hflags_common_32(env, fp_el, mmu_idx, flags); } -static uint32_t rebuild_hflags_a64(CPUARMState *env, int el, int fp_el, - ARMMMUIdx mmu_idx) +static CPUARMTBFlags rebuild_hflags_a64(CPUARMState *env, int el, int fp_el, + ARMMMUIdx mmu_idx) { - uint32_t flags = rebuild_hflags_aprofile(env); + CPUARMTBFlags flags = rebuild_hflags_aprofile(env); ARMMMUIdx stage1 = stage_1_mmu_idx(mmu_idx); uint64_t tcr = regime_tcr(env, mmu_idx)->raw_tcr; uint64_t sctlr; int tbii, tbid; - flags = FIELD_DP32(flags, TBFLAG_ANY, AARCH64_STATE, 1); + DP_TBFLAG_ANY(flags, AARCH64_STATE, 1); /* Get control bits for tagged addresses. */ tbid = aa64_va_parameter_tbi(tcr, mmu_idx); tbii = tbid & ~aa64_va_parameter_tbid(tcr, mmu_idx); - flags = FIELD_DP32(flags, TBFLAG_A64, TBII, tbii); - flags = FIELD_DP32(flags, TBFLAG_A64, TBID, tbid); + DP_TBFLAG_A64(flags, TBII, tbii); + DP_TBFLAG_A64(flags, TBID, tbid); if (cpu_isar_feature(aa64_sve, env_archcpu(env))) { int sve_el = sve_exception_el(env, el); @@ -13668,14 +13679,18 @@ static uint32_t rebuild_hflags_a64(CPUARMState *env, int el, int fp_el, } else { zcr_len = sve_zcr_len_for_el(env, el); } - flags = FIELD_DP32(flags, TBFLAG_A64, SVEEXC_EL, sve_el); - flags = FIELD_DP32(flags, TBFLAG_A64, ZCR_LEN, zcr_len); + DP_TBFLAG_A64(flags, SVEEXC_EL, sve_el); + DP_TBFLAG_A64(flags, ZCR_LEN, zcr_len); } sctlr = regime_sctlr(env, stage1); + if (sctlr & SCTLR_A) { + DP_TBFLAG_ANY(flags, ALIGN_MEM, 1); + } + if (arm_cpu_data_is_big_endian_a64(el, sctlr)) { - flags = FIELD_DP32(flags, TBFLAG_ANY, BE_DATA, 1); + DP_TBFLAG_ANY(flags, BE_DATA, 1); } if (cpu_isar_feature(aa64_pauth, env_archcpu(env))) { @@ -13686,14 +13701,14 @@ static uint32_t rebuild_hflags_a64(CPUARMState *env, int el, int fp_el, * The decision of which action to take is left to a helper. */ if (sctlr & (SCTLR_EnIA | SCTLR_EnIB | SCTLR_EnDA | SCTLR_EnDB)) { - flags = FIELD_DP32(flags, TBFLAG_A64, PAUTH_ACTIVE, 1); + DP_TBFLAG_A64(flags, PAUTH_ACTIVE, 1); } } if (cpu_isar_feature(aa64_bti, env_archcpu(env))) { /* Note that SCTLR_EL[23].BT == SCTLR_BT1. */ if (sctlr & (el == 0 ? SCTLR_BT0 : SCTLR_BT1)) { - flags = FIELD_DP32(flags, TBFLAG_A64, BT, 1); + DP_TBFLAG_A64(flags, BT, 1); } } @@ -13705,7 +13720,7 @@ static uint32_t rebuild_hflags_a64(CPUARMState *env, int el, int fp_el, case ARMMMUIdx_SE10_1: case ARMMMUIdx_SE10_1_PAN: /* TODO: ARMv8.3-NV */ - flags = FIELD_DP32(flags, TBFLAG_A64, UNPRIV, 1); + DP_TBFLAG_A64(flags, UNPRIV, 1); break; case ARMMMUIdx_E20_2: case ARMMMUIdx_E20_2_PAN: @@ -13716,7 +13731,7 @@ static uint32_t rebuild_hflags_a64(CPUARMState *env, int el, int fp_el, * gated by HCR_EL2. == '11', and so is LDTR. */ if (env->cp15.hcr_el2 & HCR_TGE) { - flags = FIELD_DP32(flags, TBFLAG_A64, UNPRIV, 1); + DP_TBFLAG_A64(flags, UNPRIV, 1); } break; default: @@ -13734,24 +13749,23 @@ static uint32_t rebuild_hflags_a64(CPUARMState *env, int el, int fp_el, * 4) If no Allocation Tag Access, then all accesses are Unchecked. */ if (allocation_tag_access_enabled(env, el, sctlr)) { - flags = FIELD_DP32(flags, TBFLAG_A64, ATA, 1); + DP_TBFLAG_A64(flags, ATA, 1); if (tbid && !(env->pstate & PSTATE_TCO) && (sctlr & (el == 0 ? SCTLR_TCF0 : SCTLR_TCF))) { - flags = FIELD_DP32(flags, TBFLAG_A64, MTE_ACTIVE, 1); + DP_TBFLAG_A64(flags, MTE_ACTIVE, 1); } } /* And again for unprivileged accesses, if required. */ - if (FIELD_EX32(flags, TBFLAG_A64, UNPRIV) + if (EX_TBFLAG_A64(flags, UNPRIV) && tbid && !(env->pstate & PSTATE_TCO) && (sctlr & SCTLR_TCF0) && allocation_tag_access_enabled(env, 0, sctlr)) { - flags = FIELD_DP32(flags, TBFLAG_A64, MTE0_ACTIVE, 1); + DP_TBFLAG_A64(flags, MTE0_ACTIVE, 1); } /* Cache TCMA as well as TBI. */ - flags = FIELD_DP32(flags, TBFLAG_A64, TCMA, - aa64_va_parameter_tcma(tcr, mmu_idx)); + DP_TBFLAG_A64(flags, TCMA, aa64_va_parameter_tcma(tcr, mmu_idx)); } #ifdef TARGET_CHERI @@ -13762,7 +13776,7 @@ static uint32_t rebuild_hflags_a64(CPUARMState *env, int el, int fp_el, return rebuild_hflags_common(env, fp_el, mmu_idx, flags); } -static uint32_t rebuild_hflags_internal(CPUARMState *env) +static CPUARMTBFlags rebuild_hflags_internal(CPUARMState *env) { int el = arm_current_el(env); int fp_el = fp_exception_el(env, el); @@ -13791,6 +13805,7 @@ void HELPER(rebuild_hflags_m32_newel)(CPUARMState *env) int el = arm_current_el(env); int fp_el = fp_exception_el(env, el); ARMMMUIdx mmu_idx = arm_mmu_idx_el(env, el); + env->hflags = rebuild_hflags_m32(env, fp_el, mmu_idx); } @@ -13833,15 +13848,17 @@ void HELPER(rebuild_hflags_a64)(CPUARMState *env, int el) static inline void assert_hflags_rebuild_correctly(CPUARMState *env) { #ifdef CONFIG_DEBUG_TCG - uint32_t env_flags_current = env->hflags; + CPUARMTBFlags c = env->hflags; #ifdef TARGET_CHERI uint32_t cheri_flags = env->chflags; #endif - uint32_t env_flags_rebuilt = rebuild_hflags_internal(env); + CPUARMTBFlags r = rebuild_hflags_internal(env); - if (unlikely(env_flags_current != env_flags_rebuilt)) { - fprintf(stderr, "TCG hflags mismatch (current:0x%08x rebuilt:0x%08x)\n", - env_flags_current, env_flags_rebuilt); + if (unlikely(c.flags != r.flags || c.flags2 != r.flags2)) { + fprintf(stderr, "TCG hflags mismatch " + "(current:(0x%08x,0x" TARGET_FMT_lx ")" + " rebuilt:(0x%08x,0x" TARGET_FMT_lx ")\n", + c.flags, c.flags2, r.flags, r.flags2); abort(); } #ifdef TARGET_CHERI @@ -13856,28 +13873,25 @@ static inline void assert_hflags_rebuild_correctly(CPUARMState *env) } void aarch_cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc, - target_ulong *cs_base, target_ulong *cs_top, - uint32_t *cheri_flags, uint32_t *pflags) + target_ulong *cs_base, target_ulong *pcc_base, + target_ulong *pcc_top, uint32_t *cheri_flags, + uint32_t *pflags) { - uint32_t flags = env->hflags; + CPUARMTBFlags flags; - *cs_base = 0; assert_hflags_rebuild_correctly(env); + flags = env->hflags; - if (FIELD_EX32(flags, TBFLAG_ANY, AARCH64_STATE)) { + if (EX_TBFLAG_ANY(flags, AARCH64_STATE)) { *pc = get_aarch_reg_as_x(&env->pc); - #ifdef TARGET_CHERI cheri_cpu_get_tb_cpu_state(_cheri_get_pcc_unchecked(env), - cheri_get_ddc(env), cs_base, cs_top, + cheri_get_ddc(env), pcc_base, pcc_top, cheri_flags); *cheri_flags |= (env->chflags << TB_FLAG_CHERI_SPARE_INDEX_START); -#else - *cs_base = 0; #endif - if (cpu_isar_feature(aa64_bti, env_archcpu(env))) { - flags = FIELD_DP32(flags, TBFLAG_A64, BTYPE, env->btype); + DP_TBFLAG_A64(flags, BTYPE, env->btype); } } else { *pc = env->regs[15]; @@ -13886,7 +13900,7 @@ void aarch_cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc, if (arm_feature(env, ARM_FEATURE_M_SECURITY) && FIELD_EX32(env->v7m.fpccr[M_REG_S], V7M_FPCCR, S) != env->v7m.secure) { - flags = FIELD_DP32(flags, TBFLAG_M32, FPCCR_S_WRONG, 1); + DP_TBFLAG_M32(flags, FPCCR_S_WRONG, 1); } if ((env->v7m.fpccr[env->v7m.secure] & R_V7M_FPCCR_ASPEN_MASK) && @@ -13898,12 +13912,12 @@ void aarch_cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc, * active FP context; we must create a new FP context before * executing any FP insn. */ - flags = FIELD_DP32(flags, TBFLAG_M32, NEW_FP_CTXT_NEEDED, 1); + DP_TBFLAG_M32(flags, NEW_FP_CTXT_NEEDED, 1); } bool is_secure = env->v7m.fpccr[M_REG_S] & R_V7M_FPCCR_S_MASK; if (env->v7m.fpccr[is_secure] & R_V7M_FPCCR_LSPACT_MASK) { - flags = FIELD_DP32(flags, TBFLAG_M32, LSPACT, 1); + DP_TBFLAG_M32(flags, LSPACT, 1); } } else { /* @@ -13911,21 +13925,18 @@ void aarch_cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc, * Note that VECLEN+VECSTRIDE are RES0 for M-profile. */ if (arm_feature(env, ARM_FEATURE_XSCALE)) { - flags = FIELD_DP32(flags, TBFLAG_A32, - XSCALE_CPAR, env->cp15.c15_cpar); + DP_TBFLAG_A32(flags, XSCALE_CPAR, env->cp15.c15_cpar); } else { - flags = FIELD_DP32(flags, TBFLAG_A32, VECLEN, - env->vfp.vec_len); - flags = FIELD_DP32(flags, TBFLAG_A32, VECSTRIDE, - env->vfp.vec_stride); + DP_TBFLAG_A32(flags, VECLEN, env->vfp.vec_len); + DP_TBFLAG_A32(flags, VECSTRIDE, env->vfp.vec_stride); } if (env->vfp.xregs[ARM_VFP_FPEXC] & (1 << 30)) { - flags = FIELD_DP32(flags, TBFLAG_A32, VFPEN, 1); + DP_TBFLAG_A32(flags, VFPEN, 1); } } - flags = FIELD_DP32(flags, TBFLAG_AM32, THUMB, env->thumb); - flags = FIELD_DP32(flags, TBFLAG_AM32, CONDEXEC, env->condexec_bits); + DP_TBFLAG_AM32(flags, THUMB, env->thumb); + DP_TBFLAG_AM32(flags, CONDEXEC, env->condexec_bits); } /* @@ -13935,14 +13946,14 @@ void aarch_cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc, * 0 x Inactive (the TB flag for SS is always 0) * 1 0 Active-pending * 1 1 Active-not-pending - * SS_ACTIVE is set in hflags; PSTATE_SS is computed every TB. + * SS_ACTIVE is set in hflags; PSTATE__SS is computed every TB. */ - if (FIELD_EX32(flags, TBFLAG_ANY, SS_ACTIVE) && - (env->pstate & PSTATE_SS)) { - flags = FIELD_DP32(flags, TBFLAG_ANY, PSTATE_SS, 1); + if (EX_TBFLAG_ANY(flags, SS_ACTIVE) && (env->pstate & PSTATE_SS)) { + DP_TBFLAG_ANY(flags, PSTATE__SS, 1); } - *pflags = flags; + *pflags = flags.flags; + *cs_base = flags.flags2; } #ifdef TARGET_AARCH64 diff --git a/target/arm/internals.h b/target/arm/internals.h index ad094640182..ecd5e84c951 100644 --- a/target/arm/internals.h +++ b/target/arm/internals.h @@ -26,6 +26,7 @@ #define TARGET_ARM_INTERNALS_H #include "hw/registerfields.h" +#include "tcg/tcg-gvec-desc.h" #include "cheri-lazy-capregs.h" #include "syndrome.h" @@ -1299,14 +1300,10 @@ FIELD(MTEDESC, MIDX, 0, 4) FIELD(MTEDESC, TBI, 4, 2) FIELD(MTEDESC, TCMA, 6, 2) FIELD(MTEDESC, WRITE, 8, 1) -FIELD(MTEDESC, ESIZE, 9, 5) -FIELD(MTEDESC, TSIZE, 14, 10) /* mte_checkN only */ - -bool mte_probe1(CPUARMState *env, uint32_t desc, uint64_t ptr); -uint64_t mte_check1(CPUARMState *env, uint32_t desc, - uint64_t ptr, uintptr_t ra); -uint64_t mte_checkN(CPUARMState *env, uint32_t desc, - uint64_t ptr, uintptr_t ra); +FIELD(MTEDESC, SIZEM1, 9, SIMD_DATA_BITS - 9) /* size - 1 */ + +bool mte_probe(CPUARMState *env, uint32_t desc, uint64_t ptr); +uint64_t mte_check(CPUARMState *env, uint32_t desc, uint64_t ptr, uintptr_t ra); static inline int allocation_tag_from_addr(uint64_t ptr) { diff --git a/target/arm/mte_helper.c b/target/arm/mte_helper.c index 8be17e1b707..a6fccc6e69e 100644 --- a/target/arm/mte_helper.c +++ b/target/arm/mte_helper.c @@ -121,7 +121,7 @@ static uint8_t *allocation_tag_mem(CPUARMState *env, int ptr_mmu_idx, * exception for inaccessible pages, and resolves the virtual address * into the softmmu tlb. * - * When RA == 0, this is for mte_probe1. The page is expected to be + * When RA == 0, this is for mte_probe. The page is expected to be * valid. Indicate to probe_access_flags no-fault, then assert that * we received a valid page. */ @@ -617,80 +617,6 @@ static void mte_check_fail(CPUARMState *env, uint32_t desc, } } -/* - * Perform an MTE checked access for a single logical or atomic access. - */ -static bool mte_probe1_int(CPUARMState *env, uint32_t desc, uint64_t ptr, - uintptr_t ra, int bit55) -{ - int mem_tag, mmu_idx, ptr_tag, size; - MMUAccessType type; - uint8_t *mem; - - ptr_tag = allocation_tag_from_addr(ptr); - - if (tcma_check(desc, bit55, ptr_tag)) { - return true; - } - - mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX); - type = FIELD_EX32(desc, MTEDESC, WRITE) ? MMU_DATA_STORE : MMU_DATA_LOAD; - size = FIELD_EX32(desc, MTEDESC, ESIZE); - - mem = allocation_tag_mem(env, mmu_idx, ptr, type, size, - MMU_DATA_LOAD, 1, ra); - if (!mem) { - return true; - } - - mem_tag = load_tag1(ptr, mem); - return ptr_tag == mem_tag; -} - -/* - * No-fault version of mte_check1, to be used by SVE for MemSingleNF. - * Returns false if the access is Checked and the check failed. This - * is only intended to probe the tag -- the validity of the page must - * be checked beforehand. - */ -bool mte_probe1(CPUARMState *env, uint32_t desc, uint64_t ptr) -{ - int bit55 = extract64(ptr, 55, 1); - - /* If TBI is disabled, the access is unchecked. */ - if (unlikely(!tbi_check(desc, bit55))) { - return true; - } - - return mte_probe1_int(env, desc, ptr, 0, bit55); -} - -uint64_t mte_check1(CPUARMState *env, uint32_t desc, - uint64_t ptr, uintptr_t ra) -{ - int bit55 = extract64(ptr, 55, 1); - - /* If TBI is disabled, the access is unchecked, and ptr is not dirty. */ - if (unlikely(!tbi_check(desc, bit55))) { - return ptr; - } - - if (unlikely(!mte_probe1_int(env, desc, ptr, ra, bit55))) { - mte_check_fail(env, desc, ptr, ra); - } - - return useronly_clean_ptr(ptr); -} - -uint64_t HELPER(mte_check1)(CPUARMState *env, uint32_t desc, uint64_t ptr) -{ - return mte_check1(env, desc, ptr, GETPC()); -} - -/* - * Perform an MTE checked access for multiple logical accesses. - */ - /** * checkN: * @tag: tag memory to test @@ -753,59 +679,70 @@ static int checkN(uint8_t *mem, int odd, int cmp, int count) return n; } -uint64_t mte_checkN(CPUARMState *env, uint32_t desc, - uint64_t ptr, uintptr_t ra) +/** + * mte_probe_int() - helper for mte_probe and mte_check + * @env: CPU environment + * @desc: MTEDESC descriptor + * @ptr: virtual address of the base of the access + * @fault: return virtual address of the first check failure + * + * Internal routine for both mte_probe and mte_check. + * Return zero on failure, filling in *fault. + * Return negative on trivial success for tbi disabled. + * Return positive on success with tbi enabled. + */ +static int mte_probe_int(CPUARMState *env, uint32_t desc, uint64_t ptr, + uintptr_t ra, uint64_t *fault) { int mmu_idx, ptr_tag, bit55; - uint64_t ptr_last, ptr_end, prev_page, next_page; - uint64_t tag_first, tag_end; - uint64_t tag_byte_first, tag_byte_end; - uint32_t esize, total, tag_count, tag_size, n, c; + uint64_t ptr_last, prev_page, next_page; + uint64_t tag_first, tag_last; + uint64_t tag_byte_first, tag_byte_last; + uint32_t sizem1, tag_count, tag_size, n, c; uint8_t *mem1, *mem2; MMUAccessType type; bit55 = extract64(ptr, 55, 1); + *fault = ptr; /* If TBI is disabled, the access is unchecked, and ptr is not dirty. */ if (unlikely(!tbi_check(desc, bit55))) { - return ptr; + return -1; } ptr_tag = allocation_tag_from_addr(ptr); if (tcma_check(desc, bit55, ptr_tag)) { - goto done; + return 1; } mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX); type = FIELD_EX32(desc, MTEDESC, WRITE) ? MMU_DATA_STORE : MMU_DATA_LOAD; - esize = FIELD_EX32(desc, MTEDESC, ESIZE); - total = FIELD_EX32(desc, MTEDESC, TSIZE); + sizem1 = FIELD_EX32(desc, MTEDESC, SIZEM1); - /* Find the addr of the end of the access, and of the last element. */ - ptr_end = ptr + total; - ptr_last = ptr_end - esize; + /* Find the addr of the end of the access */ + ptr_last = ptr + sizem1; /* Round the bounds to the tag granule, and compute the number of tags. */ tag_first = QEMU_ALIGN_DOWN(ptr, TAG_GRANULE); - tag_end = QEMU_ALIGN_UP(ptr_last, TAG_GRANULE); - tag_count = (tag_end - tag_first) / TAG_GRANULE; + tag_last = QEMU_ALIGN_DOWN(ptr_last, TAG_GRANULE); + tag_count = ((tag_last - tag_first) / TAG_GRANULE) + 1; /* Round the bounds to twice the tag granule, and compute the bytes. */ tag_byte_first = QEMU_ALIGN_DOWN(ptr, 2 * TAG_GRANULE); - tag_byte_end = QEMU_ALIGN_UP(ptr_last, 2 * TAG_GRANULE); + tag_byte_last = QEMU_ALIGN_DOWN(ptr_last, 2 * TAG_GRANULE); /* Locate the page boundaries. */ prev_page = ptr & TARGET_PAGE_MASK; next_page = prev_page + TARGET_PAGE_SIZE; - if (likely(tag_end - prev_page <= TARGET_PAGE_SIZE)) { + if (likely(tag_last - prev_page <= TARGET_PAGE_SIZE)) { /* Memory access stays on one page. */ - tag_size = (tag_byte_end - tag_byte_first) / (2 * TAG_GRANULE); - mem1 = allocation_tag_mem(env, mmu_idx, ptr, type, total, + tag_size = ((tag_byte_last - tag_byte_first) / (2 * TAG_GRANULE)) + 1; + mem1 = allocation_tag_mem(env, mmu_idx, ptr, type, sizem1 + 1, MMU_DATA_LOAD, tag_size, ra); if (!mem1) { - goto done; + return 1; } /* Perform all of the comparisons. */ n = checkN(mem1, ptr & TAG_GRANULE, ptr_tag, tag_count); @@ -815,9 +752,9 @@ uint64_t mte_checkN(CPUARMState *env, uint32_t desc, mem1 = allocation_tag_mem(env, mmu_idx, ptr, type, next_page - ptr, MMU_DATA_LOAD, tag_size, ra); - tag_size = (tag_byte_end - next_page) / (2 * TAG_GRANULE); + tag_size = ((tag_byte_last - next_page) / (2 * TAG_GRANULE)) + 1; mem2 = allocation_tag_mem(env, mmu_idx, next_page, type, - ptr_end - next_page, + ptr_last - next_page + 1, MMU_DATA_LOAD, tag_size, ra); /* @@ -831,31 +768,57 @@ uint64_t mte_checkN(CPUARMState *env, uint32_t desc, } if (n == c) { if (!mem2) { - goto done; + return 1; } n += checkN(mem2, 0, ptr_tag, tag_count - c); } } + if (likely(n == tag_count)) { + return 1; + } + /* - * If we failed, we know which granule. Compute the element that - * is first in that granule, and signal failure on that element. + * If we failed, we know which granule. For the first granule, the + * failure address is @ptr, the first byte accessed. Otherwise the + * failure address is the first byte of the nth granule. */ - if (unlikely(n < tag_count)) { - uint64_t fail_ofs; - - fail_ofs = tag_first + n * TAG_GRANULE - ptr; - fail_ofs = ROUND_UP(fail_ofs, esize); - mte_check_fail(env, desc, ptr + fail_ofs, ra); + if (n > 0) { + *fault = tag_first + n * TAG_GRANULE; } + return 0; +} - done: +uint64_t mte_check(CPUARMState *env, uint32_t desc, uint64_t ptr, uintptr_t ra) +{ + uint64_t fault; + int ret = mte_probe_int(env, desc, ptr, ra, &fault); + + if (unlikely(ret == 0)) { + mte_check_fail(env, desc, fault, ra); + } else if (ret < 0) { + return ptr; + } return useronly_clean_ptr(ptr); } -uint64_t HELPER(mte_checkN)(CPUARMState *env, uint32_t desc, uint64_t ptr) +uint64_t HELPER(mte_check)(CPUARMState *env, uint32_t desc, uint64_t ptr) { - return mte_checkN(env, desc, ptr, GETPC()); + return mte_check(env, desc, ptr, GETPC()); +} + +/* + * No-fault version of mte_check, to be used by SVE for MemSingleNF. + * Returns false if the access is Checked and the check failed. This + * is only intended to probe the tag -- the validity of the page must + * be checked beforehand. + */ +bool mte_probe(CPUARMState *env, uint32_t desc, uint64_t ptr) +{ + uint64_t fault; + int ret = mte_probe_int(env, desc, ptr, 0, &fault); + + return ret != 0; } /* diff --git a/target/arm/neon-ls.decode b/target/arm/neon-ls.decode index c17f5019e31..0a2a0e15db5 100644 --- a/target/arm/neon-ls.decode +++ b/target/arm/neon-ls.decode @@ -46,7 +46,7 @@ VLD_all_lanes 1111 0100 1 . 1 0 rn:4 .... 11 n:2 size:2 t:1 a:1 rm:4 \ VLDST_single 1111 0100 1 . l:1 0 rn:4 .... 00 n:2 reg_idx:3 align:1 rm:4 \ vd=%vd_dp size=0 stride=1 -VLDST_single 1111 0100 1 . l:1 0 rn:4 .... 01 n:2 reg_idx:2 align:2 rm:4 \ +VLDST_single 1111 0100 1 . l:1 0 rn:4 .... 01 n:2 reg_idx:2 . align:1 rm:4 \ vd=%vd_dp size=1 stride=%imm1_5_p1 -VLDST_single 1111 0100 1 . l:1 0 rn:4 .... 10 n:2 reg_idx:1 align:3 rm:4 \ +VLDST_single 1111 0100 1 . l:1 0 rn:4 .... 10 n:2 reg_idx:1 . align:2 rm:4 \ vd=%vd_dp size=2 stride=%imm1_6_p1 diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index fd6c58f96a8..c068dfa0d57 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -4382,13 +4382,9 @@ static void sve_cont_ldst_watchpoints(SVEContLdSt *info, CPUARMState *env, #endif } -typedef uint64_t mte_check_fn(CPUARMState *, uint32_t, uint64_t, uintptr_t); - -static inline QEMU_ALWAYS_INLINE -void sve_cont_ldst_mte_check_int(SVEContLdSt *info, CPUARMState *env, - uint64_t *vg, target_ulong addr, int esize, - int msize, uint32_t mtedesc, uintptr_t ra, - mte_check_fn *check) +static void sve_cont_ldst_mte_check(SVEContLdSt *info, CPUARMState *env, + uint64_t *vg, target_ulong addr, int esize, + int msize, uint32_t mtedesc, uintptr_t ra) { intptr_t mem_off, reg_off, reg_last; @@ -4405,7 +4401,7 @@ void sve_cont_ldst_mte_check_int(SVEContLdSt *info, CPUARMState *env, uint64_t pg = vg[reg_off >> 6]; do { if ((pg >> (reg_off & 63)) & 1) { - check(env, mtedesc, addr, ra); + mte_check(env, mtedesc, addr, ra); } reg_off += esize; mem_off += msize; @@ -4422,7 +4418,7 @@ void sve_cont_ldst_mte_check_int(SVEContLdSt *info, CPUARMState *env, uint64_t pg = vg[reg_off >> 6]; do { if ((pg >> (reg_off & 63)) & 1) { - check(env, mtedesc, addr, ra); + mte_check(env, mtedesc, addr, ra); } reg_off += esize; mem_off += msize; @@ -4431,30 +4427,6 @@ void sve_cont_ldst_mte_check_int(SVEContLdSt *info, CPUARMState *env, } } -typedef void sve_cont_ldst_mte_check_fn(SVEContLdSt *info, CPUARMState *env, - uint64_t *vg, target_ulong addr, - int esize, int msize, uint32_t mtedesc, - uintptr_t ra); - -static void sve_cont_ldst_mte_check1(SVEContLdSt *info, CPUARMState *env, - uint64_t *vg, target_ulong addr, - int esize, int msize, uint32_t mtedesc, - uintptr_t ra) -{ - sve_cont_ldst_mte_check_int(info, env, vg, addr, esize, msize, - mtedesc, ra, mte_check1); -} - -static void sve_cont_ldst_mte_checkN(SVEContLdSt *info, CPUARMState *env, - uint64_t *vg, target_ulong addr, - int esize, int msize, uint32_t mtedesc, - uintptr_t ra) -{ - sve_cont_ldst_mte_check_int(info, env, vg, addr, esize, msize, - mtedesc, ra, mte_checkN); -} - - /* * Common helper for all contiguous 1,2,3,4-register predicated stores. */ @@ -4463,8 +4435,7 @@ void sve_ldN_r(CPUARMState *env, uint64_t *vg, const target_ulong addr, uint32_t desc, const uintptr_t retaddr, const int esz, const int msz, const int N, uint32_t mtedesc, sve_ldst1_host_fn *host_fn, - sve_ldst1_tlb_fn *tlb_fn, - sve_cont_ldst_mte_check_fn *mte_check_fn) + sve_ldst1_tlb_fn *tlb_fn) { const unsigned rd = simd_data(desc); const intptr_t reg_max = simd_oprsz(desc); @@ -4493,9 +4464,9 @@ void sve_ldN_r(CPUARMState *env, uint64_t *vg, const target_ulong addr, * Handle mte checks for all active elements. * Since TBI must be set for MTE, !mtedesc => !mte_active. */ - if (mte_check_fn && mtedesc) { - mte_check_fn(&info, env, vg, addr, 1 << esz, N << msz, - mtedesc, retaddr); + if (mtedesc) { + sve_cont_ldst_mte_check(&info, env, vg, addr, 1 << esz, N << msz, + mtedesc, retaddr); } flags = info.page[0].flags | info.page[1].flags; @@ -4621,8 +4592,7 @@ void sve_ldN_r_mte(CPUARMState *env, uint64_t *vg, target_ulong addr, mtedesc = 0; } - sve_ldN_r(env, vg, addr, desc, ra, esz, msz, N, mtedesc, host_fn, tlb_fn, - N == 1 ? sve_cont_ldst_mte_check1 : sve_cont_ldst_mte_checkN); + sve_ldN_r(env, vg, addr, desc, ra, esz, msz, N, mtedesc, host_fn, tlb_fn); } #define DO_LD1_1(NAME, ESZ) \ @@ -4630,7 +4600,7 @@ void HELPER(sve_##NAME##_r)(CPUARMState *env, void *vg, \ target_ulong addr, uint32_t desc) \ { \ sve_ldN_r(env, vg, addr, desc, GETPC(), ESZ, MO_8, 1, 0, \ - sve_##NAME##_host, sve_##NAME##_tlb, NULL); \ + sve_##NAME##_host, sve_##NAME##_tlb); \ } \ void HELPER(sve_##NAME##_r_mte)(CPUARMState *env, void *vg, \ target_ulong addr, uint32_t desc) \ @@ -4644,22 +4614,22 @@ void HELPER(sve_##NAME##_le_r)(CPUARMState *env, void *vg, \ target_ulong addr, uint32_t desc) \ { \ sve_ldN_r(env, vg, addr, desc, GETPC(), ESZ, MSZ, 1, 0, \ - sve_##NAME##_le_host, sve_##NAME##_le_tlb, NULL); \ + sve_##NAME##_le_host, sve_##NAME##_le_tlb); \ } \ void HELPER(sve_##NAME##_be_r)(CPUARMState *env, void *vg, \ target_ulong addr, uint32_t desc) \ { \ sve_ldN_r(env, vg, addr, desc, GETPC(), ESZ, MSZ, 1, 0, \ - sve_##NAME##_be_host, sve_##NAME##_be_tlb, NULL); \ + sve_##NAME##_be_host, sve_##NAME##_be_tlb); \ } \ void HELPER(sve_##NAME##_le_r_mte)(CPUARMState *env, void *vg, \ - target_ulong addr, uint32_t desc) \ + target_ulong addr, uint32_t desc) \ { \ sve_ldN_r_mte(env, vg, addr, desc, GETPC(), ESZ, MSZ, 1, \ sve_##NAME##_le_host, sve_##NAME##_le_tlb); \ } \ void HELPER(sve_##NAME##_be_r_mte)(CPUARMState *env, void *vg, \ - target_ulong addr, uint32_t desc) \ + target_ulong addr, uint32_t desc) \ { \ sve_ldN_r_mte(env, vg, addr, desc, GETPC(), ESZ, MSZ, 1, \ sve_##NAME##_be_host, sve_##NAME##_be_tlb); \ @@ -4693,7 +4663,7 @@ void HELPER(sve_ld##N##bb_r)(CPUARMState *env, void *vg, \ target_ulong addr, uint32_t desc) \ { \ sve_ldN_r(env, vg, addr, desc, GETPC(), MO_8, MO_8, N, 0, \ - sve_ld1bb_host, sve_ld1bb_tlb, NULL); \ + sve_ld1bb_host, sve_ld1bb_tlb); \ } \ void HELPER(sve_ld##N##bb_r_mte)(CPUARMState *env, void *vg, \ target_ulong addr, uint32_t desc) \ @@ -4707,13 +4677,13 @@ void HELPER(sve_ld##N##SUFF##_le_r)(CPUARMState *env, void *vg, \ target_ulong addr, uint32_t desc) \ { \ sve_ldN_r(env, vg, addr, desc, GETPC(), ESZ, ESZ, N, 0, \ - sve_ld1##SUFF##_le_host, sve_ld1##SUFF##_le_tlb, NULL); \ + sve_ld1##SUFF##_le_host, sve_ld1##SUFF##_le_tlb); \ } \ void HELPER(sve_ld##N##SUFF##_be_r)(CPUARMState *env, void *vg, \ target_ulong addr, uint32_t desc) \ { \ sve_ldN_r(env, vg, addr, desc, GETPC(), ESZ, ESZ, N, 0, \ - sve_ld1##SUFF##_be_host, sve_ld1##SUFF##_be_tlb, NULL); \ + sve_ld1##SUFF##_be_host, sve_ld1##SUFF##_be_tlb); \ } \ void HELPER(sve_ld##N##SUFF##_le_r_mte)(CPUARMState *env, void *vg, \ target_ulong addr, uint32_t desc) \ @@ -4826,7 +4796,7 @@ void sve_ldnfff1_r(CPUARMState *env, void *vg, const target_ulong addr, if (fault == FAULT_FIRST) { /* Trapping mte check for the first-fault element. */ if (mtedesc) { - mte_check1(env, mtedesc, addr + mem_off, retaddr); + mte_check(env, mtedesc, addr + mem_off, retaddr); } /* @@ -4869,7 +4839,7 @@ void sve_ldnfff1_r(CPUARMState *env, void *vg, const target_ulong addr, /* Watchpoint hit, see below. */ goto do_fault; } - if (mtedesc && !mte_probe1(env, mtedesc, addr + mem_off)) { + if (mtedesc && !mte_probe(env, mtedesc, addr + mem_off)) { goto do_fault; } /* @@ -4919,7 +4889,7 @@ void sve_ldnfff1_r(CPUARMState *env, void *vg, const target_ulong addr, & BP_MEM_READ)) { goto do_fault; } - if (mtedesc && !mte_probe1(env, mtedesc, addr + mem_off)) { + if (mtedesc && !mte_probe(env, mtedesc, addr + mem_off)) { goto do_fault; } host_fn(vd, reg_off, host + mem_off); @@ -5090,8 +5060,7 @@ void sve_stN_r(CPUARMState *env, uint64_t *vg, target_ulong addr, uint32_t desc, const uintptr_t retaddr, const int esz, const int msz, const int N, uint32_t mtedesc, sve_ldst1_host_fn *host_fn, - sve_ldst1_tlb_fn *tlb_fn, - sve_cont_ldst_mte_check_fn *mte_check_fn) + sve_ldst1_tlb_fn *tlb_fn) { const unsigned rd = simd_data(desc); const intptr_t reg_max = simd_oprsz(desc); @@ -5117,9 +5086,9 @@ void sve_stN_r(CPUARMState *env, uint64_t *vg, target_ulong addr, * Handle mte checks for all active elements. * Since TBI must be set for MTE, !mtedesc => !mte_active. */ - if (mte_check_fn && mtedesc) { - mte_check_fn(&info, env, vg, addr, 1 << esz, N << msz, - mtedesc, retaddr); + if (mtedesc) { + sve_cont_ldst_mte_check(&info, env, vg, addr, 1 << esz, N << msz, + mtedesc, retaddr); } flags = info.page[0].flags | info.page[1].flags; @@ -5233,8 +5202,7 @@ void sve_stN_r_mte(CPUARMState *env, uint64_t *vg, target_ulong addr, mtedesc = 0; } - sve_stN_r(env, vg, addr, desc, ra, esz, msz, N, mtedesc, host_fn, tlb_fn, - N == 1 ? sve_cont_ldst_mte_check1 : sve_cont_ldst_mte_checkN); + sve_stN_r(env, vg, addr, desc, ra, esz, msz, N, mtedesc, host_fn, tlb_fn); } #define DO_STN_1(N, NAME, ESZ) \ @@ -5242,7 +5210,7 @@ void HELPER(sve_st##N##NAME##_r)(CPUARMState *env, void *vg, \ target_ulong addr, uint32_t desc) \ { \ sve_stN_r(env, vg, addr, desc, GETPC(), ESZ, MO_8, N, 0, \ - sve_st1##NAME##_host, sve_st1##NAME##_tlb, NULL); \ + sve_st1##NAME##_host, sve_st1##NAME##_tlb); \ } \ void HELPER(sve_st##N##NAME##_r_mte)(CPUARMState *env, void *vg, \ target_ulong addr, uint32_t desc) \ @@ -5256,13 +5224,13 @@ void HELPER(sve_st##N##NAME##_le_r)(CPUARMState *env, void *vg, \ target_ulong addr, uint32_t desc) \ { \ sve_stN_r(env, vg, addr, desc, GETPC(), ESZ, MSZ, N, 0, \ - sve_st1##NAME##_le_host, sve_st1##NAME##_le_tlb, NULL); \ + sve_st1##NAME##_le_host, sve_st1##NAME##_le_tlb); \ } \ void HELPER(sve_st##N##NAME##_be_r)(CPUARMState *env, void *vg, \ target_ulong addr, uint32_t desc) \ { \ sve_stN_r(env, vg, addr, desc, GETPC(), ESZ, MSZ, N, 0, \ - sve_st1##NAME##_be_host, sve_st1##NAME##_be_tlb, NULL); \ + sve_st1##NAME##_be_host, sve_st1##NAME##_be_tlb); \ } \ void HELPER(sve_st##N##NAME##_le_r_mte)(CPUARMState *env, void *vg, \ target_ulong addr, uint32_t desc) \ @@ -5373,7 +5341,7 @@ void sve_ld1_z(CPUARMState *env, void *vd, uint64_t *vg, void *vm, info.attrs, BP_MEM_READ, retaddr); } if (mtedesc && arm_tlb_mte_tagged(&info.attrs)) { - mte_check1(env, mtedesc, addr, retaddr); + mte_check(env, mtedesc, addr, retaddr); } host_fn(&scratch, reg_off, info.host); } else { @@ -5386,7 +5354,7 @@ void sve_ld1_z(CPUARMState *env, void *vd, uint64_t *vg, void *vm, BP_MEM_READ, retaddr); } if (mtedesc && arm_tlb_mte_tagged(&info.attrs)) { - mte_check1(env, mtedesc, addr, retaddr); + mte_check(env, mtedesc, addr, retaddr); } tlb_fn(env, &scratch, reg_off, addr, retaddr); } @@ -5552,7 +5520,7 @@ void sve_ldff1_z(CPUARMState *env, void *vd, uint64_t *vg, void *vm, */ addr = base + (off_fn(vm, reg_off) << scale); if (mtedesc) { - mte_check1(env, mtedesc, addr, retaddr); + mte_check(env, mtedesc, addr, retaddr); } tlb_fn(env, vd, reg_off, addr, retaddr); @@ -5588,7 +5556,7 @@ void sve_ldff1_z(CPUARMState *env, void *vd, uint64_t *vg, void *vm, } if (mtedesc && arm_tlb_mte_tagged(&info.attrs) && - !mte_probe1(env, mtedesc, addr)) { + !mte_probe(env, mtedesc, addr)) { goto fault; } @@ -5773,7 +5741,7 @@ void sve_st1_z(CPUARMState *env, void *vd, uint64_t *vg, void *vm, } if (mtedesc && arm_tlb_mte_tagged(&info.attrs)) { - mte_check1(env, mtedesc, addr, retaddr); + mte_check(env, mtedesc, addr, retaddr); } } i += 1; diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index 2ce7b26de2d..77727b41002 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -67,24 +67,6 @@ static inline bool cctlr_set(DisasContext *ctx, uint32_t bits) return (ctx->base.cheri_flags & mask) == mask; } -static inline MemOp memop_align_sctlr(DisasContext *ctx) -{ -#ifdef STRICT_ALIGNMENT_CHECKS - return GET_FLAG(ctx, SCTLRA) ? MO_ALIGN : 0; -#else - return 0; -#endif -} - -static inline MemOp memop_align_sctlr_size(DisasContext *ctx, int size) -{ -#ifdef STRICT_ALIGNMENT_CHECKS - return GET_FLAG(ctx, SCTLRA) ? (size << MO_ASHIFT) : 0; -#else - return 0; -#endif -} - static inline bool get_sctlr_sa(DisasContext *ctx) { #ifdef STRICT_ALIGNMENT_CHECKS @@ -108,11 +90,6 @@ static inline MemOp memop_align_sctlr(DisasContext *ctx) return 0; } -static inline MemOp memop_align_sctlr_size(DisasContext *ctx, int size) -{ - return 0; -} - static inline bool get_sctlr_sa(DisasContext *ctx) { return false; @@ -568,11 +545,11 @@ gen_mte_and_cheri_check1_mmuidx(DisasContext *s, TCGv_i64 addr, bool is_read, desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid); desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma); desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write); - desc = FIELD_DP32(desc, MTEDESC, ESIZE, 1 << log2_size); + desc = FIELD_DP32(desc, MTEDESC, SIZEM1, (1 << log2_size) - 1); tcg_desc = tcg_const_i32(desc); ret = new_tmp_a64(s); - gen_helper_mte_check1(ret, cpu_env, tcg_desc, addr); + gen_helper_mte_check(ret, cpu_env, tcg_desc, addr); tcg_temp_free_i32(tcg_desc); return arm_bounds_checked(s, ret, (1 << log2_size), base_reg, is_read, @@ -598,39 +575,31 @@ TCGv_cap_checked_ptr gen_mte_and_cheri_check1(DisasContext *s, TCGv_i64 addr, */ TCGv_cap_checked_ptr gen_mte_and_cheri_checkN(DisasContext *s, TCGv_i64 addr, bool is_read, bool is_write, - bool tag_checked, int log2_esize, - int total_size, int base_reg, - bool alternate_base, + bool tag_checked, int size, + int base_reg, bool alternate_base, bool ddc_base) { - if (total_size != (1 << log2_esize)) { - if (tag_checked && s->mte_active[0]) { - TCGv_i32 tcg_desc; - TCGv_i64 ret; - int desc = 0; + if (tag_checked && s->mte_active[0]) { + TCGv_i32 tcg_desc; + TCGv_i64 ret; + int desc = 0; - desc = FIELD_DP32(desc, MTEDESC, MIDX, get_mem_index(s)); - desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid); - desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma); - desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write); - desc = FIELD_DP32(desc, MTEDESC, ESIZE, 1 << log2_esize); - desc = FIELD_DP32(desc, MTEDESC, TSIZE, total_size); - tcg_desc = tcg_const_i32(desc); + desc = FIELD_DP32(desc, MTEDESC, MIDX, get_mem_index(s)); + desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid); + desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma); + desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write); + desc = FIELD_DP32(desc, MTEDESC, SIZEM1, size - 1); + tcg_desc = tcg_const_i32(desc); - ret = new_tmp_a64(s); - gen_helper_mte_checkN(ret, cpu_env, tcg_desc, addr); - tcg_temp_free_i32(tcg_desc); + ret = new_tmp_a64(s); + gen_helper_mte_check(ret, cpu_env, tcg_desc, addr); + tcg_temp_free_i32(tcg_desc); - return arm_bounds_checked(s, ret, total_size, base_reg, is_read, - is_write, alternate_base, ddc_base); - } else { - clean_data_tbi_and_cheri(s, addr, is_read, is_write, total_size, - base_reg, alternate_base, ddc_base); - } + return arm_bounds_checked(s, ret, size, base_reg, is_read, + is_write, alternate_base, ddc_base); } - return gen_mte_and_cheri_check1(s, addr, is_read, is_write, tag_checked, - log2_esize, base_reg, alternate_base, - ddc_base); + return clean_data_tbi_and_cheri(s, addr, is_read, is_write, size, + base_reg, alternate_base, ddc_base); } typedef struct DisasCompare64 { @@ -1218,19 +1187,18 @@ static void gen_adc_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1) * Store from GPR register to memory. */ static void do_gpr_st_memidx(DisasContext *s, TCGv_i64 source, - TCGv_cap_checked_ptr tcg_addr, int size, + TCGv_cap_checked_ptr tcg_addr, MemOp memop, int memidx, bool iss_valid, unsigned int iss_srt, bool iss_sf, bool iss_ar) { - g_assert(size <= 3); - tcg_gen_qemu_st_i64_with_checked_addr( - source, tcg_addr, memidx, s->be_data + size | memop_align_sctlr(s)); + memop = finalize_memop(s, memop); + tcg_gen_qemu_st_i64_with_checked_addr(source, tcg_addr, memidx, memop); if (iss_valid) { uint32_t syn; syn = syn_data_abort_with_iss(0, - size, + (memop & MO_SIZE), false, iss_srt, iss_sf, @@ -1241,10 +1209,12 @@ static void do_gpr_st_memidx(DisasContext *s, TCGv_i64 source, } static void do_gpr_st(DisasContext *s, TCGv_i64 source, - TCGv_cap_checked_ptr tcg_addr, int size, bool iss_valid, - unsigned int iss_srt, bool iss_sf, bool iss_ar) + TCGv_cap_checked_ptr tcg_addr, MemOp memop, + bool iss_valid, + unsigned int iss_srt, + bool iss_sf, bool iss_ar) { - do_gpr_st_memidx(s, source, tcg_addr, size, get_mem_index(s), + do_gpr_st_memidx(s, source, tcg_addr, memop, get_mem_index(s), iss_valid, iss_srt, iss_sf, iss_ar); } @@ -1252,23 +1222,16 @@ static void do_gpr_st(DisasContext *s, TCGv_i64 source, * Load from memory to GPR register */ static void do_gpr_ld_memidx(DisasContext *s, TCGv_i64 dest, - TCGv_cap_checked_ptr tcg_addr, int size, - bool is_signed, bool extend, int memidx, - bool iss_valid, unsigned int iss_srt, bool iss_sf, - bool iss_ar) + TCGv_cap_checked_ptr tcg_addr, + MemOp memop, bool extend, int memidx, + bool iss_valid, unsigned int iss_srt, + bool iss_sf, bool iss_ar) { - MemOp memop = s->be_data + size | memop_align_sctlr(s); - - g_assert(size <= 3); - - if (is_signed) { - memop += MO_SIGN; - } - + memop = finalize_memop(s, memop); tcg_gen_qemu_ld_i64_with_checked_addr(dest, tcg_addr, memidx, memop); - if (extend && is_signed) { - g_assert(size < 3); + if (extend && (memop & MO_SIGN)) { + g_assert((memop & MO_SIZE) <= MO_32); tcg_gen_ext32u_i64(dest, dest); } @@ -1281,8 +1244,8 @@ static void do_gpr_ld_memidx(DisasContext *s, TCGv_i64 dest, uint32_t syn; syn = syn_data_abort_with_iss(0, - size, - is_signed, + (memop & MO_SIZE), + (memop & MO_SIGN) != 0, iss_srt, iss_sf, iss_ar, @@ -1292,12 +1255,12 @@ static void do_gpr_ld_memidx(DisasContext *s, TCGv_i64 dest, } static void do_gpr_ld(DisasContext *s, TCGv_i64 dest, - TCGv_cap_checked_ptr tcg_addr, int size, bool is_signed, - bool extend, bool iss_valid, unsigned int iss_srt, + TCGv_cap_checked_ptr tcg_addr, + MemOp memop, bool extend, + bool iss_valid, unsigned int iss_srt, bool iss_sf, bool iss_ar) { - do_gpr_ld_memidx(s, dest, tcg_addr, size, is_signed, extend, - get_mem_index(s), + do_gpr_ld_memidx(s, dest, tcg_addr, memop, extend, get_mem_index(s), iss_valid, iss_srt, iss_sf, iss_ar); } @@ -1308,30 +1271,33 @@ static void do_fp_st(DisasContext *s, int srcidx, TCGv_cap_checked_ptr tcg_addr, int size) { /* This writes the bottom N bits of a 128 bit wide vector to memory */ - TCGv_i64 tmp = tcg_temp_new_i64(); + TCGv_i64 tmplo = tcg_temp_new_i64(); + MemOp mop; + + tcg_gen_ld_i64(tmplo, cpu_env, fp_reg_offset(s, srcidx, MO_64)); - MemOp memop = s->be_data; - MemOp align = memop_align_sctlr_size(s, size); - tcg_gen_ld_i64(tmp, cpu_env, fp_reg_offset(s, srcidx, MO_64)); if (size < 4) { - tcg_gen_qemu_st_i64_with_checked_addr(tmp, tcg_addr, get_mem_index(s), - memop + size | align); + mop = finalize_memop(s, size); + tcg_gen_qemu_st_i64_with_checked_addr(tmplo, tcg_addr, get_mem_index(s), mop); } else { bool be = s->be_data == MO_BE; TCGv_cap_checked_ptr tcg_hiaddr = tcg_temp_new_cap_checked(); + TCGv_i64 tmphi = tcg_temp_new_i64(); + tcg_gen_ld_i64(tmphi, cpu_env, fp_reg_hi_offset(s, srcidx)); + + mop = s->be_data | MO_Q; + tcg_gen_qemu_st_i64_with_checked_addr(be ? tmphi : tmplo, tcg_addr, get_mem_index(s), + mop | (s->align_mem ? MO_ALIGN_16 : 0)); tcg_gen_addi_i64((TCGv_i64)tcg_hiaddr, (TCGv_i64)tcg_addr, 8); - // If the first load is correctly aligned, so will the second - tcg_gen_qemu_st_i64_with_checked_addr(tmp, be ? tcg_hiaddr : tcg_addr, - get_mem_index(s), - memop | MO_Q | align); - tcg_gen_ld_i64(tmp, cpu_env, fp_reg_hi_offset(s, srcidx)); - tcg_gen_qemu_st_i64_with_checked_addr(tmp, be ? tcg_addr : tcg_hiaddr, - get_mem_index(s), memop | MO_Q); + tcg_gen_qemu_st_i64_with_checked_addr(be ? tmplo : tmphi, tcg_hiaddr, + get_mem_index(s), mop); + tcg_temp_free_cap_checked(tcg_hiaddr); + tcg_temp_free_i64(tmphi); } - tcg_temp_free_i64(tmp); + tcg_temp_free_i64(tmplo); } /* @@ -1343,13 +1309,12 @@ static void do_fp_ld(DisasContext *s, int destidx, /* This always zero-extends and writes to a full 128 bit wide vector */ TCGv_i64 tmplo = tcg_temp_new_i64(); TCGv_i64 tmphi = NULL; - - MemOp memop = s->be_data; - MemOp align = memop_align_sctlr_size(s, size); + MemOp mop; if (size < 4) { - tcg_gen_qemu_ld_i64_with_checked_addr(tmplo, tcg_addr, get_mem_index(s), - memop + size | align); + mop = finalize_memop(s, size); + tcg_gen_qemu_ld_i64_with_checked_addr(tmplo, tcg_addr, + get_mem_index(s), mop); } else { bool be = s->be_data == MO_BE; TCGv_cap_checked_ptr tcg_hiaddr; @@ -1357,12 +1322,13 @@ static void do_fp_ld(DisasContext *s, int destidx, tmphi = tcg_temp_new_i64(); tcg_hiaddr = tcg_temp_new_cap_checked(); + mop = s->be_data | MO_Q; + tcg_gen_qemu_ld_i64_with_checked_addr( + be ? tmphi : tmplo, tcg_addr, get_mem_index(s), + mop | (s->align_mem ? MO_ALIGN_16 : 0)); tcg_gen_addi_i64((TCGv_i64)tcg_hiaddr, (TCGv_i64)tcg_addr, 8); - tcg_gen_qemu_ld_i64_with_checked_addr(tmplo, be ? tcg_hiaddr : tcg_addr, - get_mem_index(s), - memop | MO_Q | align); - tcg_gen_qemu_ld_i64_with_checked_addr(tmphi, be ? tcg_addr : tcg_hiaddr, - get_mem_index(s), memop | MO_Q); + tcg_gen_qemu_ld_i64_with_checked_addr(be ? tmplo : tmphi, tcg_hiaddr, + get_mem_index(s), mop); tcg_temp_free_cap_checked(tcg_hiaddr); } @@ -1491,26 +1457,26 @@ static void write_vec_element_i32(DisasContext *s, TCGv_i32 tcg_src, /* Store from vector register to memory */ static void do_vec_st(DisasContext *s, int srcidx, int element, - TCGv_cap_checked_ptr tcg_addr, int size, MemOp endian) + TCGv_cap_checked_ptr tcg_addr, MemOp mop) { TCGv_i64 tcg_tmp = tcg_temp_new_i64(); - read_vec_element(s, tcg_tmp, srcidx, element, size); + read_vec_element(s, tcg_tmp, srcidx, element, mop & MO_SIZE); tcg_gen_qemu_st_i64_with_checked_addr(tcg_tmp, tcg_addr, get_mem_index(s), - endian | size | memop_align_sctlr(s)); + mop); tcg_temp_free_i64(tcg_tmp); } /* Load from memory to vector register */ static void do_vec_ld(DisasContext *s, int destidx, int element, - TCGv_cap_checked_ptr tcg_addr, int size, MemOp endian) + TCGv_cap_checked_ptr tcg_addr, MemOp mop) { TCGv_i64 tcg_tmp = tcg_temp_new_i64(); tcg_gen_qemu_ld_i64_with_checked_addr(tcg_tmp, tcg_addr, get_mem_index(s), - endian | size | memop_align_sctlr(s)); - write_vec_element(s, tcg_tmp, destidx, element, size); + mop); + write_vec_element(s, tcg_tmp, destidx, element, mop & MO_SIZE); tcg_temp_free_i64(tcg_tmp); } @@ -3318,7 +3284,8 @@ static void disas_ldst_excl(DisasContext *s, uint32_t insn) tcg_gen_mb(TCG_MO_ALL | TCG_BAR_STRL); clean_addr = gen_mte_and_cheri_check1(s, cpu_reg_sp(s, rn), false, true, rn != 31, size, rn, false, true); - do_gpr_st(s, cpu_reg(s, rt), clean_addr, size, true, rt, + /* TODO: ARMv8.4-LSE SCTLR.nAA */ + do_gpr_st(s, cpu_reg(s, rt), clean_addr, size | MO_ALIGN, true, rt, disas_ldst_compute_iss_sf(size, false, 0), is_lasr); return; @@ -3335,8 +3302,9 @@ static void disas_ldst_excl(DisasContext *s, uint32_t insn) } clean_addr = gen_mte_and_cheri_check1(s, cpu_reg_sp(s, rn), true, false, rn != 31, size, rn, false, true); - do_gpr_ld(s, cpu_reg(s, rt), clean_addr, size, false, false, true, rt, - disas_ldst_compute_iss_sf(size, false, 0), is_lasr); + /* TODO: ARMv8.4-LSE SCTLR.nAA */ + do_gpr_ld(s, cpu_reg(s, rt), clean_addr, size | MO_ALIGN, false, true, + rt, disas_ldst_compute_iss_sf(size, false, 0), is_lasr); tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ); return; @@ -3455,8 +3423,8 @@ static void disas_ld_lit(DisasContext *s, uint32_t insn) /* Only unsigned 32bit loads target 32bit registers. */ bool iss_sf = opc != 0; - do_gpr_ld(s, tcg_rt, clean_addr, size, is_signed, false, - true, rt, iss_sf, false); + do_gpr_ld(s, tcg_rt, clean_addr, size + is_signed * MO_SIGN, + false, true, rt, iss_sf, false); } tcg_temp_free_i64(dirty_addr); } @@ -3591,7 +3559,7 @@ static void disas_ldst_pair(DisasContext *s, uint32_t insn) } clean_addr = gen_mte_and_cheri_checkN(s, dirty_addr, is_load, !is_load, - (wback || rn != 31) && !set_tag, size, + (wback || rn != 31) && !set_tag, 2 << size, rn, false, true); if (is_vector) { @@ -3616,12 +3584,12 @@ static void disas_ldst_pair(DisasContext *s, uint32_t insn) /* Do not modify tcg_rt before recognizing any exception * from the second load. */ - do_gpr_ld(s, tmp, clean_addr, size, is_signed, false, false, - (unsigned int)-1, false, false); + do_gpr_ld(s, tmp, clean_addr, size + is_signed * MO_SIGN, + false, false, (unsigned int)-1, false, false); tcg_gen_addi_i64((TCGv_i64)clean_addr, (TCGv_i64)clean_addr, 1 << size); - do_gpr_ld(s, tcg_rt2, clean_addr, size, is_signed, false, false, - (unsigned int)-1, false, false); + do_gpr_ld(s, tcg_rt2, clean_addr, size + is_signed * MO_SIGN, + false, false, (unsigned int)-1, false, false); tcg_gen_mov_i64(tcg_rt, tmp); tcg_temp_free_i64(tmp); @@ -3756,8 +3724,8 @@ static void disas_ldst_reg_imm9(DisasContext *s, uint32_t insn, do_gpr_st_memidx(s, tcg_rt, clean_addr, size, memidx, iss_valid, rt, iss_sf, false); } else { - do_gpr_ld_memidx(s, tcg_rt, clean_addr, size, - is_signed, is_extended, memidx, + do_gpr_ld_memidx(s, tcg_rt, clean_addr, size + is_signed * MO_SIGN, + is_extended, memidx, iss_valid, rt, iss_sf, false); } } @@ -3862,9 +3830,8 @@ static void disas_ldst_reg_roffset(DisasContext *s, uint32_t insn, do_gpr_st(s, tcg_rt, clean_addr, size, true, rt, iss_sf, false); } else { - do_gpr_ld(s, tcg_rt, clean_addr, size, - is_signed, is_extended, - true, rt, iss_sf, false); + do_gpr_ld(s, tcg_rt, clean_addr, size + is_signed * MO_SIGN, + is_extended, true, rt, iss_sf, false); } } } @@ -3949,8 +3916,8 @@ static void disas_ldst_reg_unsigned_imm(DisasContext *s, uint32_t insn, do_gpr_st(s, tcg_rt, clean_addr, size, true, rt, iss_sf, false); } else { - do_gpr_ld(s, tcg_rt, clean_addr, size, is_signed, is_extended, - true, rt, iss_sf, false); + do_gpr_ld(s, tcg_rt, clean_addr, size + is_signed * MO_SIGN, + is_extended, true, rt, iss_sf, false); } } } @@ -4040,7 +4007,7 @@ static void disas_ldst_atomic(DisasContext *s, uint32_t insn, * full load-acquire (we only need "load-acquire processor consistent"), * but we choose to implement them as full LDAQ. */ - do_gpr_ld(s, cpu_reg(s, rt), clean_addr, size, false, false, + do_gpr_ld(s, cpu_reg(s, rt), clean_addr, size, false, true, rt, disas_ldst_compute_iss_sf(size, false, 0), true); tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ); return; @@ -4117,7 +4084,7 @@ static void disas_ldst_pac(DisasContext *s, uint32_t insn, is_wback || rn != 31, size, rn, false, true); tcg_rt = cpu_reg(s, rt); - do_gpr_ld(s, tcg_rt, clean_addr, size, /* is_signed */ false, + do_gpr_ld(s, tcg_rt, clean_addr, size, /* extend */ false, /* iss_valid */ !is_wback, /* iss_srt */ rt, /* iss_sf */ true, /* iss_ar */ false); @@ -4150,15 +4117,18 @@ static void disas_ldst_ldapr_stlr(DisasContext *s, uint32_t insn) TCGv_cap_checked_ptr clean_addr; TCGv_i64 dirty_addr; bool is_store = false; - bool is_signed = false; bool extend = false; bool iss_sf; + MemOp mop; if (!dc_isar_feature(aa64_rcpc_8_4, s)) { unallocated_encoding(s); return; } + /* TODO: ARMv8.4-LSE SCTLR.nAA */ + mop = size | MO_ALIGN; + switch (opc) { case 0: /* STLURB */ is_store = true; @@ -4170,21 +4140,21 @@ static void disas_ldst_ldapr_stlr(DisasContext *s, uint32_t insn) unallocated_encoding(s); return; } - is_signed = true; + mop |= MO_SIGN; break; case 3: /* LDAPURS* 32-bit variant */ if (size > 1) { unallocated_encoding(s); return; } - is_signed = true; + mop |= MO_SIGN; extend = true; /* zero-extend 32->64 after signed load */ break; default: g_assert_not_reached(); } - iss_sf = disas_ldst_compute_iss_sf(size, is_signed, opc); + iss_sf = disas_ldst_compute_iss_sf(size, (mop & MO_SIGN) != 0, opc); if (rn == 31) { gen_check_sp_alignment(s); @@ -4199,14 +4169,14 @@ static void disas_ldst_ldapr_stlr(DisasContext *s, uint32_t insn) if (is_store) { /* Store-Release semantics */ tcg_gen_mb(TCG_MO_ALL | TCG_BAR_STRL); - do_gpr_st(s, cpu_reg(s, rt), clean_addr, size, true, rt, iss_sf, true); + do_gpr_st(s, cpu_reg(s, rt), clean_addr, mop, true, rt, iss_sf, true); } else { /* * Load-AcquirePC semantics; we implement as the slightly more * restrictive Load-Acquire. */ - do_gpr_ld(s, cpu_reg(s, rt), clean_addr, size, is_signed, extend, - true, rt, iss_sf, true); + do_gpr_ld(s, cpu_reg(s, rt), clean_addr, mop, + extend, true, rt, iss_sf, true); tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ); } } @@ -4278,7 +4248,7 @@ static void disas_ldst_multiple_struct(DisasContext *s, uint32_t insn) bool is_q = extract32(insn, 30, 1); TCGv_cap_checked_ptr clean_addr; TCGv_i64 tcg_rn, tcg_ebytes; - MemOp endian = s->be_data; + MemOp endian, align, mop; int total; /* total bytes */ int elements; /* elements per vector */ @@ -4346,6 +4316,7 @@ static void disas_ldst_multiple_struct(DisasContext *s, uint32_t insn) } /* For our purposes, bytes are always little-endian. */ + endian = s->be_data; if (size == 0) { endian = MO_LE; } @@ -4358,18 +4329,24 @@ static void disas_ldst_multiple_struct(DisasContext *s, uint32_t insn) * promote consecutive little-endian elements below. */ clean_addr = gen_mte_and_cheri_checkN(s, tcg_rn, !is_store, is_store, - is_postidx || rn != 31, size, total, + is_postidx || rn != 31, total, rn, false, true); /* * Consecutive little-endian elements from a single register * can be promoted to a larger little-endian operation. */ + align = MO_ALIGN; if (selem == 1 && endian == MO_LE) { + align = pow2_align(size); size = 3; } - elements = (is_q ? 16 : 8) >> size; + if (!s->align_mem) { + align = 0; + } + mop = endian | size | align; + elements = (is_q ? 16 : 8) >> size; tcg_ebytes = tcg_const_i64(1 << size); for (r = 0; r < rpt; r++) { int e; @@ -4378,9 +4355,9 @@ static void disas_ldst_multiple_struct(DisasContext *s, uint32_t insn) for (xs = 0; xs < selem; xs++) { int tt = (rt + r + xs) % 32; if (is_store) { - do_vec_st(s, tt, e, clean_addr, size, endian); + do_vec_st(s, tt, e, clean_addr, mop); } else { - do_vec_ld(s, tt, e, clean_addr, size, endian); + do_vec_ld(s, tt, e, clean_addr, mop); } tcg_gen_add_i64((TCGv_i64)clean_addr, (TCGv_i64)clean_addr, tcg_ebytes); @@ -4457,6 +4434,7 @@ static void disas_ldst_single_struct(DisasContext *s, uint32_t insn) int xs, total; TCGv_cap_checked_ptr clean_addr; TCGv_i64 tcg_rn, tcg_ebytes; + MemOp mop; if (extract32(insn, 31, 1)) { unallocated_encoding(s); @@ -4517,8 +4495,9 @@ static void disas_ldst_single_struct(DisasContext *s, uint32_t insn) tcg_rn = cpu_reg_sp(s, rn); clean_addr = gen_mte_and_cheri_checkN(s, tcg_rn, is_load, !is_load, - is_postidx || rn != 31, scale, total, + is_postidx || rn != 31, total, rn, false, true); + mop = finalize_memop(s, scale); tcg_ebytes = tcg_const_i64(1 << scale); for (xs = 0; xs < selem; xs++) { @@ -4526,8 +4505,8 @@ static void disas_ldst_single_struct(DisasContext *s, uint32_t insn) /* Load and replicate to all elements */ TCGv_i64 tcg_tmp = tcg_temp_new_i64(); - tcg_gen_qemu_ld_i64_with_checked_addr( - tcg_tmp, clean_addr, get_mem_index(s), s->be_data + scale); + tcg_gen_qemu_ld_i64_with_checked_addr(tcg_tmp, clean_addr, + get_mem_index(s), mop); tcg_gen_gvec_dup_i64(scale, vec_full_reg_offset(s, rt), (is_q + 1) * 8, vec_full_reg_size(s), tcg_tmp); @@ -4535,9 +4514,9 @@ static void disas_ldst_single_struct(DisasContext *s, uint32_t insn) } else { /* Load/store one element per register */ if (is_load) { - do_vec_ld(s, rt, index, clean_addr, scale, s->be_data); + do_vec_ld(s, rt, index, clean_addr, mop); } else { - do_vec_st(s, rt, index, clean_addr, scale, s->be_data); + do_vec_st(s, rt, index, clean_addr, mop); } } tcg_gen_add_i64((TCGv_i64)clean_addr, (TCGv_i64)clean_addr, tcg_ebytes); @@ -15420,7 +15399,7 @@ static void aarch64_tr_init_disas_context(DisasContextBase *dcbase, DisasContext *dc = container_of(dcbase, DisasContext, base); CPUARMState *env = cpu->env_ptr; ARMCPU *arm_cpu = env_archcpu(env); - uint32_t tb_flags = dc->base.tb->flags; + CPUARMTBFlags tb_flags = arm_tbflags_from_tb(dc->base.tb); int bound, core_mmu_idx; dc->isar = &arm_cpu->isar; @@ -15434,28 +15413,29 @@ static void aarch64_tr_init_disas_context(DisasContextBase *dcbase, !arm_el_is_aa64(env, 3); dc->thumb = 0; dc->sctlr_b = 0; - dc->be_data = FIELD_EX32(tb_flags, TBFLAG_ANY, BE_DATA) ? MO_BE : MO_LE; + dc->be_data = EX_TBFLAG_ANY(tb_flags, BE_DATA) ? MO_BE : MO_LE; dc->condexec_mask = 0; dc->condexec_cond = 0; - core_mmu_idx = FIELD_EX32(tb_flags, TBFLAG_ANY, MMUIDX); + core_mmu_idx = EX_TBFLAG_ANY(tb_flags, MMUIDX); dc->mmu_idx = core_to_aa64_mmu_idx(core_mmu_idx); - dc->tbii = FIELD_EX32(tb_flags, TBFLAG_A64, TBII); - dc->tbid = FIELD_EX32(tb_flags, TBFLAG_A64, TBID); - dc->tcma = FIELD_EX32(tb_flags, TBFLAG_A64, TCMA); + dc->tbii = EX_TBFLAG_A64(tb_flags, TBII); + dc->tbid = EX_TBFLAG_A64(tb_flags, TBID); + dc->tcma = EX_TBFLAG_A64(tb_flags, TCMA); dc->current_el = arm_mmu_idx_to_el(dc->mmu_idx); #if !defined(CONFIG_USER_ONLY) dc->user = (dc->current_el == 0); #endif - dc->fp_excp_el = FIELD_EX32(tb_flags, TBFLAG_ANY, FPEXC_EL); - dc->sve_excp_el = FIELD_EX32(tb_flags, TBFLAG_A64, SVEEXC_EL); - dc->sve_len = (FIELD_EX32(tb_flags, TBFLAG_A64, ZCR_LEN) + 1) * 16; - dc->pauth_active = FIELD_EX32(tb_flags, TBFLAG_A64, PAUTH_ACTIVE); - dc->bt = FIELD_EX32(tb_flags, TBFLAG_A64, BT); - dc->btype = FIELD_EX32(tb_flags, TBFLAG_A64, BTYPE); - dc->unpriv = FIELD_EX32(tb_flags, TBFLAG_A64, UNPRIV); - dc->ata = FIELD_EX32(tb_flags, TBFLAG_A64, ATA); - dc->mte_active[0] = FIELD_EX32(tb_flags, TBFLAG_A64, MTE_ACTIVE); - dc->mte_active[1] = FIELD_EX32(tb_flags, TBFLAG_A64, MTE0_ACTIVE); + dc->fp_excp_el = EX_TBFLAG_ANY(tb_flags, FPEXC_EL); + dc->align_mem = EX_TBFLAG_ANY(tb_flags, ALIGN_MEM); + dc->sve_excp_el = EX_TBFLAG_A64(tb_flags, SVEEXC_EL); + dc->sve_len = (EX_TBFLAG_A64(tb_flags, ZCR_LEN) + 1) * 16; + dc->pauth_active = EX_TBFLAG_A64(tb_flags, PAUTH_ACTIVE); + dc->bt = EX_TBFLAG_A64(tb_flags, BT); + dc->btype = EX_TBFLAG_A64(tb_flags, BTYPE); + dc->unpriv = EX_TBFLAG_A64(tb_flags, UNPRIV); + dc->ata = EX_TBFLAG_A64(tb_flags, ATA); + dc->mte_active[0] = EX_TBFLAG_A64(tb_flags, MTE_ACTIVE); + dc->mte_active[1] = EX_TBFLAG_A64(tb_flags, MTE0_ACTIVE); dc->vec_len = 0; dc->vec_stride = 0; dc->cp_regs = arm_cpu->cp_regs; @@ -15482,10 +15462,10 @@ static void aarch64_tr_init_disas_context(DisasContextBase *dcbase, * emit code to generate a software step exception * end the TB */ - dc->ss_active = FIELD_EX32(tb_flags, TBFLAG_ANY, SS_ACTIVE); - dc->pstate_ss = FIELD_EX32(tb_flags, TBFLAG_ANY, PSTATE_SS); + dc->ss_active = EX_TBFLAG_ANY(tb_flags, SS_ACTIVE); + dc->pstate_ss = EX_TBFLAG_ANY(tb_flags, PSTATE__SS); dc->is_ldex = false; - dc->debug_target_el = FIELD_EX32(tb_flags, TBFLAG_ANY, DEBUG_TARGET_EL); + dc->debug_target_el = EX_TBFLAG_ANY(tb_flags, DEBUG_TARGET_EL); /* Bound the number of insns to execute to those left on the page. */ bound = -(dc->base.pc_first | TARGET_PAGE_MASK) / 4; diff --git a/target/arm/translate-a64.h b/target/arm/translate-a64.h index a31defb3097..f30e3cfbc24 100644 --- a/target/arm/translate-a64.h +++ b/target/arm/translate-a64.h @@ -54,9 +54,8 @@ TCGv_cap_checked_ptr gen_mte_and_cheri_check1(DisasContext *s, TCGv_i64 addr, bool ddc_base); TCGv_cap_checked_ptr gen_mte_and_cheri_checkN(DisasContext *s, TCGv_i64 addr, bool is_read, bool is_write, - bool tag_checked, int log2_esize, - int total_size, int base_reg, - bool alternate_base, + bool tag_checked, int size, + int base_reg, bool alternate_base, bool ddc_base); /* We should have at some point before trying to access an FP register diff --git a/target/arm/translate-cheri.inc.c b/target/arm/translate-cheri.inc.c index 2fb749a1123..8fddadd777e 100644 --- a/target/arm/translate-cheri.inc.c +++ b/target/arm/translate-cheri.inc.c @@ -446,9 +446,10 @@ static inline __attribute__((always_inline)) bool load_store_implementation( } else if (!vector) { MemOp memop = ctx->be_data + size; - bool fault_unaligned = exclusive || acquire_release; - if (fault_unaligned || memop_align_sctlr(ctx)) + if (exclusive || acquire_release) memop |= MO_ALIGN; + else + memop = finalize_memop(ctx, memop); /* enforce alignment */ TCGv_i64 tcg_rd = cpu_reg_maybe_0(ctx, rd); TCGv_i64 tcg_rd2; diff --git a/target/arm/translate-neon.c.inc b/target/arm/translate-neon.c.inc index f6c68e30ab2..a02b8369a1d 100644 --- a/target/arm/translate-neon.c.inc +++ b/target/arm/translate-neon.c.inc @@ -429,7 +429,7 @@ static bool trans_VLDST_multiple(DisasContext *s, arg_VLDST_multiple *a) { /* Neon load/store multiple structures */ int nregs, interleave, spacing, reg, n; - MemOp endian = s->be_data; + MemOp mop, align, endian; int mmu_idx = get_mem_index(s); int size = a->size; TCGv_i64 tmp64; @@ -473,20 +473,36 @@ static bool trans_VLDST_multiple(DisasContext *s, arg_VLDST_multiple *a) } /* For our purposes, bytes are always little-endian. */ + endian = s->be_data; if (size == 0) { endian = MO_LE; } + + /* Enforce alignment requested by the instruction */ + if (a->align) { + align = pow2_align(a->align + 2); /* 4 ** a->align */ + } else { + align = s->align_mem ? MO_ALIGN : 0; + } + /* * Consecutive little-endian elements from a single register * can be promoted to a larger little-endian operation. */ if (interleave == 1 && endian == MO_LE) { + /* Retain any natural alignment. */ + if (align == MO_ALIGN) { + align = pow2_align(size); + } size = 3; } + tmp64 = tcg_temp_new_i64(); addr = tcg_temp_new_i32(); tmp = tcg_const_i32(1 << size); load_reg_var(s, addr, a->rn); + + mop = endian | size | align; for (reg = 0; reg < nregs; reg++) { for (n = 0; n < 8 >> size; n++) { int xs; @@ -494,13 +510,16 @@ static bool trans_VLDST_multiple(DisasContext *s, arg_VLDST_multiple *a) int tt = a->vd + reg + spacing * xs; if (a->l) { - gen_aa32_ld_i64(s, tmp64, addr, mmu_idx, endian | size); + gen_aa32_ld_internal_i64(s, tmp64, addr, mmu_idx, mop); neon_store_element64(tt, n, size, tmp64); } else { neon_load_element64(tmp64, tt, n, size); - gen_aa32_st_i64(s, tmp64, addr, mmu_idx, endian | size); + gen_aa32_st_internal_i64(s, tmp64, addr, mmu_idx, mop); } tcg_gen_add_i32(addr, addr, tmp); + + /* Subsequent memory operations inherit alignment */ + mop &= ~MO_AMASK; } } } @@ -520,6 +539,7 @@ static bool trans_VLD_all_lanes(DisasContext *s, arg_VLD_all_lanes *a) int size = a->size; int nregs = a->n + 1; TCGv_i32 addr, tmp; + MemOp mop, align; if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { return false; @@ -530,18 +550,33 @@ static bool trans_VLD_all_lanes(DisasContext *s, arg_VLD_all_lanes *a) return false; } + align = 0; if (size == 3) { if (nregs != 4 || a->a == 0) { return false; } /* For VLD4 size == 3 a == 1 means 32 bits at 16 byte alignment */ - size = 2; - } - if (nregs == 1 && a->a == 1 && size == 0) { - return false; - } - if (nregs == 3 && a->a == 1) { - return false; + size = MO_32; + align = MO_ALIGN_16; + } else if (a->a) { + switch (nregs) { + case 1: + if (size == 0) { + return false; + } + align = MO_ALIGN; + break; + case 2: + align = pow2_align(size + 1); + break; + case 3: + return false; + case 4: + align = pow2_align(size + 2); + break; + default: + g_assert_not_reached(); + } } if (!vfp_access_check(s)) { @@ -554,13 +589,12 @@ static bool trans_VLD_all_lanes(DisasContext *s, arg_VLD_all_lanes *a) */ stride = a->t ? 2 : 1; vec_size = nregs == 1 ? stride * 8 : 8; - + mop = size | align; tmp = tcg_temp_new_i32(); addr = tcg_temp_new_i32(); load_reg_var(s, addr, a->rn); for (reg = 0; reg < nregs; reg++) { - gen_aa32_ld_i32(s, tmp, addr, get_mem_index(s), - s->be_data | size); + gen_aa32_ld_i32(s, tmp, addr, get_mem_index(s), mop); if ((vd & 1) && vec_size == 16) { /* * We cannot write 16 bytes at once because the @@ -576,6 +610,9 @@ static bool trans_VLD_all_lanes(DisasContext *s, arg_VLD_all_lanes *a) } tcg_gen_addi_i32(addr, addr, 1 << size); vd += stride; + + /* Subsequent memory operations inherit alignment */ + mop &= ~MO_AMASK; } tcg_temp_free_i32(tmp); tcg_temp_free_i32(addr); @@ -592,6 +629,7 @@ static bool trans_VLDST_single(DisasContext *s, arg_VLDST_single *a) int nregs = a->n + 1; int vd = a->vd; TCGv_i32 addr, tmp; + MemOp mop; if (!arm_dc_feature(s, ARM_FEATURE_NEON)) { return false; @@ -606,7 +644,7 @@ static bool trans_VLDST_single(DisasContext *s, arg_VLDST_single *a) switch (nregs) { case 1: if (((a->align & (1 << a->size)) != 0) || - (a->size == 2 && ((a->align & 3) == 1 || (a->align & 3) == 2))) { + (a->size == 2 && (a->align == 1 || a->align == 2))) { return false; } break; @@ -621,7 +659,7 @@ static bool trans_VLDST_single(DisasContext *s, arg_VLDST_single *a) } break; case 4: - if ((a->size == 2) && ((a->align & 3) == 3)) { + if (a->size == 2 && a->align == 3) { return false; } break; @@ -641,25 +679,58 @@ static bool trans_VLDST_single(DisasContext *s, arg_VLDST_single *a) return true; } + /* Pick up SCTLR settings */ + mop = finalize_memop(s, a->size); + + if (a->align) { + MemOp align_op; + + switch (nregs) { + case 1: + /* For VLD1, use natural alignment. */ + align_op = MO_ALIGN; + break; + case 2: + /* For VLD2, use double alignment. */ + align_op = pow2_align(a->size + 1); + break; + case 4: + if (a->size == MO_32) { + /* + * For VLD4.32, align = 1 is double alignment, align = 2 is + * quad alignment; align = 3 is rejected above. + */ + align_op = pow2_align(a->size + a->align); + } else { + /* For VLD4.8 and VLD.16, we want quad alignment. */ + align_op = pow2_align(a->size + 2); + } + break; + default: + /* For VLD3, the alignment field is zero and rejected above. */ + g_assert_not_reached(); + } + + mop = (mop & ~MO_AMASK) | align_op; + } + tmp = tcg_temp_new_i32(); addr = tcg_temp_new_i32(); load_reg_var(s, addr, a->rn); - /* - * TODO: if we implemented alignment exceptions, we should check - * addr against the alignment encoded in a->align here. - */ + for (reg = 0; reg < nregs; reg++) { if (a->l) { - gen_aa32_ld_i32(s, tmp, addr, get_mem_index(s), - s->be_data | a->size); + gen_aa32_ld_internal_i32(s, tmp, addr, get_mem_index(s), mop); neon_store_element(vd, a->reg_idx, a->size, tmp); } else { /* Store */ neon_load_element(tmp, vd, a->reg_idx, a->size); - gen_aa32_st_i32(s, tmp, addr, get_mem_index(s), - s->be_data | a->size); + gen_aa32_st_internal_i32(s, tmp, addr, get_mem_index(s), mop); } vd += a->stride; tcg_gen_addi_i32(addr, addr, 1 << a->size); + + /* Subsequent memory operations inherit alignment */ + mop &= ~MO_AMASK; } tcg_temp_free_i32(addr); tcg_temp_free_i32(tmp); diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index b7c70b10d61..fa0d0de43d4 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -4266,7 +4266,7 @@ static void do_ldr(DisasContext *s, uint32_t vofs, int len, int rn, int imm) dirty_addr = tcg_temp_new_i64(); tcg_gen_addi_i64(dirty_addr, cpu_reg_sp(s, rn), imm); clean_addr = gen_mte_and_cheri_checkN(s, dirty_addr, true, false, rn != 31, - len, MO_8, rn, false, true); + len, rn, false, true); tcg_temp_free_i64(dirty_addr); /* @@ -4358,7 +4358,7 @@ static void do_str(DisasContext *s, uint32_t vofs, int len, int rn, int imm) dirty_addr = tcg_temp_new_i64(); tcg_gen_addi_i64(dirty_addr, cpu_reg_sp(s, rn), imm); clean_addr = gen_mte_and_cheri_checkN(s, dirty_addr, false, true, rn != 31, - len, MO_8, rn, false, true); + len, rn, false, true); tcg_temp_free_i64(dirty_addr); /* Note that unpredicated load/store of vector/predicate registers @@ -4517,8 +4517,7 @@ static void do_mem_zpa(DisasContext *s, int zt, int pg, TCGv_i64 addr, desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid); desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma); desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write); - desc = FIELD_DP32(desc, MTEDESC, ESIZE, 1 << msz); - desc = FIELD_DP32(desc, MTEDESC, TSIZE, mte_n << msz); + desc = FIELD_DP32(desc, MTEDESC, SIZEM1, (mte_n << msz) - 1); desc <<= SVE_MTEDESC_SHIFT; } else { addr = clean_data_tbi(s, addr); @@ -5012,7 +5011,7 @@ static bool trans_LD1R_zpri(DisasContext *s, arg_rpri_load *a) a->rn, false, true); tcg_gen_qemu_ld_i64_with_checked_addr(temp, clean_addr, get_mem_index(s), - s->be_data | dtype_mop[a->dtype]); + finalize_memop(s, dtype_mop[a->dtype])); /* Broadcast to *all* elements. */ tcg_gen_gvec_dup_i64(esz, vec_full_reg_offset(s, a->rd), @@ -5199,7 +5198,7 @@ static void do_mem_zpz(DisasContext *s, int zt, int pg, int zm, desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid); desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma); desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write); - desc = FIELD_DP32(desc, MTEDESC, ESIZE, 1 << msz); + desc = FIELD_DP32(desc, MTEDESC, SIZEM1, (1 << msz) - 1); desc <<= SVE_MTEDESC_SHIFT; } desc = simd_desc(vsz, vsz, desc | scale); diff --git a/target/arm/translate-vfp.c.inc b/target/arm/translate-vfp.c.inc index 10766f210c1..e20d9c7ba66 100644 --- a/target/arm/translate-vfp.c.inc +++ b/target/arm/translate-vfp.c.inc @@ -1364,11 +1364,11 @@ static bool trans_VLDR_VSTR_hp(DisasContext *s, arg_VLDR_VSTR_sp *a) addr = add_reg_for_lit(s, a->rn, offset); tmp = tcg_temp_new_i32(); if (a->l) { - gen_aa32_ld16u(s, tmp, addr, get_mem_index(s)); + gen_aa32_ld_i32(s, tmp, addr, get_mem_index(s), MO_UW | MO_ALIGN); vfp_store_reg32(tmp, a->vd); } else { vfp_load_reg32(tmp, a->vd); - gen_aa32_st16(s, tmp, addr, get_mem_index(s)); + gen_aa32_st_i32(s, tmp, addr, get_mem_index(s), MO_UW | MO_ALIGN); } tcg_temp_free_i32(tmp); tcg_temp_free_i32(addr); @@ -1398,11 +1398,11 @@ static bool trans_VLDR_VSTR_sp(DisasContext *s, arg_VLDR_VSTR_sp *a) addr = add_reg_for_lit(s, a->rn, offset); tmp = tcg_temp_new_i32(); if (a->l) { - gen_aa32_ld32u(s, tmp, addr, get_mem_index(s)); + gen_aa32_ld_i32(s, tmp, addr, get_mem_index(s), MO_UL | MO_ALIGN); vfp_store_reg32(tmp, a->vd); } else { vfp_load_reg32(tmp, a->vd); - gen_aa32_st32(s, tmp, addr, get_mem_index(s)); + gen_aa32_st_i32(s, tmp, addr, get_mem_index(s), MO_UL | MO_ALIGN); } tcg_temp_free_i32(tmp); tcg_temp_free_i32(addr); @@ -1439,11 +1439,11 @@ static bool trans_VLDR_VSTR_dp(DisasContext *s, arg_VLDR_VSTR_dp *a) addr = add_reg_for_lit(s, a->rn, offset); tmp = tcg_temp_new_i64(); if (a->l) { - gen_aa32_ld64(s, tmp, addr, get_mem_index(s)); + gen_aa32_ld_i64(s, tmp, addr, get_mem_index(s), MO_Q | MO_ALIGN_4); vfp_store_reg64(tmp, a->vd); } else { vfp_load_reg64(tmp, a->vd); - gen_aa32_st64(s, tmp, addr, get_mem_index(s)); + gen_aa32_st_i64(s, tmp, addr, get_mem_index(s), MO_Q | MO_ALIGN_4); } tcg_temp_free_i64(tmp); tcg_temp_free_i32(addr); @@ -1503,12 +1503,12 @@ static bool trans_VLDM_VSTM_sp(DisasContext *s, arg_VLDM_VSTM_sp *a) for (i = 0; i < n; i++) { if (a->l) { /* load */ - gen_aa32_ld32u(s, tmp, addr, get_mem_index(s)); + gen_aa32_ld_i32(s, tmp, addr, get_mem_index(s), MO_UL | MO_ALIGN); vfp_store_reg32(tmp, a->vd + i); } else { /* store */ vfp_load_reg32(tmp, a->vd + i); - gen_aa32_st32(s, tmp, addr, get_mem_index(s)); + gen_aa32_st_i32(s, tmp, addr, get_mem_index(s), MO_UL | MO_ALIGN); } tcg_gen_addi_i32(addr, addr, offset); } @@ -1586,12 +1586,12 @@ static bool trans_VLDM_VSTM_dp(DisasContext *s, arg_VLDM_VSTM_dp *a) for (i = 0; i < n; i++) { if (a->l) { /* load */ - gen_aa32_ld64(s, tmp, addr, get_mem_index(s)); + gen_aa32_ld_i64(s, tmp, addr, get_mem_index(s), MO_Q | MO_ALIGN_4); vfp_store_reg64(tmp, a->vd + i); } else { /* store */ vfp_load_reg64(tmp, a->vd + i); - gen_aa32_st64(s, tmp, addr, get_mem_index(s)); + gen_aa32_st_i64(s, tmp, addr, get_mem_index(s), MO_Q | MO_ALIGN_4); } tcg_gen_addi_i32(addr, addr, offset); } diff --git a/target/arm/translate.c b/target/arm/translate.c index 728a77473c6..d4d1b61cda9 100644 --- a/target/arm/translate.c +++ b/target/arm/translate.c @@ -949,7 +949,23 @@ static inline void store_reg_from_load(DisasContext *s, int reg, TCGv_i32 var) #define IS_USER_ONLY 0 #endif -/* Abstractions of "generate code to do a guest load/store for +MemOp pow2_align(unsigned i) +{ + static const MemOp mop_align[] = { + 0, MO_ALIGN_2, MO_ALIGN_4, MO_ALIGN_8, MO_ALIGN_16, + /* + * FIXME: TARGET_PAGE_BITS_MIN affects TLB_FLAGS_MASK such + * that 256-bit alignment (MO_ALIGN_32) cannot be supported: + * see get_alignment_bits(). Enforce only 128-bit alignment for now. + */ + MO_ALIGN_16 + }; + g_assert(i < ARRAY_SIZE(mop_align)); + return mop_align[i]; +} + +/* + * Abstractions of "generate code to do a guest load/store for * AArch32", where a vaddr is always 32 bits (and is zero * extended if we're a 64 bit core) and data is also * 32 bits unless specifically doing a 64 bit access. @@ -957,7 +973,7 @@ static inline void store_reg_from_load(DisasContext *s, int reg, TCGv_i32 var) * that the address argument is TCGv_i32 rather than TCGv. */ -static inline TCGv gen_aa32_addr(DisasContext *s, TCGv_i32 a32, MemOp op) +static TCGv gen_aa32_addr(DisasContext *s, TCGv_i32 a32, MemOp op) { TCGv addr = tcg_temp_new(); tcg_gen_extu_i32_tl(addr, a32); @@ -969,80 +985,47 @@ static inline TCGv gen_aa32_addr(DisasContext *s, TCGv_i32 a32, MemOp op) return addr; } -static void gen_aa32_ld_i32(DisasContext *s, TCGv_i32 val, TCGv_i32 a32, - int index, MemOp opc) +/* + * Internal routines are used for NEON cases where the endianness + * and/or alignment has already been taken into account and manipulated. + */ +static void gen_aa32_ld_internal_i32(DisasContext *s, TCGv_i32 val, + TCGv_i32 a32, int index, MemOp opc) { - TCGv addr; - - if (arm_dc_feature(s, ARM_FEATURE_M) && - !arm_dc_feature(s, ARM_FEATURE_M_MAIN)) { - opc |= MO_ALIGN; - } - - addr = gen_aa32_addr(s, a32, opc); + TCGv addr = gen_aa32_addr(s, a32, opc); tcg_gen_qemu_ld_i32(val, addr, index, opc); tcg_temp_free(addr); } -static void gen_aa32_st_i32(DisasContext *s, TCGv_i32 val, TCGv_i32 a32, - int index, MemOp opc) +static void gen_aa32_st_internal_i32(DisasContext *s, TCGv_i32 val, + TCGv_i32 a32, int index, MemOp opc) { - TCGv addr; - - if (arm_dc_feature(s, ARM_FEATURE_M) && - !arm_dc_feature(s, ARM_FEATURE_M_MAIN)) { - opc |= MO_ALIGN; - } - - addr = gen_aa32_addr(s, a32, opc); + TCGv addr = gen_aa32_addr(s, a32, opc); tcg_gen_qemu_st_i32(val, addr, index, opc); tcg_temp_free(addr); } -#define DO_GEN_LD(SUFF, OPC) \ -static inline void gen_aa32_ld##SUFF(DisasContext *s, TCGv_i32 val, \ - TCGv_i32 a32, int index) \ -{ \ - gen_aa32_ld_i32(s, val, a32, index, OPC | s->be_data); \ -} +static void gen_aa32_ld_internal_i64(DisasContext *s, TCGv_i64 val, + TCGv_i32 a32, int index, MemOp opc) +{ + TCGv addr = gen_aa32_addr(s, a32, opc); -#define DO_GEN_ST(SUFF, OPC) \ -static inline void gen_aa32_st##SUFF(DisasContext *s, TCGv_i32 val, \ - TCGv_i32 a32, int index) \ -{ \ - gen_aa32_st_i32(s, val, a32, index, OPC | s->be_data); \ -} + tcg_gen_qemu_ld_i64(val, addr, index, opc); -static inline void gen_aa32_frob64(DisasContext *s, TCGv_i64 val) -{ /* Not needed for user-mode BE32, where we use MO_BE instead. */ - if (!IS_USER_ONLY && s->sctlr_b) { + if (!IS_USER_ONLY && s->sctlr_b && (opc & MO_SIZE) == MO_64) { tcg_gen_rotri_i64(val, val, 32); } -} - -static void gen_aa32_ld_i64(DisasContext *s, TCGv_i64 val, TCGv_i32 a32, - int index, MemOp opc) -{ - TCGv addr = gen_aa32_addr(s, a32, opc); - tcg_gen_qemu_ld_i64(val, addr, index, opc); - gen_aa32_frob64(s, val); tcg_temp_free(addr); } -static inline void gen_aa32_ld64(DisasContext *s, TCGv_i64 val, - TCGv_i32 a32, int index) -{ - gen_aa32_ld_i64(s, val, a32, index, MO_Q | s->be_data); -} - -static void gen_aa32_st_i64(DisasContext *s, TCGv_i64 val, TCGv_i32 a32, - int index, MemOp opc) +static void gen_aa32_st_internal_i64(DisasContext *s, TCGv_i64 val, + TCGv_i32 a32, int index, MemOp opc) { TCGv addr = gen_aa32_addr(s, a32, opc); /* Not needed for user-mode BE32, where we use MO_BE instead. */ - if (!IS_USER_ONLY && s->sctlr_b) { + if (!IS_USER_ONLY && s->sctlr_b && (opc & MO_SIZE) == MO_64) { TCGv_i64 tmp = tcg_temp_new_i64(); tcg_gen_rotri_i64(tmp, val, 32); tcg_gen_qemu_st_i64(tmp, addr, index, opc); @@ -1053,10 +1036,54 @@ static void gen_aa32_st_i64(DisasContext *s, TCGv_i64 val, TCGv_i32 a32, tcg_temp_free(addr); } +static void gen_aa32_ld_i32(DisasContext *s, TCGv_i32 val, TCGv_i32 a32, + int index, MemOp opc) +{ + gen_aa32_ld_internal_i32(s, val, a32, index, finalize_memop(s, opc)); +} + +static void gen_aa32_st_i32(DisasContext *s, TCGv_i32 val, TCGv_i32 a32, + int index, MemOp opc) +{ + gen_aa32_st_internal_i32(s, val, a32, index, finalize_memop(s, opc)); +} + +static void gen_aa32_ld_i64(DisasContext *s, TCGv_i64 val, TCGv_i32 a32, + int index, MemOp opc) +{ + gen_aa32_ld_internal_i64(s, val, a32, index, finalize_memop(s, opc)); +} + +static void gen_aa32_st_i64(DisasContext *s, TCGv_i64 val, TCGv_i32 a32, + int index, MemOp opc) +{ + gen_aa32_st_internal_i64(s, val, a32, index, finalize_memop(s, opc)); +} + +#define DO_GEN_LD(SUFF, OPC) \ + static inline void gen_aa32_ld##SUFF(DisasContext *s, TCGv_i32 val, \ + TCGv_i32 a32, int index) \ + { \ + gen_aa32_ld_i32(s, val, a32, index, OPC); \ + } + +#define DO_GEN_ST(SUFF, OPC) \ + static inline void gen_aa32_st##SUFF(DisasContext *s, TCGv_i32 val, \ + TCGv_i32 a32, int index) \ + { \ + gen_aa32_st_i32(s, val, a32, index, OPC); \ + } + +static inline void gen_aa32_ld64(DisasContext *s, TCGv_i64 val, + TCGv_i32 a32, int index) +{ + gen_aa32_ld_i64(s, val, a32, index, MO_Q); +} + static inline void gen_aa32_st64(DisasContext *s, TCGv_i64 val, TCGv_i32 a32, int index) { - gen_aa32_st_i64(s, val, a32, index, MO_Q | s->be_data); + gen_aa32_st_i64(s, val, a32, index, MO_Q); } DO_GEN_LD(8u, MO_UB) @@ -5025,16 +5052,13 @@ static void gen_load_exclusive(DisasContext *s, int rt, int rt2, TCGv_i32 tmp2 = tcg_temp_new_i32(); TCGv_i64 t64 = tcg_temp_new_i64(); - /* For AArch32, architecturally the 32-bit word at the lowest + /* + * For AArch32, architecturally the 32-bit word at the lowest * address is always Rt and the one at addr+4 is Rt2, even if * the CPU is big-endian. That means we don't want to do a - * gen_aa32_ld_i64(), which invokes gen_aa32_frob64() as if - * for an architecturally 64-bit access, but instead do a - * 64-bit access using MO_BE if appropriate and then split - * the two halves. - * This only makes a difference for BE32 user-mode, where - * frob64() must not flip the two halves of the 64-bit data - * but this code must treat BE32 user-mode like BE32 system. + * gen_aa32_ld_i64(), which checks SCTLR_B as if for an + * architecturally 64-bit access, but instead do a 64-bit access + * using MO_BE if appropriate and then split the two halves. */ TCGv taddr = gen_aa32_addr(s, addr, opc); @@ -5094,14 +5118,15 @@ static void gen_store_exclusive(DisasContext *s, int rd, int rt, int rt2, TCGv_i64 n64 = tcg_temp_new_i64(); t2 = load_reg(s, rt2); - /* For AArch32, architecturally the 32-bit word at the lowest + + /* + * For AArch32, architecturally the 32-bit word at the lowest * address is always Rt and the one at addr+4 is Rt2, even if * the CPU is big-endian. Since we're going to treat this as a * single 64-bit BE store, we need to put the two halves in the * opposite order for BE to LE, so that they end up in the right - * places. - * We don't want gen_aa32_frob64() because that does the wrong - * thing for BE32 usermode. + * places. We don't want gen_aa32_st_i64, because that checks + * SCTLR_B as if for an architectural 64-bit access. */ if (s->be_data == MO_BE) { tcg_gen_concat_i32_i64(n64, t2, t1); @@ -5237,11 +5262,11 @@ static void gen_srs(DisasContext *s, } tcg_gen_addi_i32(addr, addr, offset); tmp = load_reg(s, 14); - gen_aa32_st32(s, tmp, addr, get_mem_index(s)); + gen_aa32_st_i32(s, tmp, addr, get_mem_index(s), MO_UL | MO_ALIGN); tcg_temp_free_i32(tmp); tmp = load_cpu_field(spsr); tcg_gen_addi_i32(addr, addr, 4); - gen_aa32_st32(s, tmp, addr, get_mem_index(s)); + gen_aa32_st_i32(s, tmp, addr, get_mem_index(s), MO_UL | MO_ALIGN); tcg_temp_free_i32(tmp); if (writeback) { switch (amode) { @@ -6505,7 +6530,7 @@ static bool op_load_rr(DisasContext *s, arg_ldst_rr *a, addr = op_addr_rr_pre(s, a); tmp = tcg_temp_new_i32(); - gen_aa32_ld_i32(s, tmp, addr, mem_idx, mop | s->be_data); + gen_aa32_ld_i32(s, tmp, addr, mem_idx, mop); disas_set_da_iss(s, mop, issinfo); /* @@ -6523,10 +6548,18 @@ static bool op_store_rr(DisasContext *s, arg_ldst_rr *a, ISSInfo issinfo = make_issinfo(s, a->rt, a->p, a->w) | ISSIsWrite; TCGv_i32 addr, tmp; + /* + * In Thumb encodings of stores Rn=1111 is UNDEF; for Arm it + * is either UNPREDICTABLE or has defined behaviour + */ + if (s->thumb && a->rn == 15) { + return false; + } + addr = op_addr_rr_pre(s, a); tmp = load_reg(s, a->rt); - gen_aa32_st_i32(s, tmp, addr, mem_idx, mop | s->be_data); + gen_aa32_st_i32(s, tmp, addr, mem_idx, mop); disas_set_da_iss(s, mop, issinfo); tcg_temp_free_i32(tmp); @@ -6549,13 +6582,13 @@ static bool trans_LDRD_rr(DisasContext *s, arg_ldst_rr *a) addr = op_addr_rr_pre(s, a); tmp = tcg_temp_new_i32(); - gen_aa32_ld_i32(s, tmp, addr, mem_idx, MO_UL | s->be_data); + gen_aa32_ld_i32(s, tmp, addr, mem_idx, MO_UL | MO_ALIGN); store_reg(s, a->rt, tmp); tcg_gen_addi_i32(addr, addr, 4); tmp = tcg_temp_new_i32(); - gen_aa32_ld_i32(s, tmp, addr, mem_idx, MO_UL | s->be_data); + gen_aa32_ld_i32(s, tmp, addr, mem_idx, MO_UL | MO_ALIGN); store_reg(s, a->rt + 1, tmp); /* LDRD w/ base writeback is undefined if the registers overlap. */ @@ -6578,13 +6611,13 @@ static bool trans_STRD_rr(DisasContext *s, arg_ldst_rr *a) addr = op_addr_rr_pre(s, a); tmp = load_reg(s, a->rt); - gen_aa32_st_i32(s, tmp, addr, mem_idx, MO_UL | s->be_data); + gen_aa32_st_i32(s, tmp, addr, mem_idx, MO_UL | MO_ALIGN); tcg_temp_free_i32(tmp); tcg_gen_addi_i32(addr, addr, 4); tmp = load_reg(s, a->rt + 1); - gen_aa32_st_i32(s, tmp, addr, mem_idx, MO_UL | s->be_data); + gen_aa32_st_i32(s, tmp, addr, mem_idx, MO_UL | MO_ALIGN); tcg_temp_free_i32(tmp); op_addr_rr_post(s, a, addr, -4); @@ -6649,7 +6682,7 @@ static bool op_load_ri(DisasContext *s, arg_ldst_ri *a, addr = op_addr_ri_pre(s, a); tmp = tcg_temp_new_i32(); - gen_aa32_ld_i32(s, tmp, addr, mem_idx, mop | s->be_data); + gen_aa32_ld_i32(s, tmp, addr, mem_idx, mop); disas_set_da_iss(s, mop, issinfo); /* @@ -6667,10 +6700,18 @@ static bool op_store_ri(DisasContext *s, arg_ldst_ri *a, ISSInfo issinfo = make_issinfo(s, a->rt, a->p, a->w) | ISSIsWrite; TCGv_i32 addr, tmp; + /* + * In Thumb encodings of stores Rn=1111 is UNDEF; for Arm it + * is either UNPREDICTABLE or has defined behaviour + */ + if (s->thumb && a->rn == 15) { + return false; + } + addr = op_addr_ri_pre(s, a); tmp = load_reg(s, a->rt); - gen_aa32_st_i32(s, tmp, addr, mem_idx, mop | s->be_data); + gen_aa32_st_i32(s, tmp, addr, mem_idx, mop); disas_set_da_iss(s, mop, issinfo); tcg_temp_free_i32(tmp); @@ -6686,13 +6727,13 @@ static bool op_ldrd_ri(DisasContext *s, arg_ldst_ri *a, int rt2) addr = op_addr_ri_pre(s, a); tmp = tcg_temp_new_i32(); - gen_aa32_ld_i32(s, tmp, addr, mem_idx, MO_UL | s->be_data); + gen_aa32_ld_i32(s, tmp, addr, mem_idx, MO_UL | MO_ALIGN); store_reg(s, a->rt, tmp); tcg_gen_addi_i32(addr, addr, 4); tmp = tcg_temp_new_i32(); - gen_aa32_ld_i32(s, tmp, addr, mem_idx, MO_UL | s->be_data); + gen_aa32_ld_i32(s, tmp, addr, mem_idx, MO_UL | MO_ALIGN); store_reg(s, rt2, tmp); /* LDRD w/ base writeback is undefined if the registers overlap. */ @@ -6725,13 +6766,13 @@ static bool op_strd_ri(DisasContext *s, arg_ldst_ri *a, int rt2) addr = op_addr_ri_pre(s, a); tmp = load_reg(s, a->rt); - gen_aa32_st_i32(s, tmp, addr, mem_idx, MO_UL | s->be_data); + gen_aa32_st_i32(s, tmp, addr, mem_idx, MO_UL | MO_ALIGN); tcg_temp_free_i32(tmp); tcg_gen_addi_i32(addr, addr, 4); tmp = load_reg(s, rt2); - gen_aa32_st_i32(s, tmp, addr, mem_idx, MO_UL | s->be_data); + gen_aa32_st_i32(s, tmp, addr, mem_idx, MO_UL | MO_ALIGN); tcg_temp_free_i32(tmp); op_addr_ri_post(s, a, addr, -4); @@ -6960,7 +7001,7 @@ static bool op_stl(DisasContext *s, arg_STL *a, MemOp mop) addr = load_reg(s, a->rn); tmp = load_reg(s, a->rt); tcg_gen_mb(TCG_MO_ALL | TCG_BAR_STRL); - gen_aa32_st_i32(s, tmp, addr, get_mem_index(s), mop | s->be_data); + gen_aa32_st_i32(s, tmp, addr, get_mem_index(s), mop | MO_ALIGN); disas_set_da_iss(s, mop, a->rt | ISSIsAcqRel | ISSIsWrite); tcg_temp_free_i32(tmp); @@ -7116,7 +7157,7 @@ static bool op_lda(DisasContext *s, arg_LDA *a, MemOp mop) addr = load_reg(s, a->rn); tmp = tcg_temp_new_i32(); - gen_aa32_ld_i32(s, tmp, addr, get_mem_index(s), mop | s->be_data); + gen_aa32_ld_i32(s, tmp, addr, get_mem_index(s), mop | MO_ALIGN); disas_set_da_iss(s, mop, a->rt | ISSIsAcqRel); tcg_temp_free_i32(addr); @@ -7908,7 +7949,7 @@ static bool op_stm(DisasContext *s, arg_ldst_block *a, int min_n) } else { tmp = load_reg(s, i); } - gen_aa32_st32(s, tmp, addr, mem_idx); + gen_aa32_st_i32(s, tmp, addr, mem_idx, MO_UL | MO_ALIGN); tcg_temp_free_i32(tmp); /* No need to add after the last transfer. */ @@ -7983,7 +8024,7 @@ static bool do_ldm(DisasContext *s, arg_ldst_block *a, int min_n) } tmp = tcg_temp_new_i32(); - gen_aa32_ld32u(s, tmp, addr, mem_idx); + gen_aa32_ld_i32(s, tmp, addr, mem_idx, MO_UL | MO_ALIGN); if (user) { tmp2 = tcg_const_i32(i); gen_helper_set_user_reg(cpu_env, tmp2, tmp); @@ -8300,8 +8341,7 @@ static bool op_tbranch(DisasContext *s, arg_tbranch *a, bool half) addr = load_reg(s, a->rn); tcg_gen_add_i32(addr, addr, tmp); - gen_aa32_ld_i32(s, tmp, addr, get_mem_index(s), - half ? MO_UW | s->be_data : MO_UB); + gen_aa32_ld_i32(s, tmp, addr, get_mem_index(s), half ? MO_UW : MO_UB); tcg_temp_free_i32(addr); tcg_gen_add_i32(tmp, tmp, tmp); @@ -8382,10 +8422,10 @@ static bool trans_RFE(DisasContext *s, arg_RFE *a) /* Load PC into tmp and CPSR into tmp2. */ t1 = tcg_temp_new_i32(); - gen_aa32_ld32u(s, t1, addr, get_mem_index(s)); + gen_aa32_ld_i32(s, t1, addr, get_mem_index(s), MO_UL | MO_ALIGN); tcg_gen_addi_i32(addr, addr, 4); t2 = tcg_temp_new_i32(); - gen_aa32_ld32u(s, t2, addr, get_mem_index(s)); + gen_aa32_ld_i32(s, t2, addr, get_mem_index(s), MO_UL | MO_ALIGN); if (a->w) { /* Base writeback. */ @@ -8886,7 +8926,7 @@ static void arm_tr_init_disas_context(DisasContextBase *dcbase, CPUState *cs) DisasContext *dc = container_of(dcbase, DisasContext, base); CPUARMState *env = cs->env_ptr; ARMCPU *cpu = env_archcpu(env); - uint32_t tb_flags = dc->base.tb->flags; + CPUARMTBFlags tb_flags = arm_tbflags_from_tb(dc->base.tb); uint32_t condexec, core_mmu_idx; dc->isar = &cpu->isar; @@ -8898,46 +8938,43 @@ static void arm_tr_init_disas_context(DisasContextBase *dcbase, CPUState *cs) */ dc->secure_routed_to_el3 = arm_feature(env, ARM_FEATURE_EL3) && !arm_el_is_aa64(env, 3); - dc->thumb = FIELD_EX32(tb_flags, TBFLAG_AM32, THUMB); - dc->be_data = FIELD_EX32(tb_flags, TBFLAG_ANY, BE_DATA) ? MO_BE : MO_LE; - condexec = FIELD_EX32(tb_flags, TBFLAG_AM32, CONDEXEC); + dc->thumb = EX_TBFLAG_AM32(tb_flags, THUMB); + dc->be_data = EX_TBFLAG_ANY(tb_flags, BE_DATA) ? MO_BE : MO_LE; + condexec = EX_TBFLAG_AM32(tb_flags, CONDEXEC); dc->condexec_mask = (condexec & 0xf) << 1; dc->condexec_cond = condexec >> 4; - core_mmu_idx = FIELD_EX32(tb_flags, TBFLAG_ANY, MMUIDX); + core_mmu_idx = EX_TBFLAG_ANY(tb_flags, MMUIDX); dc->mmu_idx = core_to_arm_mmu_idx(env, core_mmu_idx); dc->current_el = arm_mmu_idx_to_el(dc->mmu_idx); #if !defined(CONFIG_USER_ONLY) dc->user = (dc->current_el == 0); #endif - dc->fp_excp_el = FIELD_EX32(tb_flags, TBFLAG_ANY, FPEXC_EL); + dc->fp_excp_el = EX_TBFLAG_ANY(tb_flags, FPEXC_EL); + dc->align_mem = EX_TBFLAG_ANY(tb_flags, ALIGN_MEM); if (arm_feature(env, ARM_FEATURE_M)) { dc->vfp_enabled = 1; dc->be_data = MO_TE; - dc->v7m_handler_mode = FIELD_EX32(tb_flags, TBFLAG_M32, HANDLER); + dc->v7m_handler_mode = EX_TBFLAG_M32(tb_flags, HANDLER); dc->v8m_secure = arm_feature(env, ARM_FEATURE_M_SECURITY) && regime_is_secure(env, dc->mmu_idx); - dc->v8m_stackcheck = FIELD_EX32(tb_flags, TBFLAG_M32, STACKCHECK); - dc->v8m_fpccr_s_wrong = - FIELD_EX32(tb_flags, TBFLAG_M32, FPCCR_S_WRONG); + dc->v8m_stackcheck = EX_TBFLAG_M32(tb_flags, STACKCHECK); + dc->v8m_fpccr_s_wrong = EX_TBFLAG_M32(tb_flags, FPCCR_S_WRONG); dc->v7m_new_fp_ctxt_needed = - FIELD_EX32(tb_flags, TBFLAG_M32, NEW_FP_CTXT_NEEDED); - dc->v7m_lspact = FIELD_EX32(tb_flags, TBFLAG_M32, LSPACT); + EX_TBFLAG_M32(tb_flags, NEW_FP_CTXT_NEEDED); + dc->v7m_lspact = EX_TBFLAG_M32(tb_flags, LSPACT); } else { - dc->be_data = - FIELD_EX32(tb_flags, TBFLAG_ANY, BE_DATA) ? MO_BE : MO_LE; - dc->debug_target_el = - FIELD_EX32(tb_flags, TBFLAG_ANY, DEBUG_TARGET_EL); - dc->sctlr_b = FIELD_EX32(tb_flags, TBFLAG_A32, SCTLR_B); - dc->hstr_active = FIELD_EX32(tb_flags, TBFLAG_A32, HSTR_ACTIVE); - dc->ns = FIELD_EX32(tb_flags, TBFLAG_A32, NS); - dc->vfp_enabled = FIELD_EX32(tb_flags, TBFLAG_A32, VFPEN); + dc->debug_target_el = EX_TBFLAG_ANY(tb_flags, DEBUG_TARGET_EL); + dc->sctlr_b = EX_TBFLAG_A32(tb_flags, SCTLR__B); + dc->hstr_active = EX_TBFLAG_A32(tb_flags, HSTR_ACTIVE); + dc->ns = EX_TBFLAG_A32(tb_flags, NS); + dc->vfp_enabled = EX_TBFLAG_A32(tb_flags, VFPEN); if (arm_feature(env, ARM_FEATURE_XSCALE)) { - dc->c15_cpar = FIELD_EX32(tb_flags, TBFLAG_A32, XSCALE_CPAR); + dc->c15_cpar = EX_TBFLAG_A32(tb_flags, XSCALE_CPAR); } else { - dc->vec_len = FIELD_EX32(tb_flags, TBFLAG_A32, VECLEN); - dc->vec_stride = FIELD_EX32(tb_flags, TBFLAG_A32, VECSTRIDE); + dc->vec_len = EX_TBFLAG_A32(tb_flags, VECLEN); + dc->vec_stride = EX_TBFLAG_A32(tb_flags, VECSTRIDE); } } dc->cp_regs = cpu->cp_regs; @@ -8958,8 +8995,8 @@ static void arm_tr_init_disas_context(DisasContextBase *dcbase, CPUState *cs) * emit code to generate a software step exception * end the TB */ - dc->ss_active = FIELD_EX32(tb_flags, TBFLAG_ANY, SS_ACTIVE); - dc->pstate_ss = FIELD_EX32(tb_flags, TBFLAG_ANY, PSTATE_SS); + dc->ss_active = EX_TBFLAG_ANY(tb_flags, SS_ACTIVE); + dc->pstate_ss = EX_TBFLAG_ANY(tb_flags, PSTATE__SS); dc->is_ldex = false; dc->page_start = dc->base.pc_first & TARGET_PAGE_MASK; @@ -9415,12 +9452,13 @@ void gen_intermediate_code(CPUState *cpu, TranslationBlock *tb, int max_insns) { DisasContext dc = { }; const TranslatorOps *ops = &arm_translator_ops; + CPUARMTBFlags tb_flags = arm_tbflags_from_tb(tb); - if (FIELD_EX32(tb->flags, TBFLAG_AM32, THUMB)) { + if (EX_TBFLAG_AM32(tb_flags, THUMB)) { ops = &thumb_translator_ops; } #ifdef TARGET_AARCH64 - if (FIELD_EX32(tb->flags, TBFLAG_ANY, AARCH64_STATE)) { + if (EX_TBFLAG_ANY(tb_flags, AARCH64_STATE)) { ops = &aarch64_translator_ops; } #ifdef TARGET_CHERI diff --git a/target/arm/translate.h b/target/arm/translate.h index e6d12bdf6e7..eee408bb598 100644 --- a/target/arm/translate.h +++ b/target/arm/translate.h @@ -87,6 +87,8 @@ typedef struct DisasContext { bool bt; /* True if any CP15 access is trapped by HSTR_EL2 */ bool hstr_active; + /* True if memory operations require alignment */ + bool align_mem; /* * >= 0, a copy of PSTATE.BTYPE, which will be 0 without v8.5-BTI. * < 0, set by the current instruction. @@ -202,6 +204,7 @@ void arm_test_cc(DisasCompare *cmp, int cc); void arm_free_cc(DisasCompare *cmp); void arm_jump_cc(DisasCompare *cmp, TCGLabel *label); void arm_gen_test_cc(int cc, TCGLabel *label); +MemOp pow2_align(unsigned i); /* Return state of Alternate Half-precision flag, caller frees result */ static inline TCGv_i32 get_ahp_flag(void) @@ -395,6 +398,17 @@ typedef void CryptoThreeOpFn(TCGv_ptr, TCGv_ptr, TCGv_ptr); typedef void AtomicThreeOpFn(TCGv_i64, TCGv_cap_checked_ptr, TCGv_i64, TCGArg, MemOp); +/** + * arm_tbflags_from_tb: + * @tb: the TranslationBlock + * + * Extract the flag values from @tb. + */ +static inline CPUARMTBFlags arm_tbflags_from_tb(const TranslationBlock *tb) +{ + return (CPUARMTBFlags){ tb->flags, tb->cs_base }; +} + /* * Enum for argument to fpstatus_ptr(). */ @@ -447,4 +461,28 @@ static inline TCGv_ptr fpstatus_ptr(ARMFPStatusFlavour flavour) return statusptr; } +/** + * finalize_memop: + * @s: DisasContext + * @opc: size+sign+align of the memory operation + * + * Build the complete MemOp for a memory operation, including alignment + * and endianness. + * + * If (op & MO_AMASK) then the operation already contains the required + * alignment, e.g. for AccType_ATOMIC. Otherwise, this an optionally + * unaligned operation, e.g. for AccType_NORMAL. + * + * In the latter case, there are configuration bits that require alignment, + * and this is applied here. Note that there is no way to indicate that + * no alignment should ever be enforced; this must be handled manually. + */ +static inline MemOp finalize_memop(DisasContext *s, MemOp opc) +{ + if (s->align_mem && !(opc & MO_AMASK)) { + opc |= MO_ALIGN; + } + return opc | s->be_data; +} + #endif /* TARGET_ARM_TRANSLATE_H */ diff --git a/target/cheri-common/cpu_cheri.h b/target/cheri-common/cpu_cheri.h index 53943b02085..5b930082491 100644 --- a/target/cheri-common/cpu_cheri.h +++ b/target/cheri-common/cpu_cheri.h @@ -126,12 +126,12 @@ static inline bool cheri_cap_perms_valid_for_exec(const cap_register_t *pcc) static inline void cheri_cpu_get_tb_cpu_state(const cap_register_t *pcc, const cap_register_t *ddc, - target_ulong *cs_base, - target_ulong *cs_top, + target_ulong *pcc_base, + target_ulong *pcc_top, uint32_t *cheri_flags) { - *cs_base = cap_get_base(pcc); - *cs_top = cap_get_top(pcc); + *pcc_base = cap_get_base(pcc); + *pcc_top = cap_get_top(pcc); cheri_debug_assert(*cheri_flags == 0); if (cap_has_capmode_flag(pcc)) *cheri_flags |= TB_FLAG_CHERI_CAPMODE; @@ -143,7 +143,7 @@ static inline void cheri_cpu_get_tb_cpu_state(const cap_register_t *pcc, *cheri_flags |= TB_FLAG_CHERI_PCC_READABLE; } - if (*cs_base == 0) + if (*pcc_base == 0) *cheri_flags |= TB_FLAG_CHERI_PCC_BASE_ZERO; if (cap_get_top_full(pcc) == CAP_MAX_TOP) *cheri_flags |= TB_FLAG_CHERI_PCC_TOP_MAX; diff --git a/target/mips/cpu.h b/target/mips/cpu.h index f337bdbf552..f103a41e16c 100644 --- a/target/mips/cpu.h +++ b/target/mips/cpu.h @@ -1477,21 +1477,21 @@ target_ulong exception_resume_pc(CPUMIPSState *env); static inline void mips_cpu_get_tb_cpu_state(CPUMIPSState *env, target_ulong *pc, - target_ulong *cs_base, target_ulong *cs_top, - uint32_t *cheri_flags, uint32_t *flags) + target_ulong *cs_base, target_ulong *pcc_base, + target_ulong *pcc_top, uint32_t *cheri_flags, + uint32_t *flags) { *pc = PC_ADDR(env); // We want the full virtual address here (no offset) + *cs_base = 0; *flags = env->hflags & (MIPS_HFLAG_TMASK | MIPS_HFLAG_BMASK | MIPS_HFLAG_HWRENA_ULR); #ifdef TARGET_CHERI cheri_cpu_get_tb_cpu_state(&env->active_tc.PCC, &env->active_tc.CHWR.DDC, - cs_base, cs_top, cheri_flags); -#else - *cs_base = 0; + pcc_base, pcc_top, cheri_flags); #endif } // Ugly macro hack to avoid having to modify cpu_get_tb_cpu_state in all targets -#define cpu_get_tb_cpu_state_6 mips_cpu_get_tb_cpu_state +#define cpu_get_tb_cpu_state_ext mips_cpu_get_tb_cpu_state static inline bool should_use_error_epc(CPUMIPSState *env) { diff --git a/target/riscv/cpu.h b/target/riscv/cpu.h index 71ea82f7e21..a8313221e6b 100644 --- a/target/riscv/cpu.h +++ b/target/riscv/cpu.h @@ -706,17 +706,17 @@ static inline uint32_t vext_get_vlmax(RISCVCPU *cpu, target_ulong vtype) static inline void riscv_cpu_get_tb_cpu_state(CPURISCVState *env, target_ulong *pc, - target_ulong *cs_base, target_ulong *cs_top, - uint32_t *cheri_flags, uint32_t *pflags) + target_ulong *cs_base, target_ulong *pcc_base, + target_ulong *pcc_top, uint32_t *cheri_flags, + uint32_t *pflags) { uint32_t flags = 0; *pc = PC_ADDR(env); // We want the full virtual address here (no offset) #ifdef TARGET_CHERI - cheri_cpu_get_tb_cpu_state(&env->PCC, &env->DDC, cs_base, cs_top, + cheri_cpu_get_tb_cpu_state(&env->PCC, &env->DDC, pcc_base, pcc_top, cheri_flags); -#else - *cs_base = 0; #endif + *cs_base = 0; if (riscv_has_ext(env, RVV)) { uint32_t vlmax = vext_get_vlmax(env_archcpu(env), env->vtype); bool vl_eq_vlmax = (env->vstart == 0) && (vlmax == env->vl); @@ -752,7 +752,7 @@ riscv_cpu_get_tb_cpu_state(CPURISCVState *env, target_ulong *pc, *pflags = flags; } // Ugly macro hack to avoid having to modify cpu_get_tb_cpu_state in all targets -#define cpu_get_tb_cpu_state_6 riscv_cpu_get_tb_cpu_state +#define cpu_get_tb_cpu_state_ext riscv_cpu_get_tb_cpu_state #ifdef CONFIG_TCG_LOG_INSTR #define RISCV_LOG_INSTR_CPU_U QEMU_LOG_INSTR_CPU_USER diff --git a/tests/tcg/aarch64/Makefile.target b/tests/tcg/aarch64/Makefile.target index 05b2622bfc9..928357b10a9 100644 --- a/tests/tcg/aarch64/Makefile.target +++ b/tests/tcg/aarch64/Makefile.target @@ -37,7 +37,7 @@ AARCH64_TESTS += bti-2 # MTE Tests ifneq ($(DOCKER_IMAGE)$(CROSS_CC_HAS_ARMV8_MTE),) -AARCH64_TESTS += mte-1 mte-2 mte-3 mte-4 mte-6 +AARCH64_TESTS += mte-1 mte-2 mte-3 mte-4 mte-5 mte-6 mte-%: CFLAGS += -march=armv8.5-a+memtag endif diff --git a/tests/tcg/aarch64/mte-5.c b/tests/tcg/aarch64/mte-5.c new file mode 100644 index 00000000000..6dbd6ab3ea7 --- /dev/null +++ b/tests/tcg/aarch64/mte-5.c @@ -0,0 +1,44 @@ +/* + * Memory tagging, faulting unaligned access. + * + * Copyright (c) 2021 Linaro Ltd + * SPDX-License-Identifier: GPL-2.0-or-later + */ + +#include "mte.h" + +void pass(int sig, siginfo_t *info, void *uc) +{ + assert(info->si_code == SEGV_MTESERR); + exit(0); +} + +int main(int ac, char **av) +{ + struct sigaction sa; + void *p0, *p1, *p2; + long excl = 1; + + enable_mte(PR_MTE_TCF_SYNC); + p0 = alloc_mte_mem(sizeof(*p0)); + + /* Create two differently tagged pointers. */ + asm("irg %0,%1,%2" : "=r"(p1) : "r"(p0), "r"(excl)); + asm("gmi %0,%1,%0" : "+r"(excl) : "r" (p1)); + assert(excl != 1); + asm("irg %0,%1,%2" : "=r"(p2) : "r"(p0), "r"(excl)); + assert(p1 != p2); + + memset(&sa, 0, sizeof(sa)); + sa.sa_sigaction = pass; + sa.sa_flags = SA_SIGINFO; + sigaction(SIGSEGV, &sa, NULL); + + /* Store store two different tags in sequential granules. */ + asm("stg %0, [%0]" : : "r"(p1)); + asm("stg %0, [%0]" : : "r"(p2 + 16)); + + /* Perform an unaligned load crossing the granules. */ + asm volatile("ldr %0, [%1]" : "=r"(p0) : "r"(p1 + 12)); + abort(); +}