1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
|
/**
* Copyright (c) 2020 Raspberry Pi (Trading) Ltd.
* Copyright (c) 2022 Nick Brassel (@tzarc)
* Copyright (c) 2022 Stefan Kerkmann (@KarlK90)
*
* SPDX-License-Identifier: BSD-3-Clause
*/
#include "pico/bootrom.h"
#include "hardware/flash.h"
#include "hardware/sync.h"
#include "hardware/structs/ssi.h"
#include "hardware/structs/ioqspi.h"
#include <stdbool.h>
#include "timer.h"
#include "wear_leveling.h"
#include "wear_leveling_internal.h"
#ifndef WEAR_LEVELING_RP2040_FLASH_BULK_COUNT
# define WEAR_LEVELING_RP2040_FLASH_BULK_COUNT 64
#endif // WEAR_LEVELING_RP2040_FLASH_BULK_COUNT
#define FLASHCMD_PAGE_PROGRAM 0x02
#define FLASHCMD_READ_STATUS 0x05
#define FLASHCMD_WRITE_ENABLE 0x06
extern uint8_t BOOT2_ROM[256];
static uint32_t BOOT2_ROM_RAM[64];
static ssi_hw_t *const ssi = (ssi_hw_t *)XIP_SSI_BASE;
// Sanity check
check_hw_layout(ssi_hw_t, ssienr, SSI_SSIENR_OFFSET);
check_hw_layout(ssi_hw_t, spi_ctrlr0, SSI_SPI_CTRLR0_OFFSET);
static void __no_inline_not_in_flash_func(flash_enable_xip_via_boot2)(void) {
((void (*)(void))BOOT2_ROM_RAM + 1)();
}
// Bitbanging the chip select using IO overrides, in case RAM-resident IRQs
// are still running, and the FIFO bottoms out. (the bootrom does the same)
static void __no_inline_not_in_flash_func(flash_cs_force)(bool high) {
uint32_t field_val = high ? IO_QSPI_GPIO_QSPI_SS_CTRL_OUTOVER_VALUE_HIGH : IO_QSPI_GPIO_QSPI_SS_CTRL_OUTOVER_VALUE_LOW;
hw_write_masked(&ioqspi_hw->io[1].ctrl, field_val << IO_QSPI_GPIO_QSPI_SS_CTRL_OUTOVER_LSB, IO_QSPI_GPIO_QSPI_SS_CTRL_OUTOVER_BITS);
}
// Also allow any unbounded loops to check whether the above abort condition
// was asserted, and terminate early
static int __no_inline_not_in_flash_func(flash_was_aborted)(void) {
return *(io_rw_32 *)(IO_QSPI_BASE + IO_QSPI_GPIO_QSPI_SD1_CTRL_OFFSET) & IO_QSPI_GPIO_QSPI_SD1_CTRL_INOVER_BITS;
}
// Put bytes from one buffer, and get bytes into another buffer.
// These can be the same buffer.
// If tx is NULL then send zeroes.
// If rx is NULL then all read data will be dropped.
//
// If rx_skip is nonzero, this many bytes will first be consumed from the FIFO,
// before reading a further count bytes into *rx.
// E.g. if you have written a command+address just before calling this function.
static void __no_inline_not_in_flash_func(flash_put_get)(const uint8_t *tx, uint8_t *rx, size_t count, size_t rx_skip) {
// Make sure there is never more data in flight than the depth of the RX
// FIFO. Otherwise, when we are interrupted for long periods, hardware
// will overflow the RX FIFO.
const uint max_in_flight = 16 - 2; // account for data internal to SSI
size_t tx_count = count;
size_t rx_count = count;
while (tx_count || rx_skip || rx_count) {
// NB order of reads, for pessimism rather than optimism
uint32_t tx_level = ssi_hw->txflr;
uint32_t rx_level = ssi_hw->rxflr;
bool did_something = false; // Expect this to be folded into control flow, not register
if (tx_count && tx_level + rx_level < max_in_flight) {
ssi->dr0 = (uint32_t)(tx ? *tx++ : 0);
--tx_count;
did_something = true;
}
if (rx_level) {
uint8_t rxbyte = ssi->dr0;
did_something = true;
if (rx_skip) {
--rx_skip;
} else {
if (rx) *rx++ = rxbyte;
--rx_count;
}
}
// APB load costs 4 cycles, so only do it on idle loops (our budget is
// 48 cyc/byte)
if (!did_something && __builtin_expect(flash_was_aborted(), 0)) break;
}
flash_cs_force(1);
}
// Convenience wrapper for above
// (And it's hard for the debug host to get the tight timing between
// cmd DR0 write and the remaining data)
static void __no_inline_not_in_flash_func(_flash_do_cmd)(uint8_t cmd, const uint8_t *tx, uint8_t *rx, size_t count) {
flash_cs_force(0);
ssi->dr0 = cmd;
flash_put_get(tx, rx, count, 1);
}
// Timing of this one is critical, so do not expose the symbol to debugger etc
static void __no_inline_not_in_flash_func(flash_put_cmd_addr)(uint8_t cmd, uint32_t addr) {
flash_cs_force(0);
addr |= cmd << 24;
for (int i = 0; i < 4; ++i) {
ssi->dr0 = addr >> 24;
addr <<= 8;
}
}
// Poll the flash status register until the busy bit (LSB) clears
static void __no_inline_not_in_flash_func(flash_wait_ready)(void) {
uint8_t stat;
do {
_flash_do_cmd(FLASHCMD_READ_STATUS, NULL, &stat, 1);
} while (stat & 0x1 && !flash_was_aborted());
}
// Set the WEL bit (needed before any program/erase operation)
static void __no_inline_not_in_flash_func(flash_enable_write)(void) {
_flash_do_cmd(FLASHCMD_WRITE_ENABLE, NULL, NULL, 0);
}
static void __no_inline_not_in_flash_func(pico_program_bulk)(uint32_t flash_address, backing_store_int_t *values, size_t item_count) {
rom_connect_internal_flash_fn connect_internal_flash = (rom_connect_internal_flash_fn)rom_func_lookup_inline(ROM_FUNC_CONNECT_INTERNAL_FLASH);
rom_flash_exit_xip_fn flash_exit_xip = (rom_flash_exit_xip_fn)rom_func_lookup_inline(ROM_FUNC_FLASH_EXIT_XIP);
rom_flash_flush_cache_fn flash_flush_cache = (rom_flash_flush_cache_fn)rom_func_lookup_inline(ROM_FUNC_FLASH_FLUSH_CACHE);
assert(connect_internal_flash && flash_exit_xip && flash_flush_cache);
static backing_store_int_t bulk_write_buffer[WEAR_LEVELING_RP2040_FLASH_BULK_COUNT];
while (item_count) {
size_t batch_size = MIN(item_count, WEAR_LEVELING_RP2040_FLASH_BULK_COUNT);
for (size_t i = 0; i < batch_size; i++, values++, item_count--) {
bulk_write_buffer[i] = ~(*values);
}
__compiler_memory_barrier();
connect_internal_flash();
flash_exit_xip();
flash_enable_write();
flash_put_cmd_addr(FLASHCMD_PAGE_PROGRAM, flash_address);
flash_put_get((uint8_t *)bulk_write_buffer, NULL, batch_size * sizeof(backing_store_int_t), 4);
flash_wait_ready();
flash_address += batch_size * sizeof(backing_store_int_t);
flash_flush_cache();
flash_enable_xip_via_boot2();
}
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// QMK Wear-Leveling Backing Store implementation
static int interrupts;
bool backing_store_init(void) {
bs_dprintf("Init\n");
memcpy(BOOT2_ROM_RAM, BOOT2_ROM, sizeof(BOOT2_ROM));
__compiler_memory_barrier();
return true;
}
bool backing_store_unlock(void) {
bs_dprintf("Unlock\n");
return true;
}
bool backing_store_erase(void) {
#ifdef WEAR_LEVELING_DEBUG_OUTPUT
uint32_t start = timer_read32();
#endif
// Ensure the backing size can be cleanly subtracted from the flash size without alignment issues.
_Static_assert((WEAR_LEVELING_BACKING_SIZE) % (FLASH_SECTOR_SIZE) == 0, "Backing size must be a multiple of FLASH_SECTOR_SIZE");
interrupts = save_and_disable_interrupts();
flash_range_erase((WEAR_LEVELING_RP2040_FLASH_BASE), (WEAR_LEVELING_BACKING_SIZE));
restore_interrupts(interrupts);
bs_dprintf("Backing store erase took %ldms to complete\n", ((long)(timer_read32() - start)));
return true;
}
bool backing_store_write(uint32_t address, backing_store_int_t value) {
return backing_store_write_bulk(address, &value, 1);
}
bool backing_store_write_bulk(uint32_t address, backing_store_int_t *values, size_t item_count) {
uint32_t offset = (WEAR_LEVELING_RP2040_FLASH_BASE) + address;
bs_dprintf("Write ");
wl_dump(offset, values, sizeof(backing_store_int_t) * item_count);
interrupts = save_and_disable_interrupts();
pico_program_bulk(offset, values, item_count);
restore_interrupts(interrupts);
return true;
}
bool backing_store_lock(void) {
return true;
}
bool backing_store_read(uint32_t address, backing_store_int_t *value) {
return backing_store_read_bulk(address, value, 1);
}
bool backing_store_read_bulk(uint32_t address, backing_store_int_t *values, size_t item_count) {
uint32_t offset = (WEAR_LEVELING_RP2040_FLASH_BASE) + address;
backing_store_int_t *loc = (backing_store_int_t *)((XIP_BASE) + offset);
for (size_t i = 0; i < item_count; ++i) {
values[i] = ~loc[i];
}
bs_dprintf("Read ");
wl_dump(offset, values, item_count * sizeof(backing_store_int_t));
return true;
}
|
