early-access version 2807
parent
ad703668f3
commit
514bed1c36
|
@ -1,7 +1,7 @@
|
||||||
yuzu emulator early access
|
yuzu emulator early access
|
||||||
=============
|
=============
|
||||||
|
|
||||||
This is the source code for early-access 2806.
|
This is the source code for early-access 2807.
|
||||||
|
|
||||||
## Legal Notice
|
## Legal Notice
|
||||||
|
|
||||||
|
|
|
@ -23,9 +23,29 @@ template <typename VaType, VaType UnmappedVa, typename PaType, PaType UnmappedPa
|
||||||
bool PaContigSplit, size_t AddressSpaceBits, typename ExtraBlockInfo = EmptyStruct>
|
bool PaContigSplit, size_t AddressSpaceBits, typename ExtraBlockInfo = EmptyStruct>
|
||||||
requires AddressSpaceValid<VaType, AddressSpaceBits>
|
requires AddressSpaceValid<VaType, AddressSpaceBits>
|
||||||
class FlatAddressSpaceMap {
|
class FlatAddressSpaceMap {
|
||||||
private:
|
public:
|
||||||
std::function<void(VaType, VaType)>
|
/// The maximum VA that this AS can technically reach
|
||||||
unmapCallback{}; //!< Callback called when the mappings in an region have changed
|
static constexpr VaType VaMaximum{(1ULL << (AddressSpaceBits - 1)) +
|
||||||
|
((1ULL << (AddressSpaceBits - 1)) - 1)};
|
||||||
|
|
||||||
|
explicit FlatAddressSpaceMap(VaType va_limit,
|
||||||
|
std::function<void(VaType, VaType)> unmap_callback = {});
|
||||||
|
|
||||||
|
FlatAddressSpaceMap() = default;
|
||||||
|
|
||||||
|
void Map(VaType virt, PaType phys, VaType size, ExtraBlockInfo extra_info = {}) {
|
||||||
|
std::scoped_lock lock(block_mutex);
|
||||||
|
MapLocked(virt, phys, size, extra_info);
|
||||||
|
}
|
||||||
|
|
||||||
|
void Unmap(VaType virt, VaType size) {
|
||||||
|
std::scoped_lock lock(block_mutex);
|
||||||
|
UnmapLocked(virt, size);
|
||||||
|
}
|
||||||
|
|
||||||
|
VaType GetVALimit() const {
|
||||||
|
return va_limit;
|
||||||
|
}
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
/**
|
/**
|
||||||
|
@ -33,68 +53,55 @@ protected:
|
||||||
* another block with a different phys address is hit
|
* another block with a different phys address is hit
|
||||||
*/
|
*/
|
||||||
struct Block {
|
struct Block {
|
||||||
VaType virt{UnmappedVa}; //!< VA of the block
|
/// VA of the block
|
||||||
PaType phys{UnmappedPa}; //!< PA of the block, will increase 1-1 with VA until a new block
|
VaType virt{UnmappedVa};
|
||||||
//!< is encountered
|
/// PA of the block, will increase 1-1 with VA until a new block is encountered
|
||||||
[[no_unique_address]] ExtraBlockInfo extraInfo;
|
PaType phys{UnmappedPa};
|
||||||
|
[[no_unique_address]] ExtraBlockInfo extra_info;
|
||||||
|
|
||||||
Block() = default;
|
Block() = default;
|
||||||
|
|
||||||
Block(VaType virt_, PaType phys_, ExtraBlockInfo extraInfo_)
|
Block(VaType virt_, PaType phys_, ExtraBlockInfo extra_info_)
|
||||||
: virt(virt_), phys(phys_), extraInfo(extraInfo_) {}
|
: virt(virt_), phys(phys_), extra_info(extra_info_) {}
|
||||||
|
|
||||||
constexpr bool Valid() {
|
bool Valid() const {
|
||||||
return virt != UnmappedVa;
|
return virt != UnmappedVa;
|
||||||
}
|
}
|
||||||
|
|
||||||
constexpr bool Mapped() {
|
bool Mapped() const {
|
||||||
return phys != UnmappedPa;
|
return phys != UnmappedPa;
|
||||||
}
|
}
|
||||||
|
|
||||||
constexpr bool Unmapped() {
|
bool Unmapped() const {
|
||||||
return phys == UnmappedPa;
|
return phys == UnmappedPa;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool operator<(const VaType& pVirt) const {
|
bool operator<(const VaType& p_virt) const {
|
||||||
return virt < pVirt;
|
return virt < p_virt;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
std::mutex blockMutex;
|
|
||||||
std::vector<Block> blocks{Block{}};
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Maps a PA range into the given AS region
|
* @brief Maps a PA range into the given AS region
|
||||||
* @note blockMutex MUST be locked when calling this
|
* @note block_mutex MUST be locked when calling this
|
||||||
*/
|
*/
|
||||||
void MapLocked(VaType virt, PaType phys, VaType size, ExtraBlockInfo extraInfo);
|
void MapLocked(VaType virt, PaType phys, VaType size, ExtraBlockInfo extra_info);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Unmaps the given range and merges it with other unmapped regions
|
* @brief Unmaps the given range and merges it with other unmapped regions
|
||||||
* @note blockMutex MUST be locked when calling this
|
* @note block_mutex MUST be locked when calling this
|
||||||
*/
|
*/
|
||||||
void UnmapLocked(VaType virt, VaType size);
|
void UnmapLocked(VaType virt, VaType size);
|
||||||
|
|
||||||
public:
|
std::mutex block_mutex;
|
||||||
static constexpr VaType VaMaximum{(1ULL << (AddressSpaceBits - 1)) +
|
std::vector<Block> blocks{Block{}};
|
||||||
((1ULL << (AddressSpaceBits - 1)) -
|
|
||||||
1)}; //!< The maximum VA that this AS can technically reach
|
|
||||||
|
|
||||||
VaType vaLimit{VaMaximum}; //!< A soft limit on the maximum VA of the AS
|
/// a soft limit on the maximum VA of the AS
|
||||||
|
VaType va_limit{VaMaximum};
|
||||||
|
|
||||||
FlatAddressSpaceMap(VaType vaLimit, std::function<void(VaType, VaType)> unmapCallback = {});
|
private:
|
||||||
|
/// Callback called when the mappings in an region have changed
|
||||||
FlatAddressSpaceMap() = default;
|
std::function<void(VaType, VaType)> unmap_callback{};
|
||||||
|
|
||||||
void Map(VaType virt, PaType phys, VaType size, ExtraBlockInfo extraInfo = {}) {
|
|
||||||
std::scoped_lock lock(blockMutex);
|
|
||||||
MapLocked(virt, phys, size, extraInfo);
|
|
||||||
}
|
|
||||||
|
|
||||||
void Unmap(VaType virt, VaType size) {
|
|
||||||
std::scoped_lock lock(blockMutex);
|
|
||||||
UnmapLocked(virt, size);
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -108,14 +115,8 @@ class FlatAllocator
|
||||||
private:
|
private:
|
||||||
using Base = FlatAddressSpaceMap<VaType, UnmappedVa, bool, false, false, AddressSpaceBits>;
|
using Base = FlatAddressSpaceMap<VaType, UnmappedVa, bool, false, false, AddressSpaceBits>;
|
||||||
|
|
||||||
VaType currentLinearAllocEnd; //!< The end address for the initial linear allocation pass, once
|
|
||||||
//!< this reaches the AS limit the slower allocation path will be
|
|
||||||
//!< used
|
|
||||||
|
|
||||||
public:
|
public:
|
||||||
VaType vaStart; //!< The base VA of the allocator, no allocations will be below this
|
explicit FlatAllocator(VaType virt_start, VaType va_limit = Base::VaMaximum);
|
||||||
|
|
||||||
FlatAllocator(VaType vaStart, VaType vaLimit = Base::VaMaximum);
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Allocates a region in the AS of the given size and returns its address
|
* @brief Allocates a region in the AS of the given size and returns its address
|
||||||
|
@ -131,5 +132,19 @@ public:
|
||||||
* @brief Frees an AS region so it can be used again
|
* @brief Frees an AS region so it can be used again
|
||||||
*/
|
*/
|
||||||
void Free(VaType virt, VaType size);
|
void Free(VaType virt, VaType size);
|
||||||
|
|
||||||
|
VaType GetVAStart() const {
|
||||||
|
return virt_start;
|
||||||
|
}
|
||||||
|
|
||||||
|
private:
|
||||||
|
/// The base VA of the allocator, no allocations will be below this
|
||||||
|
VaType virt_start;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The end address for the initial linear allocation pass
|
||||||
|
* Once this reaches the AS limit the slower allocation path will be used
|
||||||
|
*/
|
||||||
|
VaType current_linear_alloc_end;
|
||||||
};
|
};
|
||||||
} // namespace Common
|
} // namespace Common
|
||||||
|
|
|
@ -30,137 +30,151 @@
|
||||||
FlatAllocator<VaType, UnmappedVa, AddressSpaceBits>
|
FlatAllocator<VaType, UnmappedVa, AddressSpaceBits>
|
||||||
|
|
||||||
namespace Common {
|
namespace Common {
|
||||||
MAP_MEMBER_CONST()::FlatAddressSpaceMap(VaType vaLimit_,
|
MAP_MEMBER_CONST()::FlatAddressSpaceMap(VaType va_limit_,
|
||||||
std::function<void(VaType, VaType)> unmapCallback_)
|
std::function<void(VaType, VaType)> unmap_callback_)
|
||||||
: unmapCallback(std::move(unmapCallback_)), vaLimit(vaLimit_) {
|
: va_limit{va_limit_}, unmap_callback{std::move(unmap_callback_)} {
|
||||||
if (vaLimit > VaMaximum)
|
if (va_limit > VaMaximum) {
|
||||||
UNREACHABLE_MSG("Invalid VA limit!");
|
UNREACHABLE_MSG("Invalid VA limit!");
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
MAP_MEMBER(void)::MapLocked(VaType virt, PaType phys, VaType size, ExtraBlockInfo extraInfo) {
|
MAP_MEMBER(void)::MapLocked(VaType virt, PaType phys, VaType size, ExtraBlockInfo extra_info) {
|
||||||
VaType virtEnd{virt + size};
|
VaType virt_end{virt + size};
|
||||||
|
|
||||||
if (virtEnd > vaLimit)
|
if (virt_end > va_limit) {
|
||||||
UNREACHABLE_MSG("Trying to map a block past the VA limit: virtEnd: 0x{:X}, vaLimit: 0x{:X}",
|
UNREACHABLE_MSG(
|
||||||
virtEnd, vaLimit);
|
"Trying to map a block past the VA limit: virt_end: 0x{:X}, va_limit: 0x{:X}", virt_end,
|
||||||
|
va_limit);
|
||||||
|
}
|
||||||
|
|
||||||
auto blockEndSuccessor{std::lower_bound(blocks.begin(), blocks.end(), virtEnd)};
|
auto block_end_successor{std::lower_bound(blocks.begin(), blocks.end(), virt_end)};
|
||||||
if (blockEndSuccessor == blocks.begin())
|
if (block_end_successor == blocks.begin()) {
|
||||||
UNREACHABLE_MSG("Trying to map a block before the VA start: virtEnd: 0x{:X}", virtEnd);
|
UNREACHABLE_MSG("Trying to map a block before the VA start: virt_end: 0x{:X}", virt_end);
|
||||||
|
}
|
||||||
|
|
||||||
auto blockEndPredecessor{std::prev(blockEndSuccessor)};
|
auto block_end_predecessor{std::prev(block_end_successor)};
|
||||||
|
|
||||||
if (blockEndSuccessor != blocks.end()) {
|
if (block_end_successor != blocks.end()) {
|
||||||
// We have blocks in front of us, if one is directly in front then we don't have to add a
|
// We have blocks in front of us, if one is directly in front then we don't have to add a
|
||||||
// tail
|
// tail
|
||||||
if (blockEndSuccessor->virt != virtEnd) {
|
if (block_end_successor->virt != virt_end) {
|
||||||
PaType tailPhys{[&]() -> PaType {
|
PaType tailPhys{[&]() -> PaType {
|
||||||
if constexpr (!PaContigSplit) {
|
if constexpr (!PaContigSplit) {
|
||||||
return blockEndPredecessor
|
// Always propagate unmapped regions rather than calculating offset
|
||||||
->phys; // Always propagate unmapped regions rather than calculating offset
|
return block_end_predecessor->phys;
|
||||||
} else {
|
} else {
|
||||||
if (blockEndPredecessor->Unmapped())
|
if (block_end_predecessor->Unmapped()) {
|
||||||
return blockEndPredecessor->phys; // Always propagate unmapped regions
|
// Always propagate unmapped regions rather than calculating offset
|
||||||
// rather than calculating offset
|
return block_end_predecessor->phys;
|
||||||
else
|
} else {
|
||||||
return blockEndPredecessor->phys + virtEnd - blockEndPredecessor->virt;
|
return block_end_predecessor->phys + virt_end - block_end_predecessor->virt;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}()};
|
}()};
|
||||||
|
|
||||||
if (blockEndPredecessor->virt >= virt) {
|
if (block_end_predecessor->virt >= virt) {
|
||||||
// If this block's start would be overlapped by the map then reuse it as a tail
|
// If this block's start would be overlapped by the map then reuse it as a tail
|
||||||
// block
|
// block
|
||||||
blockEndPredecessor->virt = virtEnd;
|
block_end_predecessor->virt = virt_end;
|
||||||
blockEndPredecessor->phys = tailPhys;
|
block_end_predecessor->phys = tailPhys;
|
||||||
blockEndPredecessor->extraInfo = blockEndPredecessor->extraInfo;
|
block_end_predecessor->extra_info = block_end_predecessor->extra_info;
|
||||||
|
|
||||||
// No longer predecessor anymore
|
// No longer predecessor anymore
|
||||||
blockEndSuccessor = blockEndPredecessor--;
|
block_end_successor = block_end_predecessor--;
|
||||||
} else {
|
} else {
|
||||||
// Else insert a new one and we're done
|
// Else insert a new one and we're done
|
||||||
blocks.insert(blockEndSuccessor,
|
blocks.insert(block_end_successor,
|
||||||
{Block(virt, phys, extraInfo),
|
{Block(virt, phys, extra_info),
|
||||||
Block(virtEnd, tailPhys, blockEndPredecessor->extraInfo)});
|
Block(virt_end, tailPhys, block_end_predecessor->extra_info)});
|
||||||
if (unmapCallback)
|
if (unmap_callback) {
|
||||||
unmapCallback(virt, size);
|
unmap_callback(virt, size);
|
||||||
|
}
|
||||||
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// blockEndPredecessor will always be unmapped as blocks has to be terminated by an unmapped
|
// block_end_predecessor will always be unmapped as blocks has to be terminated by an
|
||||||
// chunk
|
// unmapped chunk
|
||||||
if (blockEndPredecessor != blocks.begin() && blockEndPredecessor->virt >= virt) {
|
if (block_end_predecessor != blocks.begin() && block_end_predecessor->virt >= virt) {
|
||||||
// Move the unmapped block start backwards
|
// Move the unmapped block start backwards
|
||||||
blockEndPredecessor->virt = virtEnd;
|
block_end_predecessor->virt = virt_end;
|
||||||
|
|
||||||
// No longer predecessor anymore
|
// No longer predecessor anymore
|
||||||
blockEndSuccessor = blockEndPredecessor--;
|
block_end_successor = block_end_predecessor--;
|
||||||
} else {
|
} else {
|
||||||
// Else insert a new one and we're done
|
// Else insert a new one and we're done
|
||||||
blocks.insert(blockEndSuccessor,
|
blocks.insert(block_end_successor,
|
||||||
{Block(virt, phys, extraInfo), Block(virtEnd, UnmappedPa, {})});
|
{Block(virt, phys, extra_info), Block(virt_end, UnmappedPa, {})});
|
||||||
if (unmapCallback)
|
if (unmap_callback) {
|
||||||
unmapCallback(virt, size);
|
unmap_callback(virt, size);
|
||||||
|
}
|
||||||
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
auto blockStartSuccessor{blockEndSuccessor};
|
auto block_start_successor{block_end_successor};
|
||||||
|
|
||||||
// Walk the block vector to find the start successor as this is more efficient than another
|
// Walk the block vector to find the start successor as this is more efficient than another
|
||||||
// binary search in most scenarios
|
// binary search in most scenarios
|
||||||
while (std::prev(blockStartSuccessor)->virt >= virt)
|
while (std::prev(block_start_successor)->virt >= virt) {
|
||||||
blockStartSuccessor--;
|
block_start_successor--;
|
||||||
|
|
||||||
// Check that the start successor is either the end block or something in between
|
|
||||||
if (blockStartSuccessor->virt > virtEnd) {
|
|
||||||
UNREACHABLE_MSG("Unsorted block in AS map: virt: 0x{:X}", blockStartSuccessor->virt);
|
|
||||||
} else if (blockStartSuccessor->virt == virtEnd) {
|
|
||||||
// We need to create a new block as there are none spare that we would overwrite
|
|
||||||
blocks.insert(blockStartSuccessor, Block(virt, phys, extraInfo));
|
|
||||||
} else {
|
|
||||||
// Erase overwritten blocks
|
|
||||||
if (auto eraseStart{std::next(blockStartSuccessor)}; eraseStart != blockEndSuccessor)
|
|
||||||
blocks.erase(eraseStart, blockEndSuccessor);
|
|
||||||
|
|
||||||
// Reuse a block that would otherwise be overwritten as a start block
|
|
||||||
blockStartSuccessor->virt = virt;
|
|
||||||
blockStartSuccessor->phys = phys;
|
|
||||||
blockStartSuccessor->extraInfo = extraInfo;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (unmapCallback)
|
// Check that the start successor is either the end block or something in between
|
||||||
unmapCallback(virt, size);
|
if (block_start_successor->virt > virt_end) {
|
||||||
|
UNREACHABLE_MSG("Unsorted block in AS map: virt: 0x{:X}", block_start_successor->virt);
|
||||||
|
} else if (block_start_successor->virt == virt_end) {
|
||||||
|
// We need to create a new block as there are none spare that we would overwrite
|
||||||
|
blocks.insert(block_start_successor, Block(virt, phys, extra_info));
|
||||||
|
} else {
|
||||||
|
// Erase overwritten blocks
|
||||||
|
if (auto eraseStart{std::next(block_start_successor)}; eraseStart != block_end_successor) {
|
||||||
|
blocks.erase(eraseStart, block_end_successor);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reuse a block that would otherwise be overwritten as a start block
|
||||||
|
block_start_successor->virt = virt;
|
||||||
|
block_start_successor->phys = phys;
|
||||||
|
block_start_successor->extra_info = extra_info;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (unmap_callback) {
|
||||||
|
unmap_callback(virt, size);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
MAP_MEMBER(void)::UnmapLocked(VaType virt, VaType size) {
|
MAP_MEMBER(void)::UnmapLocked(VaType virt, VaType size) {
|
||||||
VaType virtEnd{virt + size};
|
VaType virt_end{virt + size};
|
||||||
|
|
||||||
if (virtEnd > vaLimit)
|
if (virt_end > va_limit) {
|
||||||
UNREACHABLE_MSG("Trying to map a block past the VA limit: virtEnd: 0x{:X}, vaLimit: 0x{:X}",
|
UNREACHABLE_MSG(
|
||||||
virtEnd, vaLimit);
|
"Trying to map a block past the VA limit: virt_end: 0x{:X}, va_limit: 0x{:X}", virt_end,
|
||||||
|
va_limit);
|
||||||
|
}
|
||||||
|
|
||||||
auto blockEndSuccessor{std::lower_bound(blocks.begin(), blocks.end(), virtEnd)};
|
auto block_end_successor{std::lower_bound(blocks.begin(), blocks.end(), virt_end)};
|
||||||
if (blockEndSuccessor == blocks.begin())
|
if (block_end_successor == blocks.begin()) {
|
||||||
UNREACHABLE_MSG("Trying to unmap a block before the VA start: virtEnd: 0x{:X}", virtEnd);
|
UNREACHABLE_MSG("Trying to unmap a block before the VA start: virt_end: 0x{:X}", virt_end);
|
||||||
|
}
|
||||||
|
|
||||||
auto blockEndPredecessor{std::prev(blockEndSuccessor)};
|
auto block_end_predecessor{std::prev(block_end_successor)};
|
||||||
|
|
||||||
auto walkBackToPredecessor{[&](auto iter) {
|
auto walk_back_to_predecessor{[&](auto iter) {
|
||||||
while (iter->virt >= virt)
|
while (iter->virt >= virt) {
|
||||||
iter--;
|
iter--;
|
||||||
|
}
|
||||||
|
|
||||||
return iter;
|
return iter;
|
||||||
}};
|
}};
|
||||||
|
|
||||||
auto eraseBlocksWithEndUnmapped{[&](auto unmappedEnd) {
|
auto erase_blocks_with_end_unmapped{[&](auto unmappedEnd) {
|
||||||
auto blockStartPredecessor{walkBackToPredecessor(unmappedEnd)};
|
auto block_start_predecessor{walk_back_to_predecessor(unmappedEnd)};
|
||||||
auto blockStartSuccessor{std::next(blockStartPredecessor)};
|
auto block_start_successor{std::next(block_start_predecessor)};
|
||||||
|
|
||||||
auto eraseEnd{[&]() {
|
auto eraseEnd{[&]() {
|
||||||
if (blockStartPredecessor->Unmapped()) {
|
if (block_start_predecessor->Unmapped()) {
|
||||||
// If the start predecessor is unmapped then we can erase everything in our region
|
// If the start predecessor is unmapped then we can erase everything in our region
|
||||||
// and be done
|
// and be done
|
||||||
return std::next(unmappedEnd);
|
return std::next(unmappedEnd);
|
||||||
|
@ -174,158 +188,171 @@ MAP_MEMBER(void)::UnmapLocked(VaType virt, VaType size) {
|
||||||
|
|
||||||
// We can't have two unmapped regions after each other
|
// We can't have two unmapped regions after each other
|
||||||
if (eraseEnd != blocks.end() &&
|
if (eraseEnd != blocks.end() &&
|
||||||
(eraseEnd == blockStartSuccessor ||
|
(eraseEnd == block_start_successor ||
|
||||||
(blockStartPredecessor->Unmapped() && eraseEnd->Unmapped())))
|
(block_start_predecessor->Unmapped() && eraseEnd->Unmapped()))) {
|
||||||
UNREACHABLE_MSG("Multiple contiguous unmapped regions are unsupported!");
|
UNREACHABLE_MSG("Multiple contiguous unmapped regions are unsupported!");
|
||||||
|
}
|
||||||
|
|
||||||
blocks.erase(blockStartSuccessor, eraseEnd);
|
blocks.erase(block_start_successor, eraseEnd);
|
||||||
}};
|
}};
|
||||||
|
|
||||||
// We can avoid any splitting logic if these are the case
|
// We can avoid any splitting logic if these are the case
|
||||||
if (blockEndPredecessor->Unmapped()) {
|
if (block_end_predecessor->Unmapped()) {
|
||||||
if (blockEndPredecessor->virt > virt)
|
if (block_end_predecessor->virt > virt) {
|
||||||
eraseBlocksWithEndUnmapped(blockEndPredecessor);
|
erase_blocks_with_end_unmapped(block_end_predecessor);
|
||||||
|
}
|
||||||
|
|
||||||
if (unmapCallback)
|
if (unmap_callback) {
|
||||||
unmapCallback(virt, size);
|
unmap_callback(virt, size);
|
||||||
|
}
|
||||||
|
|
||||||
return; // The region is unmapped, bail out early
|
return; // The region is unmapped, bail out early
|
||||||
} else if (blockEndSuccessor->virt == virtEnd && blockEndSuccessor->Unmapped()) {
|
} else if (block_end_successor->virt == virt_end && block_end_successor->Unmapped()) {
|
||||||
eraseBlocksWithEndUnmapped(blockEndSuccessor);
|
erase_blocks_with_end_unmapped(block_end_successor);
|
||||||
|
|
||||||
if (unmapCallback)
|
if (unmap_callback) {
|
||||||
unmapCallback(virt, size);
|
unmap_callback(virt, size);
|
||||||
|
}
|
||||||
|
|
||||||
return; // The region is unmapped here and doesn't need splitting, bail out early
|
return; // The region is unmapped here and doesn't need splitting, bail out early
|
||||||
} else if (blockEndSuccessor == blocks.end()) {
|
} else if (block_end_successor == blocks.end()) {
|
||||||
// This should never happen as the end should always follow an unmapped block
|
// This should never happen as the end should always follow an unmapped block
|
||||||
UNREACHABLE_MSG("Unexpected Memory Manager state!");
|
UNREACHABLE_MSG("Unexpected Memory Manager state!");
|
||||||
} else if (blockEndSuccessor->virt != virtEnd) {
|
} else if (block_end_successor->virt != virt_end) {
|
||||||
// If one block is directly in front then we don't have to add a tail
|
// If one block is directly in front then we don't have to add a tail
|
||||||
|
|
||||||
// The previous block is mapped so we will need to add a tail with an offset
|
// The previous block is mapped so we will need to add a tail with an offset
|
||||||
PaType tailPhys{[&]() {
|
PaType tailPhys{[&]() {
|
||||||
if constexpr (PaContigSplit)
|
if constexpr (PaContigSplit) {
|
||||||
return blockEndPredecessor->phys + virtEnd - blockEndPredecessor->virt;
|
return block_end_predecessor->phys + virt_end - block_end_predecessor->virt;
|
||||||
else
|
} else {
|
||||||
return blockEndPredecessor->phys;
|
return block_end_predecessor->phys;
|
||||||
|
}
|
||||||
}()};
|
}()};
|
||||||
|
|
||||||
if (blockEndPredecessor->virt >= virt) {
|
if (block_end_predecessor->virt >= virt) {
|
||||||
// If this block's start would be overlapped by the unmap then reuse it as a tail block
|
// If this block's start would be overlapped by the unmap then reuse it as a tail block
|
||||||
blockEndPredecessor->virt = virtEnd;
|
block_end_predecessor->virt = virt_end;
|
||||||
blockEndPredecessor->phys = tailPhys;
|
block_end_predecessor->phys = tailPhys;
|
||||||
|
|
||||||
// No longer predecessor anymore
|
// No longer predecessor anymore
|
||||||
blockEndSuccessor = blockEndPredecessor--;
|
block_end_successor = block_end_predecessor--;
|
||||||
} else {
|
} else {
|
||||||
blocks.insert(blockEndSuccessor,
|
blocks.insert(block_end_successor,
|
||||||
{Block(virt, UnmappedPa, {}),
|
{Block(virt, UnmappedPa, {}),
|
||||||
Block(virtEnd, tailPhys, blockEndPredecessor->extraInfo)});
|
Block(virt_end, tailPhys, block_end_predecessor->extra_info)});
|
||||||
if (unmapCallback)
|
if (unmap_callback) {
|
||||||
unmapCallback(virt, size);
|
unmap_callback(virt, size);
|
||||||
|
}
|
||||||
|
|
||||||
return; // The previous block is mapped and ends before
|
// The previous block is mapped and ends before
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Walk the block vector to find the start predecessor as this is more efficient than another
|
// Walk the block vector to find the start predecessor as this is more efficient than another
|
||||||
// binary search in most scenarios
|
// binary search in most scenarios
|
||||||
auto blockStartPredecessor{walkBackToPredecessor(blockEndSuccessor)};
|
auto block_start_predecessor{walk_back_to_predecessor(block_end_successor)};
|
||||||
auto blockStartSuccessor{std::next(blockStartPredecessor)};
|
auto block_start_successor{std::next(block_start_predecessor)};
|
||||||
|
|
||||||
if (blockStartSuccessor->virt > virtEnd) {
|
if (block_start_successor->virt > virt_end) {
|
||||||
UNREACHABLE_MSG("Unsorted block in AS map: virt: 0x{:X}", blockStartSuccessor->virt);
|
UNREACHABLE_MSG("Unsorted block in AS map: virt: 0x{:X}", block_start_successor->virt);
|
||||||
} else if (blockStartSuccessor->virt == virtEnd) {
|
} else if (block_start_successor->virt == virt_end) {
|
||||||
// There are no blocks between the start and the end that would let us skip inserting a new
|
// There are no blocks between the start and the end that would let us skip inserting a new
|
||||||
// one for head
|
// one for head
|
||||||
|
|
||||||
// The previous block is may be unmapped, if so we don't need to insert any unmaps after it
|
// The previous block is may be unmapped, if so we don't need to insert any unmaps after it
|
||||||
if (blockStartPredecessor->Mapped())
|
if (block_start_predecessor->Mapped()) {
|
||||||
blocks.insert(blockStartSuccessor, Block(virt, UnmappedPa, {}));
|
blocks.insert(block_start_successor, Block(virt, UnmappedPa, {}));
|
||||||
} else if (blockStartPredecessor->Unmapped()) {
|
}
|
||||||
|
} else if (block_start_predecessor->Unmapped()) {
|
||||||
// If the previous block is unmapped
|
// If the previous block is unmapped
|
||||||
blocks.erase(blockStartSuccessor, blockEndPredecessor);
|
blocks.erase(block_start_successor, block_end_predecessor);
|
||||||
} else {
|
} else {
|
||||||
// Erase overwritten blocks, skipping the first one as we have written the unmapped start
|
// Erase overwritten blocks, skipping the first one as we have written the unmapped start
|
||||||
// block there
|
// block there
|
||||||
if (auto eraseStart{std::next(blockStartSuccessor)}; eraseStart != blockEndSuccessor)
|
if (auto eraseStart{std::next(block_start_successor)}; eraseStart != block_end_successor) {
|
||||||
blocks.erase(eraseStart, blockEndSuccessor);
|
blocks.erase(eraseStart, block_end_successor);
|
||||||
|
}
|
||||||
|
|
||||||
// Add in the unmapped block header
|
// Add in the unmapped block header
|
||||||
blockStartSuccessor->virt = virt;
|
block_start_successor->virt = virt;
|
||||||
blockStartSuccessor->phys = UnmappedPa;
|
block_start_successor->phys = UnmappedPa;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (unmapCallback)
|
if (unmap_callback)
|
||||||
unmapCallback(virt, size);
|
unmap_callback(virt, size);
|
||||||
}
|
}
|
||||||
|
|
||||||
ALLOC_MEMBER_CONST()::FlatAllocator(VaType vaStart_, VaType vaLimit_)
|
ALLOC_MEMBER_CONST()::FlatAllocator(VaType virt_start_, VaType va_limit_)
|
||||||
: Base(vaLimit_), currentLinearAllocEnd(vaStart_), vaStart(vaStart_) {}
|
: Base{va_limit_}, virt_start{virt_start_}, current_linear_alloc_end{virt_start_} {}
|
||||||
|
|
||||||
ALLOC_MEMBER(VaType)::Allocate(VaType size) {
|
ALLOC_MEMBER(VaType)::Allocate(VaType size) {
|
||||||
std::scoped_lock lock(this->blockMutex);
|
std::scoped_lock lock(this->block_mutex);
|
||||||
|
|
||||||
VaType allocStart{UnmappedVa};
|
VaType alloc_start{UnmappedVa};
|
||||||
VaType allocEnd{currentLinearAllocEnd + size};
|
VaType alloc_end{current_linear_alloc_end + size};
|
||||||
|
|
||||||
// Avoid searching backwards in the address space if possible
|
// Avoid searching backwards in the address space if possible
|
||||||
if (allocEnd >= currentLinearAllocEnd && allocEnd <= this->vaLimit) {
|
if (alloc_end >= current_linear_alloc_end && alloc_end <= this->va_limit) {
|
||||||
auto allocEndSuccessor{
|
auto alloc_end_successor{
|
||||||
std::lower_bound(this->blocks.begin(), this->blocks.end(), allocEnd)};
|
std::lower_bound(this->blocks.begin(), this->blocks.end(), alloc_end)};
|
||||||
if (allocEndSuccessor == this->blocks.begin())
|
if (alloc_end_successor == this->blocks.begin()) {
|
||||||
UNREACHABLE_MSG("First block in AS map is invalid!");
|
UNREACHABLE_MSG("First block in AS map is invalid!");
|
||||||
|
}
|
||||||
|
|
||||||
auto allocEndPredecessor{std::prev(allocEndSuccessor)};
|
auto alloc_end_predecessor{std::prev(alloc_end_successor)};
|
||||||
if (allocEndPredecessor->virt <= currentLinearAllocEnd) {
|
if (alloc_end_predecessor->virt <= current_linear_alloc_end) {
|
||||||
allocStart = currentLinearAllocEnd;
|
alloc_start = current_linear_alloc_end;
|
||||||
} else {
|
} else {
|
||||||
// Skip over fixed any mappings in front of us
|
// Skip over fixed any mappings in front of us
|
||||||
while (allocEndSuccessor != this->blocks.end()) {
|
while (alloc_end_successor != this->blocks.end()) {
|
||||||
if (allocEndSuccessor->virt - allocEndPredecessor->virt < size ||
|
if (alloc_end_successor->virt - alloc_end_predecessor->virt < size ||
|
||||||
allocEndPredecessor->Mapped()) {
|
alloc_end_predecessor->Mapped()) {
|
||||||
allocStart = allocEndPredecessor->virt;
|
alloc_start = alloc_end_predecessor->virt;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
allocEndPredecessor = allocEndSuccessor++;
|
alloc_end_predecessor = alloc_end_successor++;
|
||||||
|
|
||||||
// Use the VA limit to calculate if we can fit in the final block since it has no
|
// Use the VA limit to calculate if we can fit in the final block since it has no
|
||||||
// successor
|
// successor
|
||||||
if (allocEndSuccessor == this->blocks.end()) {
|
if (alloc_end_successor == this->blocks.end()) {
|
||||||
allocEnd = allocEndPredecessor->virt + size;
|
alloc_end = alloc_end_predecessor->virt + size;
|
||||||
|
|
||||||
if (allocEnd >= allocEndPredecessor->virt && allocEnd <= this->vaLimit)
|
if (alloc_end >= alloc_end_predecessor->virt && alloc_end <= this->va_limit) {
|
||||||
allocStart = allocEndPredecessor->virt;
|
alloc_start = alloc_end_predecessor->virt;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (allocStart != UnmappedVa) {
|
if (alloc_start != UnmappedVa) {
|
||||||
currentLinearAllocEnd = allocStart + size;
|
current_linear_alloc_end = alloc_start + size;
|
||||||
} else { // If linear allocation overflows the AS then find a gap
|
} else { // If linear allocation overflows the AS then find a gap
|
||||||
if (this->blocks.size() <= 2)
|
if (this->blocks.size() <= 2) {
|
||||||
UNREACHABLE_MSG("Unexpected allocator state!");
|
UNREACHABLE_MSG("Unexpected allocator state!");
|
||||||
|
|
||||||
auto searchPredecessor{this->blocks.begin()};
|
|
||||||
auto searchSuccessor{std::next(searchPredecessor)};
|
|
||||||
|
|
||||||
while (searchSuccessor != this->blocks.end() &&
|
|
||||||
(searchSuccessor->virt - searchPredecessor->virt < size ||
|
|
||||||
searchPredecessor->Mapped())) {
|
|
||||||
searchPredecessor = searchSuccessor++;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (searchSuccessor != this->blocks.end())
|
auto search_predecessor{this->blocks.begin()};
|
||||||
allocStart = searchPredecessor->virt;
|
auto search_successor{std::next(search_predecessor)};
|
||||||
else
|
|
||||||
|
while (search_successor != this->blocks.end() &&
|
||||||
|
(search_successor->virt - search_predecessor->virt < size ||
|
||||||
|
search_predecessor->Mapped())) {
|
||||||
|
search_predecessor = search_successor++;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (search_successor != this->blocks.end()) {
|
||||||
|
alloc_start = search_predecessor->virt;
|
||||||
|
} else {
|
||||||
return {}; // AS is full
|
return {}; // AS is full
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
this->MapLocked(allocStart, true, size, {});
|
this->MapLocked(alloc_start, true, size, {});
|
||||||
return allocStart;
|
return alloc_start;
|
||||||
}
|
}
|
||||||
|
|
||||||
ALLOC_MEMBER(void)::AllocateFixed(VaType virt, VaType size) {
|
ALLOC_MEMBER(void)::AllocateFixed(VaType virt, VaType size) {
|
||||||
|
|
|
@ -472,16 +472,16 @@ void nvhost_as_gpu::GetVARegionsImpl(IoctlGetVaRegions& params) {
|
||||||
|
|
||||||
params.regions = std::array<VaRegion, 2>{
|
params.regions = std::array<VaRegion, 2>{
|
||||||
VaRegion{
|
VaRegion{
|
||||||
.offset = vm.small_page_allocator->vaStart << VM::PAGE_SIZE_BITS,
|
.offset = vm.small_page_allocator->GetVAStart() << VM::PAGE_SIZE_BITS,
|
||||||
.page_size = VM::PAGE_SIZE,
|
.page_size = VM::PAGE_SIZE,
|
||||||
._pad0_{},
|
._pad0_{},
|
||||||
.pages = vm.small_page_allocator->vaLimit - vm.small_page_allocator->vaStart,
|
.pages = vm.small_page_allocator->GetVALimit() - vm.small_page_allocator->GetVAStart(),
|
||||||
},
|
},
|
||||||
VaRegion{
|
VaRegion{
|
||||||
.offset = vm.big_page_allocator->vaStart << vm.big_page_size_bits,
|
.offset = vm.big_page_allocator->GetVAStart() << vm.big_page_size_bits,
|
||||||
.page_size = vm.big_page_size,
|
.page_size = vm.big_page_size,
|
||||||
._pad0_{},
|
._pad0_{},
|
||||||
.pages = vm.big_page_allocator->vaLimit - vm.big_page_allocator->vaStart,
|
.pages = vm.big_page_allocator->GetVALimit() - vm.big_page_allocator->GetVAStart(),
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
|
@ -669,17 +669,6 @@ Device::Device(VkInstance instance_, vk::PhysicalDevice physical_, VkSurfaceKHR
|
||||||
const bool is_amd =
|
const bool is_amd =
|
||||||
driver_id == VK_DRIVER_ID_AMD_PROPRIETARY || driver_id == VK_DRIVER_ID_AMD_OPEN_SOURCE;
|
driver_id == VK_DRIVER_ID_AMD_PROPRIETARY || driver_id == VK_DRIVER_ID_AMD_OPEN_SOURCE;
|
||||||
if (is_amd) {
|
if (is_amd) {
|
||||||
// TODO(lat9nq): Add an upper bound when AMD fixes their VK_KHR_push_descriptor
|
|
||||||
const bool has_broken_push_descriptor = VK_VERSION_MAJOR(properties.driverVersion) == 2 &&
|
|
||||||
VK_VERSION_MINOR(properties.driverVersion) == 0 &&
|
|
||||||
VK_VERSION_PATCH(properties.driverVersion) >= 226;
|
|
||||||
if (khr_push_descriptor && has_broken_push_descriptor) {
|
|
||||||
LOG_WARNING(
|
|
||||||
Render_Vulkan,
|
|
||||||
"Disabling AMD driver 2.0.226 and later from broken VK_KHR_push_descriptor");
|
|
||||||
khr_push_descriptor = false;
|
|
||||||
}
|
|
||||||
|
|
||||||
// AMD drivers need a higher amount of Sets per Pool in certain circunstances like in XC2.
|
// AMD drivers need a higher amount of Sets per Pool in certain circunstances like in XC2.
|
||||||
sets_per_pool = 96;
|
sets_per_pool = 96;
|
||||||
// Disable VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT on AMD GCN4 and lower as it is broken.
|
// Disable VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT on AMD GCN4 and lower as it is broken.
|
||||||
|
|
Loading…
Reference in New Issue