Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Explorer/Assets/DCL/FeatureFlags/FeatureFlagsStrings.cs
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ public static class FeatureFlagsStrings
public const string OPTIMIZED_ASSETS = "optimized-assets";
public const string OPTIMIZED_ASSETS_BASE_URL_VARIANT = "assets-base-url";
public const string ABGEN_PIPELINE = "alfa-abgen-pipeline";
public const string ABGEN_LODS = "alfa-abgen-lods";
public const string USE_CUSTOM_MEDIA_PLAYER_WINDOWS = "use-custom-media-player-windows";
public const string USE_CUSTOM_MEDIA_PLAYER_MAC_SILICON = "use-custom-media-player-mac-silicon";
public const string USE_CUSTOM_MEDIA_PLAYER_MAC_INTEL = "use-custom-media-player-mac-intel";
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using DCL.Ipfs;
using DCL.Multiplayer.Connections.DecentralandUrls;
using DCL.SceneRunner.Scene;
using ECS.StreamableLoading.Cache.Disk.Cacheables;
using ECS.StreamableLoading.Common.Components;
Expand Down Expand Up @@ -56,14 +57,31 @@ public class DiskHashCompute : AbstractDiskHashCompute<GetISSDescriptorIntention

public static readonly DiskHashCompute INSTANCE = new ();

private readonly IDecentralandUrlsSource? decentralandUrlsSource;

private DiskHashCompute() { }

/// <summary>
/// Keys descriptors per LOD source: an abgen source publishes a different document under the same
/// scene id as production, so the two must not share disk-cache entries. Read at hash time, as the
/// source can depend on feature flags that load after construction.
/// </summary>
public DiskHashCompute(IDecentralandUrlsSource decentralandUrlsSource)
{
this.decentralandUrlsSource = decentralandUrlsSource;
}

protected override void FillPayload(IHashKeyPayload keyPayload, in GetISSDescriptorIntention asset)
{
// Entity hash uniquely identifies the deploy — when the scene re-deploys, sceneId changes
// and the old cache entry becomes inert (LRU evicts it). No need to mix in the manifest version.
keyPayload.Put(asset.SceneId);
keyPayload.Put(ITERATION_NUMBER);

string? lodSource = decentralandUrlsSource?.AbgenLodsCacheKey;

if (lodSource != null)
Comment thread
popuz marked this conversation as resolved.
keyPayload.Put(lodSource);
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,9 @@ protected override async UniTask<StreamableLoadingResult<ISSDescriptorMetadata>>

private async UniTask<ISSDescriptorMetadata?> TryLoadDescriptorAsync(string sceneId, CancellationToken ct)
{
URLAddress url = descriptorBaseUrl.Append(URLPath.FromString($"{DESCRIPTOR_PATH_PREFIX}{sceneId}_InitialSceneState.json"));
// Descriptors are written under a lower-cased scene id and the bucket is case-sensitive, so a
// mixed-case (Qm) id 404s verbatim. The LOD bundle path lower-cases it for the same reason.
URLAddress url = descriptorBaseUrl.Append(URLPath.FromString($"{DESCRIPTOR_PATH_PREFIX}{sceneId.ToLower()}_InitialSceneState.json"));

try
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,14 @@ public static class AppArgsFlags
/// <summary>Presence forces the abgen pipeline on without waiting for the abgen-pipeline feature flag.</summary>
public const string ABGEN_PIPELINE = "abgen-pipeline";

/// <summary>
/// Presence forces LOD bundles and ISS descriptors onto the abgen LOD source (abgen-cdn) without waiting
/// for the abgen-lods feature flag. Pair it with <see cref="ABGEN_PIPELINE" />: abgen publishes no LOD_0,
/// level 0 comes from the ISS path instead, and both the descriptor's assets and the manifest that gates
/// it on v49+ are resolved by the pipeline flip.
/// </summary>
public const string ABGEN_LODS = "abgen-lods";

public const string NO_LIVEKIT_MODE = "no-livekit-mode";

public const string NATIVE_SHUTDOWN_STOPWATCH = "native-shutdown-stopwatch";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -279,6 +279,7 @@ await LODContainer
debugBuilder,
dynamicWorldParams.EnableLOD,
staticContainer.GPUInstancingService,
bootstrapContainer.DecentralandUrlsSource,
cancellationToken
)
.ThrowOnFail();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@ public GlobalWorld Create(ISceneFactory sceneFactory, Entity playerEntity)

LoadISSDescriptorSystem.InjectToWorld(ref builder, webRequestController, lodGeneratorCdnUrl,
new NoCache<ISSDescriptorMetadata, GetISSDescriptorIntention>(false, false),
new DiskCacheOptions<ISSDescriptorMetadata, GetISSDescriptorIntention>(staticContainer.ISSDescriptorDiskCache, GetISSDescriptorIntention.DiskHashCompute.INSTANCE, "iss.json"));
new DiskCacheOptions<ISSDescriptorMetadata, GetISSDescriptorIntention>(staticContainer.ISSDescriptorDiskCache, new GetISSDescriptorIntention.DiskHashCompute(urlsSource), "iss.json"));

// Mutates the entity's ISSDescriptor component (class, ref-shared) in place when the resolver
// promise spawned by ResolveSceneStateByIncreasingRadiusSystem completes. Cached references in
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -340,6 +340,7 @@ private async UniTask InitializeFlowAsync(CancellationToken ct)
applicationParametersParser.TryGetValue(AppArgsFlags.GATEKEEPER_URL, out string? cliGatekeeperUrl);

bool cliAbgenPipeline = applicationParametersParser.HasFlag(AppArgsFlags.ABGEN_PIPELINE);
bool cliAbgenLods = applicationParametersParser.HasFlag(AppArgsFlags.ABGEN_LODS);

// local-ab only: the embedded abgen JIT server becomes the optimized-assets source (it serves the
// local scene and read-throughs everything else from production). Brought up to health serially,
Expand Down Expand Up @@ -372,7 +373,8 @@ private async UniTask InitializeFlowAsync(CancellationToken ct)
localAbBaseUrl,
customBaseDomain,
cliAbgenPipeline,
cliGatewayPrefix);
cliGatewayPrefix,
cliAbgenLods);
DiagnosticInfoUtils.LogEnvironment(decentralandUrlsSource);

var web3AccountFactory = new Web3AccountFactory();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -85,5 +85,13 @@ static bool IsHostWithinDomain(string host, string domain) =>
public string GetOriginalUrl(string url);

string GetHostnameForFeatureFlag();

/// <summary>
/// Non-null while LODs come from an abgen source (see the abgen-lods flag and args). LOD bundles and ISS
/// descriptors are named by scene id only, so every LOD source needs its own cache namespace: consumers
/// fold this into the LOD asset-bundle cache key and the ISS descriptor disk-cache key. Null means the
/// regular pipeline and the historical cache keys.
/// </summary>
string? AbgenLodsCacheKey => null;
}
}
3 changes: 2 additions & 1 deletion Explorer/Assets/DCL/LOD/Systems/LODContainer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ private LODContainer(IAssetsProvisioner assetsProvisioner)
IDebugContainerBuilder debugBuilder,
bool lodEnabled,
GPUInstancingService gpuInstancingService,
IDecentralandUrlsSource decentralandUrlsSource,
CancellationToken ct)
{
var container = new LODContainer(assetsProvisioner);
Expand Down Expand Up @@ -103,7 +104,7 @@ private LODContainer(IAssetsProvisioner assetsProvisioner)
staticContainer.ScenesCache, debugBuilder, staticContainer.SceneReadinessReportQueue,
textureArrayContainerFactory, c.lodSettingsAsset.Value,
staticContainer.RealmPartitionSettings, c.LodCache, lodGroupPool, new GameObject("LOD_CACHE").transform, lodEnabled, LOD_LEVELS,
staticContainer.GltfContainerAssetsCache);
staticContainer.GltfContainerAssetsCache, decentralandUrlsSource);

return UniTask.CompletedTask;
});
Expand Down
7 changes: 5 additions & 2 deletions Explorer/Assets/DCL/LOD/Systems/LODPlugin.cs
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
using DCL.Multiplayer.Connections.DecentralandUrls;
using Arch.SystemGroups;
using DCL.AvatarRendering.AvatarShape.Rendering.TextureArray;
using DCL.DebugUtilities;
Expand Down Expand Up @@ -38,13 +39,14 @@ public class LODPlugin : IDCLGlobalPlugin
private readonly Transform lodCacheParent;

private readonly IGltfContainerAssetsCache containerAssetsCache;
private readonly IDecentralandUrlsSource decentralandUrlsSource;

public LODPlugin(IPerformanceBudget memoryBudget,
IPerformanceBudget frameCapBudget, IScenesCache scenesCache, IDebugContainerBuilder debugBuilder,
ISceneReadinessReportQueue sceneReadinessReportQueue, TextureArrayContainerFactory textureArrayContainerFactory,
ILODSettingsAsset lodSettingsAsset, IRealmPartitionSettings partitionSettings,
ILODCache lodCache, IComponentPool<LODGroup> lodGroupPool, Transform lodCacheParent, bool lodEnabled,
int lodLevels, IGltfContainerAssetsCache containerAssetsCache)
int lodLevels, IGltfContainerAssetsCache containerAssetsCache, IDecentralandUrlsSource decentralandUrlsSource)
{
this.memoryBudget = memoryBudget;
this.frameCapBudget = frameCapBudget;
Expand All @@ -60,6 +62,7 @@ public LODPlugin(IPerformanceBudget memoryBudget,
this.lodCacheParent = lodCacheParent;
this.lodLevels = lodLevels;
this.containerAssetsCache = containerAssetsCache;
this.decentralandUrlsSource = decentralandUrlsSource;
}

public void InjectToWorld(ref ArchSystemsWorldBuilder<Arch.Core.World> builder, in GlobalPluginArguments arguments)
Expand All @@ -79,7 +82,7 @@ public void InjectToWorld(ref ArchSystemsWorldBuilder<Arch.Core.World> builder,
InitializeSceneLODInfoSystem.InjectToWorld(ref builder, lodCache, lodLevels, lodGroupPool,
lodCacheParent, sceneReadinessReportQueue, scenesCache);

UpdateSceneLODInfoSystem.InjectToWorld(ref builder, lodSettingsAsset);
UpdateSceneLODInfoSystem.InjectToWorld(ref builder, lodSettingsAsset, decentralandUrlsSource);
InstantiateSceneLODInfoSystem.InjectToWorld(ref builder, frameCapBudget, memoryBudget, scenesCache, sceneReadinessReportQueue, lodTextureArrayContainer, partitionSettings);
LODDebugToolsSystem.InjectToWorld(ref builder, debugBuilder, lodSettingsAsset, lodLevels);

Expand Down
11 changes: 9 additions & 2 deletions Explorer/Assets/DCL/LOD/Systems/UpdateSceneLODInfoSystem.cs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
using DCL.Diagnostics;
using DCL.Ipfs;
using DCL.LOD.Components;
using DCL.Multiplayer.Connections.DecentralandUrls;
using DCL.Utility;
using ECS.Abstract;
using ECS.LifeCycle.Components;
Expand All @@ -32,12 +33,17 @@ namespace DCL.LOD.Systems
[LogCategory(ReportCategory.LOD)]
public partial class UpdateSceneLODInfoSystem : BaseUnityLoopSystem
{
// Production LODs have always been cached under this pseudo build date; an abgen LOD source gets its own key.
private const string REGULAR_LOD_CACHE_KEY = "dummyDate";

private readonly ILODSettingsAsset lodSettingsAsset;
private readonly IDecentralandUrlsSource? decentralandUrlsSource;
private IReadOnlyList<SceneAssetBundleManifest>? manifestCache;

public UpdateSceneLODInfoSystem(World world, ILODSettingsAsset lodSettingsAsset) : base(world)
public UpdateSceneLODInfoSystem(World world, ILODSettingsAsset lodSettingsAsset, IDecentralandUrlsSource? decentralandUrlsSource = null) : base(world)
{
this.lodSettingsAsset = lodSettingsAsset;
this.decentralandUrlsSource = decentralandUrlsSource;
}

protected override void Update(float t)
Expand Down Expand Up @@ -86,7 +92,8 @@ private void StartLODPromise(ref SceneLODInfo sceneLODInfo, ref PartitionCompone
// descriptor in None state — no ISS for this scene; fall through to legacy LOD.
}

AssetBundleManifestVersion lodManifest = AssetBundleManifestVersion.CreateForLOD($"LOD/{level.ToString()}", "dummyDate");
// LOD files are named by scene id only, so the Unity asset-bundle cache key must carry the LOD source.
AssetBundleManifestVersion lodManifest = AssetBundleManifestVersion.CreateForLOD($"LOD/{level.ToString()}", decentralandUrlsSource?.AbgenLodsCacheKey ?? REGULAR_LOD_CACHE_KEY);

var assetBundleIntention = GetAssetBundleIntention.FromHash(
lodManifest.GetCdnRequestHash($"{sceneDefinitionComponent.Definition.id.ToLower()}_{level.ToString()}"),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using UnityEngine.Pool;

// ReSharper disable once CheckNamespace
Expand Down Expand Up @@ -37,6 +38,12 @@ protected enum CacheBehaviour
// (see GetFeatureFlagsUrl), so the subdomain is shared rather than the whole url.
private const string FEATURE_FLAGS_SUBDOMAIN = "feature-flags";

// An abgen source keeps its LOD generation under a single "LOD/" prefix, beside the asset bundles on the
// same CDN: LOD/{level}/{sceneId}_{level}_{platform} for the bundles, whose "LOD/{level}" comes from the
// manifest version, and LOD/lods-unity/manifests/{sceneId}_InitialSceneState.json for the descriptors,
// whose path carries no such segment — so the descriptor base is the one that takes the prefix.
Comment thread
dalkia marked this conversation as resolved.
private const string ABGEN_LODS_DESCRIPTOR_SUBPATH = "/LOD";

// A base domain feeds host-trust checks, so anything that could turn it into a different authority
// (scheme, userinfo, port, path) is rejected rather than silently accepted.
private static readonly char[] BASE_DOMAIN_FORBIDDEN_CHARS = { '/', ':', '@', '?', '#', ' ', '\t' };
Expand All @@ -48,6 +55,7 @@ protected enum CacheBehaviour
private readonly string? gatekeeperBaseOverride;
private readonly string? localAbBaseOverride;
private readonly bool abgenPipelineForced;
private readonly bool abgenLodsForced;

/// <summary>
/// The domain <see cref="RawUrl" /> composes every host from.
Expand All @@ -63,7 +71,8 @@ public DecentralandUrlsSource(
string? cliGatekeeperUrl = null,
string? localAbBaseUrl = null,
string? customBaseDomain = null,
bool abgenPipelineForced = false)
bool abgenPipelineForced = false,
bool abgenLodsForced = false)
{
this.environment = environment;
BaseDomain = ResolveBaseDomain(environment, customBaseDomain);
Expand All @@ -73,6 +82,7 @@ public DecentralandUrlsSource(
ReportHub.Log(ReportCategory.STARTUP, $"Gatekeeper base override: {gatekeeperBaseOverride ?? "(default)"} (source: {source})");
localAbBaseOverride = localAbBaseUrl?.TrimEnd('/');
this.abgenPipelineForced = abgenPipelineForced;
this.abgenLodsForced = abgenLodsForced;

realmData.RealmType.OnUpdate += ResetRealmDependentUrls;
}
Expand Down Expand Up @@ -256,6 +266,54 @@ private UrlData ResolveAbgenPipelineUrl(string regularHost, string abgenHost)
: regularHost;
}

/// <summary>
/// The abgen LOD source. LOD bundles and ISS descriptors flip together: the descriptors and the bundles
/// describe one generation, as the abgen registry and abgen-cdn do for asset bundles. The "--abgen-lods"
/// arg forces it on without the flag, exactly as "--abgen-pipeline" does for the asset bundles.
/// <paramref name="abgenSubPath" /> is appended to the abgen host and left off the regular one, which
/// lays its LOD generation out differently (see <see cref="ABGEN_LODS_DESCRIPTOR_SUBPATH" />).
/// FeatureFlagsDependent for the same reasons as <see cref="ResolveAbgenPipelineUrl" />.
/// </summary>
private UrlData ResolveAbgenLodsUrl(UrlData regularHost, string abgenSubPath)
{
string abgenHost = $"https://abgen-cdn.{BaseDomain}{abgenSubPath}";

if (abgenLodsForced)
return new UrlData(CacheBehaviour.FeatureFlagsDependent, abgenHost);

FeatureFlagsConfiguration featureFlags = FeatureFlagsConfiguration.Instance;

if (featureFlags.IsEmpty)
return new UrlData(CacheBehaviour.FeatureFlagsDependent, regularHost.Url!);

return featureFlags.IsEnabled(FeatureFlagsStrings.ABGEN_LODS)
? new UrlData(CacheBehaviour.FeatureFlagsDependent, abgenHost)
: regularHost;
}

private bool AbgenLodsActive =>
abgenLodsForced
|| (!FeatureFlagsConfiguration.Instance.IsEmpty && FeatureFlagsConfiguration.Instance.IsEnabled(FeatureFlagsStrings.ABGEN_LODS));

/// <inheritdoc />
public string? AbgenLodsCacheKey
{
get
{
if (!AbgenLodsActive)
return null;

// The resolved base identifies the source; sanitised so it can travel in file names and hash payloads.
string baseUrl = Url(DecentralandUrl.LodAssetBundlesCDN);
var key = new StringBuilder("abgen-lods-", baseUrl.Length + 11);

foreach (char c in baseUrl)
key.Append(char.IsLetterOrDigit(c) ? c : '-');

return key.ToString();
}
}

/// <summary>
/// Composes the feature-flags host before any instance exists (the pre-login whitelist fetch), from the
/// same <see cref="ResolveBaseDomain" /> decision the instance uses, so the two cannot diverge.
Expand Down Expand Up @@ -318,9 +376,12 @@ protected virtual UrlData RawUrl(DecentralandUrl decentralandUrl) =>
DecentralandUrl.Market => $"https://market.{BaseDomain}",
DecentralandUrl.AssetBundlesCDN => ResolveOptimizedAssetsUrl(ResolveAbgenPipelineUrl($"https://ab-cdn.{BaseDomain}", $"https://abgen-cdn.{BaseDomain}")),

// LOD bundles are only produced by the regular pipeline, so they never follow the abgen flip
DecentralandUrl.LodAssetBundlesCDN => ResolveOptimizedAssetsUrl($"https://ab-cdn.{BaseDomain}"),
DecentralandUrl.LodGeneratorCDN => ResolveOptimizedAssetsUrl($"https://lod-generator-unity-cdn.{BaseDomain}"),
// LOD bundles ({base}/LOD/{level}/{sceneId}_{level}_{platform}) and ISS descriptors
// ({base}/lods-unity/manifests/{sceneId}_InitialSceneState.json) come from the regular pipeline unless the
// abgen-lods flip moves both onto an abgen source, which nests them under ABGEN_LODS_DESCRIPTOR_SUBPATH;
// they never follow the asset-bundle abgen flip on their own.
DecentralandUrl.LodAssetBundlesCDN => ResolveAbgenLodsUrl(ResolveOptimizedAssetsUrl($"https://ab-cdn.{BaseDomain}"), string.Empty),
DecentralandUrl.LodGeneratorCDN => ResolveAbgenLodsUrl(ResolveOptimizedAssetsUrl($"https://lod-generator-unity-cdn.{BaseDomain}"), ABGEN_LODS_DESCRIPTOR_SUBPATH),
DecentralandUrl.ArchipelagoStatus => $"https://archipelago-ea-stats.{BaseDomain}/status",
DecentralandUrl.ArchipelagoHotScenes => $"https://archipelago-ea-stats.{BaseDomain}/hot-scenes",
DecentralandUrl.GatekeeperStatus => $"{RawUrl(DecentralandUrl.Gatekeeper).Url!}/status",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -119,8 +119,9 @@ public GatewayUrlsSource(
string? localAbBaseUrl = null,
string? customBaseDomain = null,
bool abgenPipelineForced = false,
string? cliGatewayPrefix = null)
: base(environment, realmData, launchMode, gatekeeperMode, customGatekeeperUrl, cliGatekeeperUrl, localAbBaseUrl, customBaseDomain, abgenPipelineForced)
string? cliGatewayPrefix = null,
bool abgenLodsForced = false)
: base(environment, realmData, launchMode, gatekeeperMode, customGatekeeperUrl, cliGatekeeperUrl, localAbBaseUrl, customBaseDomain, abgenPipelineForced, abgenLodsForced)
{
this.cliGatewayPrefix = cliGatewayPrefix;

Expand Down
Loading
Loading