From 90dbfa26500a08b908a98a8c7e911030af7d306a Mon Sep 17 00:00:00 2001 From: vazois Date: Mon, 13 May 2024 19:52:56 -0700 Subject: [PATCH] add code snippet reference to github code --- website/docs/dev/cluster/migration.md | 204 ++------------------------ website/docs/dev/cluster/sharding.md | 189 ++---------------------- website/docusaurus.config.js | 11 +- website/package.json | 1 + 4 files changed, 38 insertions(+), 367 deletions(-) diff --git a/website/docs/dev/cluster/migration.md b/website/docs/dev/cluster/migration.md index 3fe3f4ca9cf..905a3529386 100644 --- a/website/docs/dev/cluster/migration.md +++ b/website/docs/dev/cluster/migration.md @@ -72,73 +72,9 @@ On success of updating the local config command, the processing thread of the pr This series of steps ensures that on return, the processing thread guarantees to the command issuer (or the method caller if state transition happens internally), that the state transition is visible to all threads that were active. Therefore, any active client session will process subsequent commands considering the new slot state. -
- Spin-Wait Config Transition - ```bash - . - . - . - clusterSession?.AcquireCurrentEpoch(); - . - . - . - - case SlotState.MIGRATING: - setSlotsSucceeded = clusterProvider.clusterManager.TryPrepareSlotForMigration(slot, nodeid, out errorMessage); - break; - - . - . - . - - if (setSlotsSucceeded) - { - UnsafeWaitForConfigTransition(); - - while (!RespWriteUtils.WriteDirect(CmdStrings.RESP_OK, ref dcurr, dend)) - SendAndReset(); - } - - . - . - . - - /// - /// Release epoch, wait for config transition and re-acquire the epoch - /// - public void UnsafeWaitForConfigTransition() - { - ReleaseCurrentEpoch(); - clusterProvider.WaitForConfigTransition(); - AcquireCurrentEpoch(); - } - - /// - /// Wait for config transition - /// - /// - internal bool WaitForConfigTransition() - { - var server = storeWrapper.GetServer(); - BumpCurrentEpoch(); - while (true) - { - retry: - var currentEpoch = GarnetCurrentEpoch; - Thread.Yield(); - var sessions = server.ActiveClusterSessions(); - foreach (var s in sessions) - { - var entryEpoch = s.LocalCurrentEpoch; - if (entryEpoch != 0 && entryEpoch >= currentEpoch) - goto retry; - } - break; - } - return true; - } - ``` -
+```csharp reference title="Wait for Config" +https://github.com/microsoft/garnet/blob/951cf82c120d4069e940e832db03bfa018c688ea/libs/cluster/Server/ClusterProvider.cs#L271-L296 +``` During migration, the change in slot state (i.e., ```MIGRATING```) is transient from the perspective of the source node. This means that until migration completes the slot is still owned by the source node. @@ -147,33 +83,13 @@ Therefore, depending on the context of the operation being executed, the actual For example, the ```CLUSTER NODES``` will use *workerId* property (through GetSlotRange(*workerId*)) since it has to return actual owner of the node even during migration. At the same, time it needs to return all nodes that are in ```MIGRATING``` or ```IMPORTING``` state and the node-id associated with that state which can be done by inspecting the *_workerId* variable (through GetSpecialStates(*workerId*)). -
- HashSlot Definition - ```bash - /// - /// Get formatted (using CLUSTER NODES format) worker info. - /// - /// Offset of worker in the worker list. - /// Formatted string. - public string GetNodeInfo(ushort workerId) - { - return $"{workers[workerId].Nodeid} " + - $"{workers[workerId].Address}:{workers[workerId].Port}@{workers[workerId].Port + 10000},{workers[workerId].hostname} " + - $"{(workerId == 1 ? "myself," : "")}{(workers[workerId].Role == NodeRole.PRIMARY ? "master" : "slave")} " + - $"{(workers[workerId].Role == NodeRole.REPLICA ? workers[workerId].ReplicaOfNodeId : "-")} " + - $"0 " + - $"0 " + - $"{workers[workerId].ConfigEpoch} " + - $"connected" + - $"{GetSlotRange(workerId)}" + - $"{GetSpecialStates(workerId)}\n"; - } - ``` -
+```csharp reference title="CLUSTER NODES Implementation" +https://github.com/microsoft/garnet/blob/951cf82c120d4069e940e832db03bfa018c688ea/libs/cluster/Server/ClusterConfig.cs#L499-L521 +``` ## Migrate KEYS Implementation Details -Using the ```KEYS`` option, Garnet will iterate through the provided list of keys and migrate them in batches to the target node. +Using the ```KEYS``` option, Garnet will iterate through the provided list of keys and migrate them in batches to the target node. When using this option, the issuer of the migration command will have to make sure that the slot state is set appropriately in the source, and target node. In addition, the issuer has to provide all keys that map to a specific slot either in one call to ```MIGRATE``` or across multiple call before the migration completes. When all key-value pairs have migrated to the target node, the issuer has to reset the slot state and assign ownership of the slot to the new node. @@ -187,33 +103,9 @@ It is possible that a given key cannot be retrieved from either store, because i In that case, execution proceeds to the next available key and no specific error is raised. When data transmission completes, and depending if COPY option is enabled, ```MigrateKeys``` deletes the keys from the both stores. -
- Migrate KEYS methods - ```bash - /// - /// Method used to migrate individual keys from main store to target node. - /// Used for MIGRATE KEYS option - /// - /// List of pairs of address to the network receive buffer, key size - /// Output keys not found in main store so we can scan the object store next - /// True on success, false otherwise - private bool MigrateKeysFromMainStore(ref List<(long, long)> keysWithSize, out List<(long, long)> objectStoreKeys); - - /// - /// Method used to migrate individual keys from object store to target node. - /// Used for MIGRATE KEYS option - /// - /// List of pairs of address to the network receive buffer, key size that were not found in main store - /// True on success, false otherwise - private bool MigrateKeysFromObjectStore(ref List<(long, long)> objectStoreKeys); - - /// - /// Method used to migrate keys from main and object stores. - /// Used for MIGRATE KEYS option - /// - public bool MigrateKeys(); - ``` -
+```csharp reference title="MigrateKeys Main Method" +https://github.com/microsoft/garnet/blob/951cf82c120d4069e940e832db03bfa018c688ea/libs/cluster/Server/Migration/MigrateSessionKeys.cs#L146-L169 +``` ## Migrate SLOTS Details @@ -234,76 +126,6 @@ The slot ownership exchange becomes visible to the whole cluster by bumping the Finally, the source node will issue ```CLUSTER SETSLOT NODE``` to the target node to explicitly make it an owner of the corresponding slot collection. This last step is not necessary and it is used only to speed up the config propagation. -
- Migrate SLOTS Task - ```bash - /// - /// Migrate slots session background task - /// - private void BeginAsyncMigrationTask() - { - //1. Set target node to import state - if (!TrySetSlotRanges(GetSourceNodeId, MigrateState.IMPORT)) - { - logger?.LogError("Failed to set remote slots {slots} to import state", string.Join(',', GetSlots)); - TryRecoverFromFailure(); - Status = MigrateState.FAIL; - return; - } - - #region transitionLocalSlotToMigratingState - //2. Set source node to migrating state and wait for local threads to see changed state. - if (!TryPrepareLocalForMigration()) - { - logger?.LogError("Failed to set local slots {slots} to migrate state", string.Join(',', GetSlots)); - TryRecoverFromFailure(); - Status = MigrateState.FAIL; - return; - } - - if (!clusterProvider.WaitForConfigTransition()) return; - #endregion - - #region migrateData - //3. Migrate actual data - if (!MigrateSlotsDataDriver()) - { - logger?.LogError($"MigrateSlotsDriver failed"); - TryRecoverFromFailure(); - Status = MigrateState.FAIL; - return; - } - #endregion - #region migrateData - //3. Migrate actual data - if (!MigrateSlotsDataDriver()) - { - logger?.LogError($"MigrateSlotsDriver failed"); - TryRecoverFromFailure(); - Status = MigrateState.FAIL; - return; - } - #endregion - - #region transferSlotOwnnershipToTargetNode - //5. Clear local migration set. - if (!RelinquishOwnership()) - { - logger?.LogError($"Failed to relinquish ownerhsip to target node"); - TryRecoverFromFailure(); - Status = MigrateState.FAIL; - return; - } - - //6. Change ownership of slots to target node. - if (!TrySetSlotRanges(GetTargetNodeId, MigrateState.NODE)) - { - logger?.LogError($"Failed to assign ownerhsip to target node"); - TryRecoverFromFailure(); - Status = MigrateState.FAIL; - return; - } - #endregion - } - ``` -
\ No newline at end of file +```csharp reference title="MigrateSlots Background Task" +https://github.com/microsoft/garnet/blob/951cf82c120d4069e940e832db03bfa018c688ea/libs/cluster/Server/Migration/MigrationDriver.cs#L54-L126 +``` \ No newline at end of file diff --git a/website/docs/dev/cluster/sharding.md b/website/docs/dev/cluster/sharding.md index 8f8efe282c5..6c4b41aae1f 100644 --- a/website/docs/dev/cluster/sharding.md +++ b/website/docs/dev/cluster/sharding.md @@ -19,21 +19,9 @@ It can only change its current role only after receiving a ```CLUSTER REPLICATE` We follow this constrain to avoid having to deal with cluster misconfiguration in the event of network partitions. This convention also extends to slot assignment, which is managed through direct requests to cluster instances made using the ```CLUSTER [ADDSLOTS|DELSLOTS]``` and ```CLUSTER [ADDSLOTSRANGE|DELSLOTSRANGE]``` commands. -
- ClusterConfig Definition - ```bash - /// - /// Cluster configuration - /// - internal sealed partial class ClusterConfig - { - ... - readonly HashSlot[] slotMap; - readonly Worker[] workers; - ... - } - ``` -
+```csharp reference title="Hashlot & Worker Array Declaration" +https://github.com/microsoft/garnet/blob/8856dc3990fb0863141cb902bbf64c13202d5f85/libs/cluster/Server/ClusterConfig.cs#L16-L42 +``` Initially, the cluster nodes are empty, taking the role of a **PRIMARY**, having no assigned slots and with no knowledge of any other node in the cluster. The local node contains information only about itself stored at workers[1] while workers[0] is reserved for special use to indicate unassigned slots. @@ -41,147 +29,27 @@ Garnet cluster nodes are connected to each other through the ```CLUSTER MEET``` This message forces a remote node to add the sender to its list of trusted nodes. Remote nodes are stored in any order starting from workers[2]. -
- Worker Definition - ```bash - /// - /// Cluster worker definition - /// - public struct Worker - { - /// - /// Unique node ID - /// - public string nodeid; - - /// - /// IP address - /// - public string address; - - /// - /// Port - /// - public int port; - - /// - /// Configuration epoch. - /// - public long configEpoch; - - /// - /// Current config epoch used for voting. - /// - public long currentConfigEpoch; - - /// - /// Last config epoch this worker has voted for. - /// - public long lastVotedConfigEpoch; - - /// - /// Role of node (i.e 0: primary 1: replica). - /// - public NodeRole role; - - /// - /// Node ID that this node is replicating (i.e. primary id). - /// - public string replicaOfNodeId; - - /// - /// Replication offset (readonly value for information only) - /// - public long replicationOffset; - - /// - /// Hostname of this instance - /// - public string hostname; - - /// - /// ToString - /// - /// - public override string ToString() => $"{nodeid} {address} {port} {configEpoch} {role} {replicaOfNodeId}"; - } - ``` -
+```csharp reference title="Worker Definition" +https://github.com/microsoft/garnet/blob/951cf82c120d4069e940e832db03bfa018c688ea/libs/cluster/Server/Worker.cs#L28-L85 +``` Information about the individual slot assignment is captured within the configuration object using an array of HashSlot struct type. It maintains information about the slot state and corresponding owner. The slot owner is represented using the offset in the local copy of the workers array. The slot state is used to determine how to serve requests for keys that map to the relevant slot. -
- HashSlot Definition - ```bash - /// - /// Hashslot info - /// - [StructLayout(LayoutKind.Explicit)] - public struct HashSlot - { - /// - /// WorkerId of slot owner. - /// - [FieldOffset(0)] - public ushort _workerId; - - /// - /// State of this slot. - /// - [FieldOffset(2)] - public SlotState _state; - - /// - /// Slot in migrating state points to target node though still owned by local node until migration completes. - /// - public ushort workerId => _state == SlotState.MIGRATING ? (ushort)1 : _workerId; - } - ``` -
+```csharp reference title="HashSlot Definition" +https://github.com/microsoft/garnet/blob/951cf82c120d4069e940e832db03bfa018c688ea/libs/cluster/Server/HashSlot.cs#L43-L61 +``` At cluster startup slots are are unassigned, hence their initial state is set to **OFFLINE** and workerId to 0. When a slot is assigned to a specific node, its state is set to **STABLE** and workerId (from the perspective of the local configuration copy) to the corresponding offset of the owner node in workers array. Owners of a slot can perform read/write and migrate operations on the data associated with that specific slot. Replicas can serve read requests only for keys mapped to slots owned by their primary. -
- SlotState Definition - ```bash - /// - /// NodeRole identifier - /// - public enum SlotState : byte - { - /// - /// Slot not assigned - /// - OFFLINE = 0x0, - /// - /// Slot assigned and ready to be used. - /// - STABLE, - /// - /// Slot is being moved to another node. - /// - MIGRATING, - /// - /// Reverse of migrating, preparing node to receive commands for that slot. - /// - IMPORTING, - /// - /// Slot in FAIL state. - /// - FAIL, - /// - /// - /// - NODE, - } - ``` -
+```csharp reference title="SlotState Definition" +https://github.com/microsoft/garnet/blob/951cf82c120d4069e940e832db03bfa018c688ea/libs/cluster/Server/HashSlot.cs#L11-L37 +``` ### Configuration Update Propagation @@ -206,36 +74,9 @@ Slot verification involves inspecting the key or keys associated with a given co Garnet primary nodes can serve *read* and *read-write* requests for slots that they own, while Garnet replica nodes can only serve read requests for slots that their primary owns. On failure of the slot verification step, the corresponding command will not be processed and the slot verification method will write a redirection message directly to the network buffer. -
- Slot Verification Methods - ```bash - /// - /// Single key slot verify (check only, do not write result to network) - /// - unsafe bool CheckSingleKeySlotVerify(ArgSlice keySlice, bool readOnly, byte SessionAsking); - - /// - /// Key array slot verify (write result to network) - /// - unsafe bool NetworkKeyArraySlotVerify(int keyCount, ref byte* ptr, byte* endPtr, bool interleavedKeys, bool readOnly, byte SessionAsking, ref byte* dcurr, ref byte* dend, out bool retVal); - - /// - /// Key array slot verify (write result to network) - /// - unsafe bool NetworkKeyArraySlotVerify(ref ArgSlice[] keys, bool readOnly, byte SessionAsking, ref byte* dcurr, ref byte* dend, int count = -1); - - /// - /// Single key slot verify (write result to network) - /// - unsafe bool NetworkSingleKeySlotVerify(byte[] key, bool readOnly, byte SessionAsking, ref byte* dcurr, ref byte* dend); - - /// - /// Single key slot verify (write result to network) - /// - unsafe bool NetworkSingleKeySlotVerify(ArgSlice keySlice, bool readOnly, byte SessionAsking, ref byte* dcurr, ref byte* dend); - ``` -
- +```csharp reference title="Slot Verification Methods" +https://github.com/microsoft/garnet/blob/951cf82c120d4069e940e832db03bfa018c688ea/libs/server/Cluster/IClusterSession.cs#L47-L67 +``` ## Redirection Messages From the perspective of a single node, any requests for keys mapping towards an unassigned slot will result in ```-CLUSTERDOWN Hashlot not served``` message. diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index d991aba04db..0ac01801923 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -148,6 +148,7 @@ const config = {

`, }, prism: { + additionalLanguages: ['csharp'], theme: prismThemes.github, darkTheme: prismThemes.dracula, }, @@ -159,9 +160,15 @@ const config = { clarity: { ID: "loh6v65ww5", }, + // github codeblock theme configuration + codeblock: { + showGithubLink: true, + githubLinkLabel: 'View on GitHub', + showRunmeLink: false, + runmeLinkLabel: 'Checkout via Runme' + }, }), - - themes: ['@docusaurus/theme-mermaid'], + themes: ['@docusaurus/theme-mermaid', 'docusaurus-theme-github-codeblock'], }; export default config; diff --git a/website/package.json b/website/package.json index 94c852d462a..6f41c87b383 100644 --- a/website/package.json +++ b/website/package.json @@ -20,6 +20,7 @@ "@mdx-js/react": "^3.0.0", "clsx": "^1.2.1", "docusaurus-plugin-clarity": "^2.1.0", + "docusaurus-theme-github-codeblock": "^2.0.2", "prism-react-renderer": "^2.1.0", "react": "^18.0.0", "react-dom": "^18.0.0"