Skip to content

Commit

Permalink
"Refactor AI and improve Snake game behavior"
Browse files Browse the repository at this point in the history 
This commit features various changes in the SnakeAgent and SpriteBoardDisplay scripts that refine the behavior of the Snake game. The snake's movement is now defined by rotating in response to action, rather than using explicit directions, allowing for smoother and more logical gameplay. The agent's heuristic function has been adjusted to match this new movement style.

The behavior has also been enhanced by reducing the VectorObservationSize to optimize performance, and the game board size has been reduced to 8x8 for simpler and quicker gameplay during testing.

An unused ONNX file (used for ML model storage) was also removed, which cleaned up the project and reduced unnecessary files.

In SpriteBoardDisplay, collected rewards are visualized to offer better insight on the agent's performance. The calculation for rewards, based on food proximity, has been simplified and the display now includes a reward gradient.
  • Loading branch information
@Pristar4
Pristar4 committed Jul 11, 2023
1 parent 06e785f commit 88f850b
Show file tree
Hide file tree
Showing 9 changed files with 722 additions and 383 deletions.
2 changes: 1 addition & 1 deletion Assets/ML-Agents/Timers/AI_timers.json
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"count":1,"self":15.8643904,"total":16.2532724,"children":{"InitializeActuators":{"count":4,"self":0.0005043,"total":0.0005043,"children":null},"InitializeSensors":{"count":4,"self":0.0015023,"total":0.0015023,"children":null},"AgentSendState":{"count":118,"self":0.0016319,"total":0.033648,"children":{"CollectObservations":{"count":472,"self":0.0283876,"total":0.0283876,"children":null},"WriteActionMask":{"count":472,"self":0.0005137,"total":0.0005137,"children":null},"RequestDecision":{"count":472,"self":0.0031148,"total":0.0031148,"children":null}}},"DecideAction":{"count":118,"self":0.0668677,"total":0.0668677,"children":null},"AgentAct":{"count":118,"self":0.2863591,"total":0.2863591,"children":null}},"gauges":{"SnakeAi.CumulativeReward":{"count":9,"max":-10,"min":-10,"runningAverage":-10,"value":-10,"weightedAverage":-10}},"metadata":{"timer_format_version":"0.1.0","start_time_seconds":"1689076069","unity_version":"2023.1.3f1","command_line_arguments":"C:\\Program Files\\Unity\\Hub\\Editor\\2023.1.3f1\\Editor\\Unity.exe -projectpath C:\\Users\\felix\\SnakeGame -useHub -hubIPC -cloudEnvironment production","communication_protocol_version":"1.5.0","com.unity.ml-agents_version":"2.3.0-exp.3","scene_name":"AI","end_time_seconds":"1689076086"}}
{"count":1,"self":40.0425568,"total":40.238061599999995,"children":{"InitializeActuators":{"count":16,"self":0.0020276,"total":0.0020276,"children":null},"InitializeSensors":{"count":16,"self":0.0005068,"total":0.0005068,"children":null},"AgentSendState":{"count":60,"self":0.0030371,"total":0.070967099999999991,"children":{"CollectObservations":{"count":480,"self":0.0035174,"total":0.0035174,"children":null},"WriteActionMask":{"count":480,"self":0,"total":0,"children":null},"RequestDecision":{"count":480,"self":0.0644126,"total":0.0644126,"children":null}}},"DecideAction":{"count":60,"self":0.045922,"total":0.045922,"children":null},"AgentAct":{"count":60,"self":0.076081799999999991,"total":0.076081799999999991,"children":null}},"gauges":{"SnakeAi.CumulativeReward":{"count":106,"max":-0.003001213,"min":-10.008,"runningAverage":-9.815557,"value":-10.005,"weightedAverage":-10.0048523}},"metadata":{"timer_format_version":"0.1.0","start_time_seconds":"1689087858","unity_version":"2023.1.3f1","command_line_arguments":"C:\\Program Files\\Unity\\Hub\\Editor\\2023.1.3f1\\Editor\\Unity.exe -projectpath C:\\Users\\felix\\SnakeGame -useHub -hubIPC -cloudEnvironment production","communication_protocol_version":"1.5.0","com.unity.ml-agents_version":"2.3.0-exp.3","scene_name":"AI","end_time_seconds":"1689087899"}}
Binary file added Assets/SnakeAi.onnx
View file
Binary file not shown.
2 changes: 1 addition & 1 deletion Assets/SnakeGame/_Levels/SnakeAi.onnx.meta → Assets/SnakeAi.onnx.meta
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

37 changes: 19 additions & 18 deletions Assets/SnakeGame/Prefabs/Env.prefab
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7288,7 +7288,7 @@ RectTransform:
m_AnchorMin: {x: 0, y: 0}
m_AnchorMax: {x: 0, y: 0}
m_AnchoredPosition: {x: 0, y: 0}
m_SizeDelta: {x: 282.0294, y: 100}
m_SizeDelta: {x: 10, y: 5}
m_Pivot: {x: 0.5, y: 0.5}
--- !u!222 &946531939060861097
CanvasRenderer:
Expand Down Expand Up @@ -7345,11 +7345,11 @@ MonoBehaviour:
m_faceColor:
serializedVersion: 2
rgba: 4294967295
m_fontSize: 44.1
m_fontSize: 2.2
m_fontSizeBase: 36
m_fontWeight: 400
m_enableAutoSizing: 1
m_fontSizeMin: 18
m_fontSizeMin: 1
m_fontSizeMax: 72
m_fontStyle: 0
m_HorizontalAlignment: 2
Expand Down Expand Up @@ -29664,9 +29664,9 @@ RectTransform:
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 4131823533888670791}
m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
m_LocalRotation: {x: -0, y: -0, z: -0, w: 1}
m_LocalPosition: {x: 0, y: 0, z: 0}
m_LocalScale: {x: 0, y: 0, z: 0}
m_LocalScale: {x: 1, y: 1, z: 1}
m_ConstrainProportionsScale: 0
m_Children:
- {fileID: 446629320299934005}
Expand All @@ -29675,9 +29675,9 @@ RectTransform:
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
m_AnchorMin: {x: 0, y: 0}
m_AnchorMax: {x: 0, y: 0}
m_AnchoredPosition: {x: 0, y: 0}
m_SizeDelta: {x: 0, y: 0}
m_Pivot: {x: 0, y: 0}
m_AnchoredPosition: {x: 10, y: 0}
m_SizeDelta: {x: 10, y: 10}
m_Pivot: {x: 0.5, y: 0.5}
--- !u!223 &1096734699123948553
Canvas:
m_ObjectHideFlags: 0
Expand All @@ -29687,7 +29687,7 @@ Canvas:
m_GameObject: {fileID: 4131823533888670791}
m_Enabled: 1
serializedVersion: 3
m_RenderMode: 0
m_RenderMode: 2
m_Camera: {fileID: 0}
m_PlaneDistance: 100
m_PixelPerfect: 0
Expand All @@ -29714,7 +29714,7 @@ MonoBehaviour:
m_Name:
m_EditorClassIdentifier:
m_UiScaleMode: 0
m_ReferencePixelsPerUnit: 100
m_ReferencePixelsPerUnit: 1
m_ScaleFactor: 1
m_ReferenceResolution: {x: 800, y: 600}
m_ScreenMatchMode: 0
Expand All @@ -29723,7 +29723,7 @@ MonoBehaviour:
m_FallbackScreenDPI: 96
m_DefaultSpriteDPI: 96
m_DynamicPixelsPerUnit: 1
m_PresetInfoIsWorld: 0
m_PresetInfoIsWorld: 1
--- !u!114 &6765205610209944106
MonoBehaviour:
m_ObjectHideFlags: 0
Expand Down Expand Up @@ -54561,7 +54561,7 @@ Transform:
m_GameObject: {fileID: 8055136750122848586}
m_LocalRotation: {x: -0, y: -0, z: -0, w: 1}
m_LocalPosition: {x: 0, y: 0, z: 0}
m_LocalScale: {x: 3, y: 3, z: 3}
m_LocalScale: {x: 1, y: 1, z: 1}
m_ConstrainProportionsScale: 1
m_Children: []
m_Father: {fileID: 5044792707948238565}
Expand Down Expand Up @@ -54626,6 +54626,7 @@ MonoBehaviour:
player9Material: {fileID: 2100000, guid: bf1fa418dbc388449911fc305701e03e, type: 2}
player10Material: {fileID: 2100000, guid: 6179b896aa70d5846a21dcaea752a7f8, type: 2}
snakeDirectionPrefab: {fileID: 3901235777154034643, guid: d5a3e5a39e2c8944197dd94ce0c7065e, type: 3}
isShowingReward: 0
--- !u!1 &8064525913032508700
GameObject:
m_ObjectHideFlags: 0
Expand Down Expand Up @@ -57308,7 +57309,7 @@ RectTransform:
m_AnchorMin: {x: 0, y: 0}
m_AnchorMax: {x: 0, y: 0}
m_AnchoredPosition: {x: 0, y: 0}
m_SizeDelta: {x: 282.0294, y: 100}
m_SizeDelta: {x: 10, y: 5}
m_Pivot: {x: 0.5, y: 0.5}
--- !u!222 &622147202083944118
CanvasRenderer:
Expand Down Expand Up @@ -57365,11 +57366,11 @@ MonoBehaviour:
m_faceColor:
serializedVersion: 2
rgba: 4294967295
m_fontSize: 46.45
m_fontSize: 2.2
m_fontSizeBase: 36
m_fontWeight: 400
m_enableAutoSizing: 1
m_fontSizeMin: 18
m_fontSizeMin: 1
m_fontSizeMax: 72
m_fontStyle: 0
m_HorizontalAlignment: 2
Expand Down Expand Up @@ -59295,7 +59296,7 @@ RectTransform:
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 8579210737594092273}
m_LocalRotation: {x: -0, y: -0, z: -0, w: 1}
m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
m_LocalPosition: {x: 0, y: 0, z: 0}
m_LocalScale: {x: 1, y: 1, z: 1}
m_ConstrainProportionsScale: 0
Expand All @@ -59307,8 +59308,8 @@ RectTransform:
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
m_AnchorMin: {x: 0, y: 0}
m_AnchorMax: {x: 1, y: 1}
m_AnchoredPosition: {x: -813.18835, y: 231.44495}
m_SizeDelta: {x: -1626.367, y: -758.7516}
m_AnchoredPosition: {x: 0, y: 0}
m_SizeDelta: {x: 0, y: 0}
m_Pivot: {x: 0.5, y: 0.5}
--- !u!222 &8923728759590459196
CanvasRenderer:
Expand Down
6 changes: 3 additions & 3 deletions Assets/SnakeGame/Prefabs/SnakeAgent.prefab
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,8 @@ MonoBehaviour:
hasUpgradedFromAgentParameters: 1
MaxStep: 0
boardDisplay: {fileID: 0}
width: 25
height: 25
width: 8
height: 8
players:
- snakeId: 0
inputSchemer: {fileID: 11400000, guid: 892832864e8092948b7a24eab06bfcad, type: 2}
Expand All @@ -78,7 +78,7 @@ MonoBehaviour:
m_Name:
m_EditorClassIdentifier:
m_BrainParameters:
VectorObservationSize: 630
VectorObservationSize: 71
NumStackedVectorObservations: 1
m_ActionSpec:
m_NumContinuousActions: 0
Expand Down
69 changes: 51 additions & 18 deletions Assets/SnakeGame/Scripts/SnakeAgent.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,9 @@ public class SnakeAgent : Agent {
[SerializeField] private int numberOfSnakes = 1;
[SerializeField] private int startSize = 1;
[SerializeField] private bool isDisplayOn;

#endregion

private float currentReward;


Expand All @@ -37,7 +39,7 @@ public class SnakeAgent : Agent {
new(0, -1),
new(-1, 0),
};
private Vector2Int _lastInputDirection = Vector2Int.up; // keeps track of the last input direction
private Vector2Int _inputDirection = Vector2Int.up; // keeps track of the last input direction

private float _currentTimer;
private int _highScore;
Expand Down Expand Up @@ -68,14 +70,15 @@ private void Update() {
tempDirection = new Vector2Int(-1, 0);
}

if (tempDirection != Vector2Int.zero) _lastInputDirection = tempDirection;
if (tempDirection != Vector2Int.zero) _inputDirection = tempDirection;
}

#endregion

private int ActionIndexForDirection(Vector2Int direction) {
int index = Array.IndexOf(_actionDirections, direction);


if (index == -1) {
Debug.LogError("Invalid direction");
return 0; // This should be reasonable default value.
Expand Down Expand Up @@ -126,6 +129,7 @@ public override void CollectObservations(VectorSensor sensor) {
sensor.AddObservation(board.Snakes[0].Direction);
sensor.AddObservation(board.Snakes[0].Position);
sensor.AddObservation(PreviousDistance);
sensor.AddObservation(board.FoodPositions[0]);
int[] array = board.GetBoardAsArray();

for (int i = 0; i < array.Length; i++) {
Expand All @@ -144,26 +148,21 @@ public override void CollectObservations(VectorSensor sensor) {
}
}

private Vector2Int RotateClockwise(Vector2Int direction) {
return new Vector2Int(direction.y, -direction.x);
}

private Vector2Int RotateCounterClockwise(Vector2Int direction) {
return new Vector2Int(-direction.y, direction.x);
}

public override void OnActionReceived(ActionBuffers actions) {
// up = 0, right = 1, down = 2, left = 3
int action = actions.DiscreteActions[0];

if (board.Snakes.Length == 0) {
return;
}

// check if the direction is opposite of the current direction and if so print a warning:
var backDirection = board.Snakes[0].Direction * -1;
var actionDirection = GetDirectionFromAction(action);


if (backDirection == actionDirection
) {
Debug.LogWarning("<color=red>You are going in the opposite direction!</color>");

}


var snake = board.Snakes[0];
int lengthAtTimeStep = snake.Length;
Vector2 currentFoodPosition, previousFoodPosition = Vector2.zero;
Expand All @@ -173,7 +172,18 @@ public override void OnActionReceived(ActionBuffers actions) {
previousFoodPosition = currentFoodPosition;
}

snake.NextDirection = GetDirectionFromAction(action);
switch (action) {
case 0: //turn left
snake.NextDirection = RotateCounterClockwise(snake.Direction);
break;
case 1: //go straight, no need to change the direction
snake.NextDirection = snake.Direction;
break;
case 2: //turn right
snake.NextDirection = RotateClockwise(snake.Direction);
break;
}

_snakeController.FinalizeDirection(snake);
_snakeController.CheckCollisions(board);

Expand Down Expand Up @@ -207,6 +217,7 @@ public override void OnActionReceived(ActionBuffers actions) {
var snakeNewPosition = board.Snakes[0].Position;
// Distance Reward
PreviousDistance = Vector2.Distance(snakeNewPosition, previousFoodPosition);

if (board.FoodPositions.Count > 0) {
currentFoodPosition = board.FoodPositions[0];
// Calculate Euclidean distance
Expand All @@ -217,7 +228,7 @@ public override void OnActionReceived(ActionBuffers actions) {
/ (lengthAtTimeStep + currentDistance));

// Apply the reward
AddReward(reward);
// AddReward(reward);
}


Expand All @@ -227,6 +238,8 @@ public override void OnActionReceived(ActionBuffers actions) {
board.Snakes[0].AteFood = false;
}

AddReward(-0.001f);

if (!IsSnakeAlive(snake)) {
Debug.Log("Dead : " + -10f);
AddReward(-10f);
Expand All @@ -236,8 +249,28 @@ public override void OnActionReceived(ActionBuffers actions) {

public override void Heuristic(in ActionBuffers actionsOut) {
Debug.Log("Heuristic");
Vector2Int currentDirection = board.Snakes[0].Direction;
int relativeDirection = GetRelativeDirection(currentDirection, _inputDirection);
var discreteActionsOut = actionsOut.DiscreteActions;
discreteActionsOut[0] = ActionIndexForDirection(_lastInputDirection);
discreteActionsOut[0] = relativeDirection;
}

private int GetRelativeDirection(Vector2Int currentDirection, Vector2Int inputDirection) {
var clockwiseDirection = RotateClockwise(currentDirection);
var counterClockwiseDirection = RotateCounterClockwise(currentDirection);




if (inputDirection == counterClockwiseDirection)
return 0; // turn left
if (inputDirection == currentDirection)
return 1; // go straight
if (inputDirection == clockwiseDirection)
return 2; // turn right


return 1;
}
}
}
Loading

0 comments on commit 88f850b

Please sign in to comment.