Prepare alpha 11 with online diagnostics
This commit is contained in:
Emporex committed
1 parent
d53467baf2
commit
4d9e6d9185
14 files changed
+550
-77
No files matched your search
+240
-25
@@ -48,6 +48,27 @@ var snapshot_bytes_max := 0
|
||||
var snapshot_parse_max_us := 0
|
||||
var snapshot_large_gap_count := 0
|
||||
var snapshot_coalesced_count := 0
|
||||
var snapshot_queue_dropped_count := 0
|
||||
var snapshot_queue_max := 0
|
||||
var snapshot_bytes_total := 0
|
||||
var snapshot_tick_previous := -1
|
||||
var snapshot_tick_jump_max := 0
|
||||
var snapshot_tick_jumps := 0
|
||||
var snapshot_gap_samples: Array[int] = []
|
||||
var diagnostic_file: FileAccess
|
||||
var diagnostic_path := ""
|
||||
var diagnostic_match_key := ""
|
||||
var diagnostic_match_id := ""
|
||||
var diagnostic_finished_match_key := ""
|
||||
var diagnostic_open_failed := false
|
||||
var diagnostic_slot := -1
|
||||
var diagnostic_frame_max_ms := 0
|
||||
var diagnostic_frame_stalls := 0
|
||||
var diagnostic_apply_max_us := 0
|
||||
var diagnostic_enemy_count := 0
|
||||
var diagnostic_tower_count := 0
|
||||
var diagnostic_last_summary := ""
|
||||
var diagnostic_records: Array[Dictionary] = []
|
||||
var pending_snapshot_payloads: Array[String] = []
|
||||
var friends: Array[Dictionary] = []
|
||||
var friend_statuses: Dictionary = {}
|
||||
@@ -63,6 +84,8 @@ func _ready() -> void:
|
||||
|
||||
func _process(_delta: float) -> void:
|
||||
_flush_pending_snapshots()
|
||||
if diagnostic_file != null and Time.get_ticks_msec() - snapshot_report_ms >= 10000:
|
||||
_report_diagnostic_window()
|
||||
|
||||
|
||||
func _flush_pending_snapshots() -> void:
|
||||
@@ -78,6 +101,16 @@ func _flush_pending_snapshots() -> void:
|
||||
snapshot_parse_max_us = maxi(snapshot_parse_max_us, Time.get_ticks_usec() - parse_start_us)
|
||||
if not parsed is Dictionary:
|
||||
continue
|
||||
var server_tick := int(parsed.get("tick", -1))
|
||||
if server_tick >= 0:
|
||||
if snapshot_tick_previous >= 0:
|
||||
var tick_jump := server_tick - snapshot_tick_previous
|
||||
snapshot_tick_jump_max = maxi(snapshot_tick_jump_max, tick_jump)
|
||||
if tick_jump > 1:
|
||||
snapshot_tick_jumps += 1
|
||||
if tick_jump > 3:
|
||||
_write_diagnostic_record("tick_jump", {"from_tick": snapshot_tick_previous, "to_tick": server_tick, "steps": tick_jump})
|
||||
snapshot_tick_previous = server_tick
|
||||
latest = parsed
|
||||
var shots = parsed.get("shots", [])
|
||||
if shots is Array:
|
||||
@@ -90,6 +123,179 @@ func _flush_pending_snapshots() -> void:
|
||||
match_snapshot_received.emit(latest)
|
||||
|
||||
|
||||
func _begin_match_diagnostics() -> void:
|
||||
if current_match == null:
|
||||
return
|
||||
var match_key := String(current_match.match_id).sha256_text().substr(0, 12)
|
||||
if match_key == diagnostic_finished_match_key:
|
||||
return
|
||||
if diagnostic_match_key == match_key and (diagnostic_file != null or diagnostic_open_failed):
|
||||
return
|
||||
if diagnostic_file != null:
|
||||
_finish_match_diagnostics("new_match")
|
||||
diagnostic_match_key = match_key
|
||||
diagnostic_match_id = String(current_match.match_id)
|
||||
diagnostic_finished_match_key = ""
|
||||
diagnostic_open_failed = false
|
||||
diagnostic_slot = -1
|
||||
diagnostic_records.clear()
|
||||
snapshot_previous_ms = 0
|
||||
snapshot_report_ms = Time.get_ticks_msec()
|
||||
snapshot_tick_previous = -1
|
||||
_reset_diagnostic_window()
|
||||
var stamp := Time.get_datetime_string_from_system(true).replace(":", "-")
|
||||
diagnostic_path = "user://online-diag-%s-%s-%d.jsonl" % [stamp, match_key, Time.get_ticks_usec()]
|
||||
diagnostic_file = FileAccess.open(diagnostic_path, FileAccess.WRITE)
|
||||
if diagnostic_file == null:
|
||||
diagnostic_open_failed = true
|
||||
push_warning("Online-Diagnose konnte nicht gespeichert werden: %s" % diagnostic_path)
|
||||
return
|
||||
_write_diagnostic_record("start", {
|
||||
"match": match_key,
|
||||
"host": current_host,
|
||||
"version": str(ProjectSettings.get_setting("application/config/version", "")),
|
||||
"transport": "nakama_websocket",
|
||||
"expected_snapshots_per_second": 10
|
||||
})
|
||||
print("ONLINE DIAG file=%s" % ProjectSettings.globalize_path(diagnostic_path))
|
||||
|
||||
|
||||
func set_diagnostic_slot(slot: int, viewing_right: bool) -> void:
|
||||
if diagnostic_slot == slot:
|
||||
return
|
||||
diagnostic_slot = slot
|
||||
_write_diagnostic_record("side", {"slot": slot, "viewing_right": viewing_right})
|
||||
|
||||
|
||||
func record_online_frame(delta: float) -> void:
|
||||
if diagnostic_file == null:
|
||||
return
|
||||
var frame_ms := roundi(delta * 1000.0)
|
||||
diagnostic_frame_max_ms = maxi(diagnostic_frame_max_ms, frame_ms)
|
||||
if frame_ms > 200:
|
||||
diagnostic_frame_stalls += 1
|
||||
if frame_ms > 1000:
|
||||
_write_diagnostic_record("frame_stall", {"duration_ms": frame_ms, "last_server_tick": snapshot_tick_previous})
|
||||
|
||||
|
||||
func record_snapshot_apply(duration_us: int, enemy_count: int, tower_count: int) -> void:
|
||||
diagnostic_apply_max_us = maxi(diagnostic_apply_max_us, duration_us)
|
||||
diagnostic_enemy_count = enemy_count
|
||||
diagnostic_tower_count = tower_count
|
||||
|
||||
|
||||
func _write_diagnostic_record(event_name: String, details: Dictionary) -> void:
|
||||
var record := details.duplicate(true)
|
||||
record["event"] = event_name
|
||||
record["utc"] = Time.get_datetime_string_from_system(true)
|
||||
record["slot"] = diagnostic_slot
|
||||
if event_name != "start":
|
||||
diagnostic_records.append(record)
|
||||
if diagnostic_records.size() > 160:
|
||||
diagnostic_records.pop_front()
|
||||
if diagnostic_file == null:
|
||||
return
|
||||
diagnostic_file.store_line(JSON.stringify(record))
|
||||
diagnostic_file.flush()
|
||||
|
||||
|
||||
func _report_diagnostic_window() -> void:
|
||||
var now_ms := Time.get_ticks_msec()
|
||||
var sorted_gaps := snapshot_gap_samples.duplicate()
|
||||
sorted_gaps.sort()
|
||||
var p95_gap := 0
|
||||
if not sorted_gaps.is_empty():
|
||||
p95_gap = sorted_gaps[clampi(int(ceil(float(sorted_gaps.size()) * 0.95)) - 1, 0, sorted_gaps.size() - 1)]
|
||||
var seconds := maxf(0.001, float(now_ms - snapshot_report_ms) / 1000.0)
|
||||
var record := {
|
||||
"seconds": snappedf(seconds, 0.001),
|
||||
"snapshots": snapshot_count,
|
||||
"snapshots_per_second": snappedf(float(snapshot_count) / seconds, 0.01),
|
||||
"gap_p95_ms": p95_gap,
|
||||
"gap_max_ms": snapshot_gap_max_ms,
|
||||
"gaps_over_500_ms": snapshot_large_gap_count,
|
||||
"last_server_tick": snapshot_tick_previous,
|
||||
"tick_jump_max": snapshot_tick_jump_max,
|
||||
"tick_jumps_over_1": snapshot_tick_jumps,
|
||||
"bytes_avg": snapshot_bytes_total / maxi(1, snapshot_count),
|
||||
"bytes_max": snapshot_bytes_max,
|
||||
"parse_max_us": snapshot_parse_max_us,
|
||||
"apply_max_us": diagnostic_apply_max_us,
|
||||
"frame_max_ms": diagnostic_frame_max_ms,
|
||||
"frame_stalls_over_200_ms": diagnostic_frame_stalls,
|
||||
"coalesced": snapshot_coalesced_count,
|
||||
"queue_dropped": snapshot_queue_dropped_count,
|
||||
"queue_max": snapshot_queue_max,
|
||||
"enemies": diagnostic_enemy_count,
|
||||
"towers": diagnostic_tower_count
|
||||
}
|
||||
diagnostic_last_summary = JSON.stringify(record)
|
||||
_write_diagnostic_record("window", record)
|
||||
print("ONLINE DIAG %s" % diagnostic_last_summary)
|
||||
snapshot_report_ms = now_ms
|
||||
_reset_diagnostic_window()
|
||||
|
||||
|
||||
func _reset_diagnostic_window() -> void:
|
||||
snapshot_count = 0
|
||||
snapshot_gap_max_ms = 0
|
||||
snapshot_bytes_max = 0
|
||||
snapshot_parse_max_us = 0
|
||||
snapshot_large_gap_count = 0
|
||||
snapshot_coalesced_count = 0
|
||||
snapshot_queue_dropped_count = 0
|
||||
snapshot_queue_max = 0
|
||||
snapshot_bytes_total = 0
|
||||
snapshot_tick_jump_max = 0
|
||||
snapshot_tick_jumps = 0
|
||||
snapshot_gap_samples.clear()
|
||||
diagnostic_frame_max_ms = 0
|
||||
diagnostic_frame_stalls = 0
|
||||
diagnostic_apply_max_us = 0
|
||||
|
||||
|
||||
func _finish_match_diagnostics(reason: String) -> void:
|
||||
if diagnostic_match_key.is_empty():
|
||||
return
|
||||
if snapshot_count > 0 or Time.get_ticks_msec() - snapshot_report_ms >= 1000:
|
||||
_report_diagnostic_window()
|
||||
_write_diagnostic_record("end", {"reason": reason})
|
||||
var match_id := diagnostic_match_id
|
||||
if not match_id.is_empty() and is_authenticated() and not diagnostic_records.is_empty():
|
||||
var report := {"match_id": match_id, "version": str(ProjectSettings.get_setting("application/config/version", "development")),
|
||||
"records": diagnostic_records.duplicate(true)}
|
||||
while JSON.stringify(report).to_utf8_buffer().size() > 60000 and report["records"].size() > 1:
|
||||
report["records"].pop_front()
|
||||
_upload_match_diagnostics.call_deferred(report)
|
||||
diagnostic_file = null
|
||||
diagnostic_finished_match_key = diagnostic_match_key
|
||||
diagnostic_match_key = ""
|
||||
diagnostic_match_id = ""
|
||||
diagnostic_open_failed = false
|
||||
diagnostic_records.clear()
|
||||
|
||||
|
||||
func _upload_match_diagnostics(report: Dictionary) -> void:
|
||||
for attempt in 3:
|
||||
if not is_authenticated():
|
||||
break
|
||||
var result = await client.rpc_async(session, "submit_online_diagnostics", JSON.stringify(report))
|
||||
var parsed := _parse_rpc_result(result)
|
||||
if bool(parsed.get("accepted", false)):
|
||||
print("ONLINE DIAG hochgeladen: Runde %s, Seite %d" % [String(parsed.get("round_id", "")), int(parsed.get("slot", -1))])
|
||||
return
|
||||
await get_tree().create_timer(2.0 * (attempt + 1)).timeout
|
||||
push_warning("Online-Diagnose konnte nicht hochgeladen werden; lokale Datei bleibt erhalten.")
|
||||
|
||||
|
||||
func get_diagnostic_bundle() -> String:
|
||||
if diagnostic_file != null and (snapshot_count > 0 or Time.get_ticks_msec() - snapshot_report_ms >= 1000):
|
||||
_report_diagnostic_window()
|
||||
if diagnostic_path.is_empty() or not FileAccess.file_exists(diagnostic_path):
|
||||
return "Noch keine Online-Diagnose vorhanden."
|
||||
return "Void Bastion Online-Diagnose\nDatei: %s\n\n%s" % [ProjectSettings.globalize_path(diagnostic_path), FileAccess.get_file_as_string(diagnostic_path)]
|
||||
|
||||
|
||||
func _is_headless_or_editor() -> bool:
|
||||
return DisplayServer.get_name() == "headless" or Engine.is_editor_hint()
|
||||
|
||||
@@ -298,15 +504,22 @@ func refresh_friends() -> bool:
|
||||
return true
|
||||
|
||||
|
||||
func add_friend_by_username(username: String) -> bool:
|
||||
var normalized := username.strip_edges()
|
||||
if not is_authenticated() or normalized.is_empty() or normalized == session.username:
|
||||
return false
|
||||
var result = await client.add_friends_async(session, null, PackedStringArray([normalized]))
|
||||
func add_friend_by_display_name(display_name: String) -> Dictionary:
|
||||
var normalized := display_name.strip_edges()
|
||||
if not is_authenticated() or normalized.is_empty():
|
||||
return {"accepted": false, "error": "offline"}
|
||||
var lookup_result = await client.rpc_async(session, "find_player_by_name", JSON.stringify({"display_name": normalized}))
|
||||
var found := _parse_rpc_result(lookup_result)
|
||||
if not bool(found.get("found", false)):
|
||||
return {"accepted": false, "error": String(found.get("error", "not_found"))}
|
||||
var user_id := String(found.get("user_id", ""))
|
||||
if user_id.is_empty() or user_id == session.user_id:
|
||||
return {"accepted": false, "error": "self"}
|
||||
var result = await client.add_friends_async(session, PackedStringArray([user_id]))
|
||||
if result == null or result.is_exception():
|
||||
return false
|
||||
return {"accepted": false, "error": "send_failed"}
|
||||
await refresh_friends()
|
||||
return true
|
||||
return {"accepted": true, "display_name": String(found.get("display_name", normalized))}
|
||||
|
||||
|
||||
func accept_friend(user_id: String) -> bool:
|
||||
@@ -416,12 +629,14 @@ func begin_matchmaking(mode := "quick", difficulty := 1) -> bool:
|
||||
func cancel_matchmaking() -> void:
|
||||
intentional_disconnect = true
|
||||
pending_snapshot_payloads.clear()
|
||||
_finish_match_diagnostics("cancelled_matchmaking")
|
||||
var match_to_leave = current_match
|
||||
current_match = null
|
||||
if socket and not matchmaker_ticket.is_empty():
|
||||
await socket.remove_matchmaker_async(matchmaker_ticket)
|
||||
matchmaker_ticket = ""
|
||||
if current_match != null and not private_room_code.is_empty():
|
||||
await socket.leave_match_async(current_match.match_id)
|
||||
current_match = null
|
||||
if socket and match_to_leave != null:
|
||||
await socket.leave_match_async(match_to_leave.match_id)
|
||||
private_room_code = ""
|
||||
matchmaking_state_changed.emit("idle")
|
||||
|
||||
@@ -526,9 +741,11 @@ func request_wave_start() -> bool:
|
||||
|
||||
func leave_current_match() -> void:
|
||||
pending_snapshot_payloads.clear()
|
||||
if socket and current_match != null:
|
||||
await socket.leave_match_async(current_match.match_id)
|
||||
_finish_match_diagnostics("left_match")
|
||||
var match_to_leave = current_match
|
||||
current_match = null
|
||||
if socket and match_to_leave != null:
|
||||
await socket.leave_match_async(match_to_leave.match_id)
|
||||
private_room_code = ""
|
||||
lobby_ready = false
|
||||
|
||||
@@ -537,30 +754,25 @@ func _on_match_state(data) -> void:
|
||||
if current_match == null or data.match_id != current_match.match_id:
|
||||
return
|
||||
if data.op_code == OP_SNAPSHOT:
|
||||
_begin_match_diagnostics()
|
||||
var now_ms := Time.get_ticks_msec()
|
||||
if snapshot_previous_ms > 0:
|
||||
var gap_ms := now_ms - snapshot_previous_ms
|
||||
snapshot_gap_max_ms = maxi(snapshot_gap_max_ms, gap_ms)
|
||||
snapshot_gap_samples.append(gap_ms)
|
||||
if gap_ms > 500:
|
||||
snapshot_large_gap_count += 1
|
||||
_write_diagnostic_record("snapshot_gap", {"duration_ms": gap_ms, "last_server_tick": snapshot_tick_previous})
|
||||
snapshot_previous_ms = now_ms
|
||||
snapshot_count += 1
|
||||
snapshot_bytes_max = maxi(snapshot_bytes_max, String(data.data).to_utf8_buffer().size())
|
||||
var payload_bytes := String(data.data).to_utf8_buffer().size()
|
||||
snapshot_bytes_max = maxi(snapshot_bytes_max, payload_bytes)
|
||||
snapshot_bytes_total += payload_bytes
|
||||
pending_snapshot_payloads.append(String(data.data))
|
||||
snapshot_queue_max = maxi(snapshot_queue_max, pending_snapshot_payloads.size())
|
||||
if pending_snapshot_payloads.size() > MAX_PENDING_SNAPSHOTS:
|
||||
pending_snapshot_payloads.pop_front()
|
||||
snapshot_coalesced_count += 1
|
||||
if snapshot_report_ms == 0:
|
||||
snapshot_report_ms = now_ms
|
||||
elif now_ms - snapshot_report_ms >= 10000:
|
||||
print("ONLINE DIAG snapshots=%d coalesced=%d max_gap_ms=%d gaps_over_500ms=%d max_bytes=%d max_parse_us=%d" % [snapshot_count, snapshot_coalesced_count, snapshot_gap_max_ms, snapshot_large_gap_count, snapshot_bytes_max, snapshot_parse_max_us])
|
||||
snapshot_report_ms = now_ms
|
||||
snapshot_count = 0
|
||||
snapshot_coalesced_count = 0
|
||||
snapshot_gap_max_ms = 0
|
||||
snapshot_large_gap_count = 0
|
||||
snapshot_bytes_max = 0
|
||||
snapshot_parse_max_us = 0
|
||||
snapshot_queue_dropped_count += 1
|
||||
elif data.op_code == OP_EVENT:
|
||||
var parsed = JSON.parse_string(data.data)
|
||||
if parsed is Dictionary:
|
||||
@@ -569,6 +781,7 @@ func _on_match_state(data) -> void:
|
||||
|
||||
func _on_socket_closed() -> void:
|
||||
print("ONLINE DIAG websocket_closed reconnect=%s" % str(not intentional_disconnect and current_match != null))
|
||||
_write_diagnostic_record("socket_closed", {"reconnect": not intentional_disconnect and current_match != null})
|
||||
pending_snapshot_payloads.clear()
|
||||
presence_status = ""
|
||||
friend_statuses.clear()
|
||||
@@ -621,6 +834,7 @@ func _recover_match(match_id: String) -> void:
|
||||
var rejoined = await socket.join_match_async(match_id)
|
||||
if rejoined != null and not rejoined.is_exception():
|
||||
print("ONLINE DIAG websocket_reconnected attempt=%d" % (attempt + 1))
|
||||
_write_diagnostic_record("socket_reconnected", {"attempt": attempt + 1})
|
||||
current_match = rejoined
|
||||
intentional_disconnect = false
|
||||
reconnect_in_progress = false
|
||||
@@ -631,6 +845,7 @@ func _recover_match(match_id: String) -> void:
|
||||
return
|
||||
reconnect_in_progress = false
|
||||
print("ONLINE DIAG websocket_reconnect_failed")
|
||||
_write_diagnostic_record("socket_reconnect_failed", {})
|
||||
matchmaking_state_changed.emit("connection_lost")
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user