From ce16589d2834c76a57a6140de9f4034de45772b7 Mon Sep 17 00:00:00 2001 From: Val Lorentz Date: Fri, 2 Jun 2023 23:14:25 +0200 Subject: [PATCH 1/2] Add support for pagination cursors Not tested live yet --- lib/matrix_client/chat_history.ex | 150 +++++++++++++++++++++++++----- lib/matrix_client/client.ex | 40 ++++++++ test/irc/handler_test.exs | 89 +++++++++++++++++- test/test_helper.exs | 28 +++++- 4 files changed, 279 insertions(+), 28 deletions(-) diff --git a/lib/matrix_client/chat_history.ex b/lib/matrix_client/chat_history.ex index 464900a..81671b8 100644 --- a/lib/matrix_client/chat_history.ex +++ b/lib/matrix_client/chat_history.ex @@ -22,16 +22,36 @@ defmodule M51.MatrixClient.ChatHistory do def after_(sup_pid, room_id, anchor, limit) do client = M51.IrcConn.Supervisor.matrix_client(sup_pid) - case parse_anchor(anchor) do - {:ok, event_id} -> + case parse_anchor(anchor, true) do + {:ok, :msgid, event_id} -> case M51.MatrixClient.Client.get_event_context( client, room_id, event_id, limit * 2 ) do - {:ok, events} -> {:ok, process_events(sup_pid, room_id, events["events_after"])} - {:error, message} -> {:error, Kernel.inspect(message)} + {:ok, events} -> + {:ok, + process_events(sup_pid, room_id, events["events_after"], Map.get(events, "end"), nil)} + + {:error, message} -> + {:error, Kernel.inspect(message)} + end + + {:ok, :cursor, cursor} -> + case M51.MatrixClient.Client.get_events_from_cursor( + client, + room_id, + "f", + cursor, + limit + ) do + {:ok, events} -> + {:ok, + process_events(sup_pid, room_id, events["chunk"], Map.get(events, "end"), nil)} + + {:error, message} -> + {:error, Kernel.inspect(message)} end {:error, message} -> @@ -42,9 +62,9 @@ defmodule M51.MatrixClient.ChatHistory do def around(sup_pid, room_id, anchor, limit) do client = M51.IrcConn.Supervisor.matrix_client(sup_pid) - case parse_anchor(anchor) do - {:ok, event_id} -> - case M51.MatrixClient.Client.get_event_context(client, room_id, event_id, limit) do + case parse_anchor(anchor, false) do + {:ok, :msgid, event_id} -> + case M51.MatrixClient.Client.get_event_context(client, room_id, event_id, limit - 1) do {:ok, events} -> # TODO: if there aren't enough events after (resp. before), allow more # events before (resp. after) than half the limit. @@ -53,9 +73,16 @@ defmodule M51.MatrixClient.ChatHistory do events_before = events["events_before"] |> Enum.slice(0, nb_before) |> Enum.reverse() events_after = events["events_after"] |> Enum.slice(0, nb_after) - events = Enum.concat([events_before, [events["event"]], events_after]) + events_list = Enum.concat([events_before, [events["event"]], events_after]) - {:ok, process_events(sup_pid, room_id, events)} + {:ok, + process_events( + sup_pid, + room_id, + events_list, + Map.get(events, "end"), + Map.get(events, "start") + )} {:error, message} -> {:error, Kernel.inspect(message)} @@ -69,8 +96,8 @@ defmodule M51.MatrixClient.ChatHistory do def before(sup_pid, room_id, anchor, limit) do client = M51.IrcConn.Supervisor.matrix_client(sup_pid) - case parse_anchor(anchor) do - {:ok, event_id} -> + case parse_anchor(anchor, true) do + {:ok, :msgid, event_id} -> case M51.MatrixClient.Client.get_event_context( client, room_id, @@ -78,7 +105,36 @@ defmodule M51.MatrixClient.ChatHistory do limit * 2 ) do {:ok, events} -> - {:ok, process_events(sup_pid, room_id, Enum.reverse(events["events_before"]))} + {:ok, + process_events( + sup_pid, + room_id, + Enum.reverse(events["events_before"]), + Map.get(events, "start"), + nil + )} + + {:error, message} -> + {:error, Kernel.inspect(message)} + end + + {:ok, :cursor, cursor} -> + case M51.MatrixClient.Client.get_events_from_cursor( + client, + room_id, + "b", + cursor, + limit + ) do + {:ok, events} -> + {:ok, + process_events( + sup_pid, + room_id, + Enum.reverse(events["chunk"]), + Map.get(events, "end"), + nil + )} {:error, message} -> {:error, Kernel.inspect(message)} @@ -98,28 +154,41 @@ defmodule M51.MatrixClient.ChatHistory do limit ) do {:ok, events} -> - {:ok, process_events(sup_pid, room_id, Enum.reverse(events["chunk"]))} + {:ok, + process_events( + sup_pid, + room_id, + Enum.reverse(events["chunk"]), + Map.get(events, "end"), + nil + )} {:error, message} -> {:error, Kernel.inspect(message)} end end - defp parse_anchor(anchor) do + defp parse_anchor(anchor, allow_cursor) do case String.split(anchor, "=", parts: 2) do ["msgid", msgid] -> - {:ok, msgid} + {:ok, :msgid, msgid} + + ["cursor", cursor] when allow_cursor -> + {:ok, :cursor, cursor} + + ["cursor", _] -> + {:error, "Invalid anchor: '#{anchor}', it should start with 'msgid='."} ["timestamp", _] -> {:error, "CHATHISTORY with timestamps is not supported. See https://github.com/progval/matrix2051/issues/1"} _ -> - {:error, "Invalid anchor: '#{anchor}', it should start with 'msgid='."} + {:error, "Invalid anchor: '#{anchor}', it should start with 'msgid=' or 'cursor='."} end end - defp process_events(sup_pid, room_id, events) do + defp process_events(sup_pid, room_id, events, next, prev) do pid = self() write = fn cmd -> send(pid, {:command, cmd}) end @@ -154,12 +223,45 @@ defmodule M51.MatrixClient.ChatHistory do |> Task.await() # Collect all commands - Stream.unfold(nil, fn _ -> - receive do - {:command, cmd} -> {cmd, nil} - {:finished_processing} -> nil - end - end) - |> Enum.to_list() + batch_content = + Stream.unfold(nil, fn _ -> + receive do + {:command, cmd} -> {cmd, nil} + {:finished_processing} -> nil + end + end) + |> Enum.to_list() + + # Prepend cursors, if any + case {next, prev} do + {nil, nil} -> + batch_content + + {next, nil} -> + cursors = %M51.Irc.Command{ + command: "CHATHISTORY", + params: ["CURSORS", room_id, next] + } + + [cursors | batch_content] + + {nil, prev} -> + # what do we do here? + # https://github.com/ircv3/ircv3-specifications/pull/525/files#r1214764104 + cursors = %M51.Irc.Command{ + command: "CHATHISTORY", + params: ["CURSORS", room_id, "*", prev] + } + + [cursors | batch_content] + + {next, prev} -> + cursors = %M51.Irc.Command{ + command: "CHATHISTORY", + params: ["CURSORS", room_id, next, prev] + } + + [cursors | batch_content] + end end end diff --git a/lib/matrix_client/client.ex b/lib/matrix_client/client.ex index ff35fe7..e8d2108 100644 --- a/lib/matrix_client/client.ex +++ b/lib/matrix_client/client.ex @@ -325,6 +325,36 @@ defmodule M51.MatrixClient.Client do {:reply, reply, state} end + @impl true + def handle_call({:get_events_from_cursor, channel, dir, cursor, limit}, _from, state) do + %M51.MatrixClient.Client{ + state: :connected, + irc_pid: irc_pid, + raw_client: raw_client + } = state + + matrix_state = M51.IrcConn.Supervisor.matrix_state(irc_pid) + + reply = + case M51.MatrixClient.State.room_from_irc_channel(matrix_state, channel) do + nil -> + {:error, {:room_not_found, channel}} + + {room_id, _room} -> + path = + "/_matrix/client/v3/rooms/#{urlquote(room_id)}/messages?" <> + URI.encode_query(%{"limit" => limit, "dir" => dir, "from" => cursor}) + + case M51.Matrix.RawClient.get(raw_client, path) do + {:ok, events} -> {:ok, events} + {:error, error} -> {:error, error} + {:error, nil, error} -> {:error, error} + end + end + + {:reply, reply, state} + end + @impl true def handle_call({:get_latest_events, channel, limit}, _from, state) do %M51.MatrixClient.Client{ @@ -559,6 +589,16 @@ defmodule M51.MatrixClient.Client do GenServer.call(pid, {:get_event_context, channel, event_id, limit}, @timeout) end + @doc """ + Returns a page of events just before/after those returned by a previous call + to get_event_context/4 or get_events_from_cursor/5 + + https://matrix.org/docs/spec/client_server/r0.6.1#id131 + """ + def get_events_from_cursor(pid, channel, dir, cursor, limit) do + GenServer.call(pid, {:get_events_from_cursor, channel, dir, cursor, limit}, @timeout) + end + @doc """ Returns latest events of a room diff --git a/test/irc/handler_test.exs b/test/irc/handler_test.exs index a413726..e953c67 100644 --- a/test/irc/handler_test.exs +++ b/test/irc/handler_test.exs @@ -901,7 +901,7 @@ defmodule M51.IrcConn.HandlerTest do ) end - test "CHATHISTORY AROUND", %{handler: handler} do + test "CHATHISTORY AROUND msgid", %{handler: handler} do do_connection_registration(handler, ["message-tags"]) send(handler, cmd("@label=l1 CHATHISTORY AROUND #chan msgid=$event3 5")) @@ -934,6 +934,8 @@ defmodule M51.IrcConn.HandlerTest do {batch_id, line} = assert_open_batch() assert line == "@label=l2 BATCH +#{batch_id} :chathistory\r\n" + assert_line("@batch=#{batch_id} CHATHISTORY CURSORS #chan * :start3\r\n") + assert_line( "@batch=#{batch_id};msgid=$event1 :nick:example.org!nick@example.org PRIVMSG #chan :first message\r\n" ) @@ -956,6 +958,8 @@ defmodule M51.IrcConn.HandlerTest do {batch_id, line} = assert_open_batch() assert line == "@label=l3 BATCH +#{batch_id} :chathistory\r\n" + assert_line("@batch=#{batch_id} CHATHISTORY CURSORS #chan end2 :start2\r\n") + assert_line( "@batch=#{batch_id};msgid=$event2 :nick:example.org!nick@example.org PRIVMSG #chan :second message\r\n" ) @@ -974,6 +978,8 @@ defmodule M51.IrcConn.HandlerTest do {batch_id, line} = assert_open_batch() assert line == "@label=l4 BATCH +#{batch_id} :chathistory\r\n" + assert_line("@batch=#{batch_id} CHATHISTORY CURSORS #chan end1 :start1\r\n") + assert_line( "@batch=#{batch_id};msgid=$event2 :nick:example.org!nick@example.org PRIVMSG #chan :second message\r\n" ) @@ -988,6 +994,8 @@ defmodule M51.IrcConn.HandlerTest do {batch_id, line} = assert_open_batch() assert line == "@label=l5 BATCH +#{batch_id} :chathistory\r\n" + assert_line("@batch=#{batch_id} CHATHISTORY CURSORS #chan end0 :start0\r\n") + assert_line( "@batch=#{batch_id};msgid=$event3 :nick:example.org!nick@example.org PRIVMSG #chan :third message\r\n" ) @@ -995,7 +1003,27 @@ defmodule M51.IrcConn.HandlerTest do assert_line("BATCH :-#{batch_id}\r\n") end - test "CHATHISTORY BEFORE", %{handler: handler} do + test "CHATHISTORY AROUND timestamp", %{handler: handler} do + do_connection_registration(handler, ["message-tags"]) + + send(handler, cmd("@label=l1 CHATHISTORY AROUND #chan timestamp=2019-01-04T14:34:16.123Z 5")) + + assert_line( + "@label=l1 FAIL CHATHISTORY MESSAGE_ERROR AROUND :CHATHISTORY with timestamps is not supported. See https://github.com/progval/matrix2051/issues/1\r\n" + ) + end + + test "CHATHISTORY AROUND cursor", %{handler: handler} do + do_connection_registration(handler, ["message-tags"]) + + send(handler, cmd("@label=l1 CHATHISTORY AROUND #chan cursor=blah 5")) + + assert_line( + "@label=l1 FAIL CHATHISTORY MESSAGE_ERROR AROUND :Invalid anchor: 'cursor=blah', it should start with 'msgid='.\r\n" + ) + end + + test "CHATHISTORY BEFORE msgid", %{handler: handler} do do_connection_registration(handler, ["message-tags"]) send(handler, cmd("@label=l1 CHATHISTORY BEFORE #chan msgid=$event3 2")) @@ -1016,6 +1044,8 @@ defmodule M51.IrcConn.HandlerTest do {batch_id, line} = assert_open_batch() assert line == "@label=l2 BATCH +#{batch_id} :chathistory\r\n" + assert_line("@batch=#{batch_id} CHATHISTORY CURSORS #chan :start2\r\n") + assert_line( "@batch=#{batch_id};msgid=$event2 :nick:example.org!nick@example.org PRIVMSG #chan :second message\r\n" ) @@ -1023,6 +1053,32 @@ defmodule M51.IrcConn.HandlerTest do assert_line("BATCH :-#{batch_id}\r\n") end + test "CHATHISTORY BEFORE timestamp", %{handler: handler} do + do_connection_registration(handler, ["message-tags"]) + + send(handler, cmd("@label=l1 CHATHISTORY BEFORE #chan timestamp=2019-01-04T14:34:16.123Z 5")) + + assert_line( + "@label=l1 FAIL CHATHISTORY MESSAGE_ERROR BEFORE :CHATHISTORY with timestamps is not supported. See https://github.com/progval/matrix2051/issues/1\r\n" + ) + end + + test "CHATHISTORY BEFORE cursor", %{handler: handler} do + do_connection_registration(handler, ["message-tags"]) + + send(handler, cmd("@label=l1 CHATHISTORY BEFORE #chan cursor=blah 1")) + {batch_id, line} = assert_open_batch() + assert line == "@label=l1 BATCH +#{batch_id} :chathistory\r\n" + + assert_line("@batch=#{batch_id} CHATHISTORY CURSORS #chan :endcursor\r\n") + + assert_line( + "@batch=#{batch_id};msgid=$event :nick:example.org!nick@example.org PRIVMSG #chan :event in direction b from blah\r\n" + ) + + assert_line("BATCH :-#{batch_id}\r\n") + end + test "CHATHISTORY LATEST", %{handler: handler} do do_connection_registration(handler, ["message-tags"]) @@ -1051,7 +1107,7 @@ defmodule M51.IrcConn.HandlerTest do assert_line("BATCH :-#{batch_id}\r\n") end - test "CHATHISTORY AFTER", %{handler: handler} do + test "CHATHISTORY AFTER msgid", %{handler: handler} do do_connection_registration(handler, ["message-tags"]) send(handler, cmd("@label=l1 CHATHISTORY AFTER #chan msgid=$event3 2")) @@ -1072,10 +1128,37 @@ defmodule M51.IrcConn.HandlerTest do {batch_id, line} = assert_open_batch() assert line == "@label=l2 BATCH +#{batch_id} :chathistory\r\n" + assert_line("@batch=#{batch_id} CHATHISTORY CURSORS #chan :end2\r\n") + assert_line( "@batch=#{batch_id};msgid=$event4 :nick:example.org!nick@example.org PRIVMSG #chan :fourth message\r\n" ) assert_line("BATCH :-#{batch_id}\r\n") end + + test "CHATHISTORY AFTER timestamp", %{handler: handler} do + do_connection_registration(handler, ["message-tags"]) + + send(handler, cmd("@label=l1 CHATHISTORY AFTER #chan timestamp=2019-01-04T14:34:16.123Z 5")) + + assert_line( + "@label=l1 FAIL CHATHISTORY MESSAGE_ERROR AFTER :CHATHISTORY with timestamps is not supported. See https://github.com/progval/matrix2051/issues/1\r\n" + ) + end + + test "CHATHISTORY AFTER cursor", %{handler: handler} do + do_connection_registration(handler, ["message-tags"]) + + send(handler, cmd("@label=l1 CHATHISTORY AFTER #chan cursor=blah 5")) + {batch_id, line} = assert_open_batch() + assert line == "@label=l1 BATCH +#{batch_id} :chathistory\r\n" + + assert_line("@batch=#{batch_id} CHATHISTORY CURSORS #chan :endcursor\r\n") + + assert_line( + "@batch=#{batch_id};msgid=$event :nick:example.org!nick@example.org PRIVMSG #chan :event in direction f from blah\r\n" + ) + assert_line("BATCH :-#{batch_id}\r\n") + end end diff --git a/test/test_helper.exs b/test/test_helper.exs index 66acf53..fde8103 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -185,6 +185,8 @@ defmodule MockMatrixClient do case limit do 0 -> %{ + "start" => "start0", + "end" => "end0", "events_before" => [], "event" => event3, "events_after" => [] @@ -192,6 +194,8 @@ defmodule MockMatrixClient do 1 -> %{ + "start" => "start1", + "end" => "end1", "events_before" => [event2], "event" => event3, "events_after" => [] @@ -199,6 +203,8 @@ defmodule MockMatrixClient do 2 -> %{ + "start" => "start2", + "end" => "end2", "events_before" => [event2], "event" => event3, "events_after" => [event4] @@ -207,6 +213,7 @@ defmodule MockMatrixClient do 3 -> %{ # reverse-chronological order, as per the spec + "start" => "start3", "events_before" => [event2, event1], "event" => event3, "events_after" => [event4] @@ -224,6 +231,23 @@ defmodule MockMatrixClient do {:reply, {:ok, reply}, state} end + @impl true + def handle_call({:get_events_from_cursor, _channel, direction, cursor, _limit}, _from, state) do + event = %{ + "content" => %{ + "body" => "event in direction #{direction} from #{cursor}", + "msgtype" => "m.text" + }, + "event_id" => "$event", + "origin_server_ts" => 1_632_946_233_579, + "sender" => "@nick:example.org", + "type" => "m.room.message", + "unsigned" => %{} + } + events = %{"state" => [], "chunk" => [event], "start" => "startcursor", "end" => "endcursor"} + {:reply, {:ok, events}, state} + end + @impl true def handle_call({:get_latest_events, _channel, limit}, _from, state) do events = @@ -274,7 +298,9 @@ defmodule MockMatrixClient do # "For dir=b events will be in reverse-chronological order" |> Enum.reverse() - {:reply, {:ok, %{"state" => [], "chunk" => events}}, state} + {:reply, + {:ok, %{"state" => [], "chunk" => events, "from" => "fromcursor", "to" => "tocursor"}}, + state} end :w From 8325b01ef822c4733381951a0c92ab77ffe9e726 Mon Sep 17 00:00:00 2001 From: Val Lorentz Date: Tue, 25 Jul 2023 15:20:22 +0200 Subject: [PATCH 2/2] Fix merge --- test/irc/handler_test.exs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/irc/handler_test.exs b/test/irc/handler_test.exs index db8c728..f0d7154 100644 --- a/test/irc/handler_test.exs +++ b/test/irc/handler_test.exs @@ -1160,6 +1160,7 @@ defmodule M51.IrcConn.HandlerTest do "@batch=#{batch_id};msgid=$event :nick:example.org!nick@example.org PRIVMSG #chan :event in direction f from blah\r\n" ) assert_line("BATCH :-#{batch_id}\r\n") + end test "redact a message for no reason", %{handler: handler} do do_connection_registration(handler)