From fe62ea8ee39e5460b663359daf4759e9a379e538 Mon Sep 17 00:00:00 2001 From: Michael Schaefermeyer Date: Tue, 19 Jul 2016 01:01:18 +0200 Subject: Initial commit --- lib/boltex.ex | 13 +++ lib/boltex/bolt.ex | 209 ++++++++++++++++++++++++++++++++++++++ lib/boltex/pack_stream.ex | 113 +++++++++++++++++++++ lib/boltex/pack_stream/encoder.ex | 110 ++++++++++++++++++++ lib/boltex/utils.ex | 14 +++ 5 files changed, 459 insertions(+) create mode 100644 lib/boltex.ex create mode 100644 lib/boltex/bolt.ex create mode 100644 lib/boltex/pack_stream.ex create mode 100644 lib/boltex/pack_stream/encoder.ex create mode 100644 lib/boltex/utils.ex (limited to 'lib') diff --git a/lib/boltex.ex b/lib/boltex.ex new file mode 100644 index 0000000..9de7d3b --- /dev/null +++ b/lib/boltex.ex @@ -0,0 +1,13 @@ +defmodule Boltex do + alias Boltex.Bolt + + def test(host, port, query) do + {:ok, p} = :gen_tcp.connect host, port, [active: false, mode: :binary, packet: :raw] + + :ok = Bolt.handshake :gen_tcp, p + :ok = Bolt.init :gen_tcp, p + + Enum.map Bolt.run_statement(:gen_tcp, p, query), &IO.inspect/1 + end + +end diff --git a/lib/boltex/bolt.ex b/lib/boltex/bolt.ex new file mode 100644 index 0000000..dc0c04d --- /dev/null +++ b/lib/boltex/bolt.ex @@ -0,0 +1,209 @@ +defmodule Boltex.Bolt do + alias Boltex.{Utils, PackStream} + require Logger + + @recv_timeout 1_000 + @max_chunk_size 65_535 + + @user_agent "Boltex/1.0" + @hs_magic << 0x60, 0x60, 0xB0, 0x17 >> + @hs_version << 1 :: 32, 0 :: 32, 0 :: 32, 0 :: 32 >> + + @zero_chunk << 0, 0 >> + + @sig_init 0x01 + @sig_ack_failure 0x0E + @sig_reset 0x0F + @sig_run 0x10 + @sig_discard_all 0x2F + @sig_pull_all 0x3F + @sig_success 0x70 + @sig_record 0x71 + @sig_ignored 0x7E + @sig_failure 0x7F + + @summary ~w(success ignored failure)a + + @moduledoc """ + The Boltex.Bolt module handles the Bolt protocol specific steps (i.e. + handshake, init) as well as sending and receiving messages and wrapping + them in chunks. + + It abstracts transportation, expecing the transport layer to define + send/2 and recv/3 analogous to :gen_tcp. + """ + + @doc "Does the handshake" + def handshake(transport, port) do + transport.send port, @hs_magic <> @hs_version + case transport.recv(port, 4, @recv_timeout) do + {:ok, << 1 :: 32 >>} -> + :ok + + response -> + Logger.error "Handshake failed. Received: #{Utils.hex_encode response})" + {:error, :handshake_failed} + end + end + + @doc """ + Initialises the connection. + """ + def init(transport, port, params \\ %{}) do + send_messages transport, port, [{[@user_agent, params], @sig_init}] + + case receive_data(transport, port) do + {:success, %{}} -> + :ok + + response -> + Logger.error "Init failed. Received: #{Utils.hex_encode response})" + {:error, :init_failed} + end + end + + @doc """ + Sends a list of messages using the Bolt protocol and PackStream encoding. + + Messages have to be in the form of {[messages], signature}. + """ + def send_messages(transport, port, messages) do + Enum.map(messages, &generate_binary_message/1) + |> generate_chunks + |> Enum.each(&(transport.send(port, &1))) + end + + defp generate_binary_message({messages, signature}) do + messages = List.wrap messages + struct_size = length messages + + << 0xB :: 4, struct_size :: 4, signature >> <> + Utils.reduce_to_binary(messages, &PackStream.encode/1) + end + + defp generate_chunks(messages, chunks \\ [], current_chunk \\ <<>>) + defp generate_chunks([], chunks, current_chunk) do + [current_chunk | chunks] + |> Enum.reverse + end + defp generate_chunks([message | messages], chunks, current_chunk) + when byte_size(current_chunk <> message) <= @max_chunk_size do + message_size = byte_size message + current_chunk = + current_chunk <> + << message_size :: 16 >> <> + message <> + @zero_chunk + + generate_chunks messages, chunks, current_chunk + end + defp generate_chunks([chunk | chunks], chunks, current_chunk) do + oversized_chunk = current_chunk <> chunk + {first, rest} = binary_part oversized_chunk, 0, @max_chunk_size + first_size = byte_size first + rest_size = byte_size rest + current_chunk = current_chunk <> << first_size :: 16 >> <> first + new_chunk = << rest_size :: 16 >> <> rest + + generate_chunks chunks, [current_chunk | chunks], new_chunk + end + + @doc """ + Runs a statement (most like Cypher statement) and returns a list of the + records and a summary. + + Records are represented using PackStream's record data type. Their Elixir + representation is a Keyword with the indexse `:sig` and `:fields`. + + ## Examples + + iex> Boltex.Bolt.run_statement("MATCH (n) RETURN n") + [ + {:record, [sig: 1, fields: [1, "Exmaple", "Labels", %{"some_attribute" => "some_value"}, + {:success, %{"type" => "r"}} + ] + """ + def run_statement(transport, port, statement, params \\ %{}) do + send_messages transport, port, [ + {[statement, params], @sig_run}, + {[nil], @sig_pull_all} + ] + + with {:success, %{}} <- receive_data(transport, port), + do: receive_data transport, port + end + + @doc """ + Receives data. + + This function is supposed to be called after a request to the server has been + made. It receives data chunks, mends them (if they were split between frames) + and decodes them using PackStream. + + When just a single message is received (i.e. to acknowledge a command), this + function returns a tuple with two items, the first being the signature and the + second being the message(s) itself. If a list of messages is received it will + return a list of the former. + + The same goes for the messages: If there was a single data point in a message + said data point will be returned by itself. If there were multiple data points, + the list will be returned. + + The signature is represented as one of the following: + + * `:success` + * `:record` + * `:ignored` + * `:failure` + """ + def receive_data(transport, port, previous \\ []) do + case do_receive_data(transport, port) |> unpack do + {:record, _} = data -> + receive_data transport, port, [data | previous] + + {status, _} = data when status in @summary and previous == [] -> + data + + {status, _} = data when status in @summary -> + Enum.reverse [data | previous] + end + end + + defp do_receive_data(transport, port) do + with {:ok, <>} <- transport.recv(port, 2, @recv_timeout), + do: do_receive_data(transport, port, chunk_size) + end + defp do_receive_data(transport, port, chunk_size) do + with {:ok, data} <- transport.recv(port, chunk_size, @recv_timeout) + do + case transport.recv(port, 2, @recv_timeout) do + {:ok, @zero_chunk} -> + data + {:ok, <>} -> + data <> do_receive_data(transport, port, chunk_size) + end + else + {:error, :timeout} -> + {:error, :no_more_data_received} + other -> + IO.inspect Utils.hex_encode other + raise "receive failed" + end + end + + @doc """ + Unpacks (or in other words parses) a message. + """ + def unpack(<< 0x0B :: 4, packages :: 4, status, message :: binary >>) do + response = PackStream.decode(message) + response = if packages == 1, do: List.first(response), else: response + + case status do + @sig_success -> {:success, response} + @sig_record -> {:record, response} + @sig_ignored -> {:ignored, response} + @sig_failure -> {:failure, response} + other -> raise "Couldn't decode #{Utils.hex_encode << other >>}" + end + end +end diff --git a/lib/boltex/pack_stream.ex b/lib/boltex/pack_stream.ex new file mode 100644 index 0000000..ada9667 --- /dev/null +++ b/lib/boltex/pack_stream.ex @@ -0,0 +1,113 @@ +defmodule Boltex.PackStream do + @moduledoc """ + The PackStream implementation for Bolt. + + This module defines a decode function, that will take a binary stream of data + and recursively turn it into a list of Elixir data types. + + It further defines a function for encoding Elixir data types into a binary + stream, using the Boltex.PackStream.Encoder protocol. + """ + + @doc """ + Encodes a list of items into their binary representation. + + As developers tend to be lazy, single objects may be passed. + + ## Examples + + iex> Boltex.PackStream.encode "hello world" + <<0x8B, 0x68, 0x65, 0x6C, 0x6C, 0x6F, 0x20, 0x77, 0x6F, 0x72, 0x6C, 0x64>> + """ + def encode(item), do: Boltex.PackStream.Encoder.encode(item) + + ## + # Decode + @doc "Decodes a binary stream recursively into Elixir data types" + # Null + def decode(<< 0xC0, rest :: binary >>), do: [nil| decode(rest)] + + # Boolean + def decode(<< 0xC3, rest :: binary >>), do: [true | decode(rest)] + def decode(<< 0xC2, rest :: binary >>), do: [false | decode(rest)] + + # Float + def decode(<< 0xC1, number :: float, rest :: binary >>) do + [number | decode(rest)] + end + + # Strings + def decode(<< 0x8 :: 4, str_length :: 4, rest :: bytes >>) do + decode_text rest, str_length + end + def decode(<< 0xD0, str_length, rest :: bytes >>) do + decode_text rest, str_length + end + def decode(<< 0xD1, str_length :: 16, rest :: bytes >>) do + decode_text rest, str_length + end + def decode(<< 0xD2, str_length :: 32, rest :: binary >>) do + decode_text rest, str_length + end + + # Lists + # FIXME: Make sure list size is correct, only collect items within list-size. + def decode(<< 0x9 :: 4, _list_size :: 4, list :: binary >>), do: [decode list] + def decode(<< 0xD4, _list_size :: 8, list :: binary >>), do: [decode list] + def decode(<< 0xD5, _list_size :: 16, list :: binary >>), do: [decode list] + def decode(<< 0xD6, _list_size :: 32, list :: binary >>), do: [decode list] + def decode(<< 0xD7, list :: binary >>) do + position = + for(<>, do: byte) + |> Enum.find_index(&(&1 == 0xDF)) + + << list :: binary-size(position), 0xDF, rest :: binary >> = list + + [decode(list) | decode(rest)] + end + + # Maps + # FIXME: Make sure map size is correct, only collect items within map-size. + def decode(<< 0xA :: 4, _entries :: 4, map :: binary>>), do: decode_map(map) + def decode(<< 0xD8, _entries, map :: binary >>), do: decode_map(map) + def decode(<< 0xD9, _entries :: 16, map :: binary >>), do: decode_map(map) + def decode(<< 0xDA, _entries :: 32, map :: binary >>), do: decode_map(map) + def decode(<< 0xDB, map :: binary >>) do + position = + for(<>, do: byte) + |> Enum.find_index(&(&1 == 0xDF)) + + << map:: binary-size(position), 0xDF, rest :: binary >> = map + + decode_map(map) ++ decode(rest) + end + + # Struct + def decode(<< 0xB :: 4, _struct_size :: 4, sig :: 8>> <> struct) do + [sig: sig, fields: decode(struct)] + end + + def decode(<<0, 0>>), do: [] + def decode(""), do: [] + + # Integers + def decode(<< 0xC8, int, rest :: binary >>), do: [int | decode(rest)] + def decode(<< 0xC9, int :: 16, rest :: binary >>), do: [int | decode(rest)] + def decode(<< 0xCA, int :: 32, rest :: binary >>), do: [int | decode(rest)] + def decode(<< 0xCB, int :: 64, rest :: binary >>), do: [int | decode(rest)] + def decode(<< int, rest :: binary >>), do: [int | decode(rest)] + + defp decode_text(bytes, str_length) do + << string :: binary-size(str_length), rest :: binary >> = bytes + + [string | decode(rest)] + end + + defp decode_map(map) do + decode(map) + |> Enum.chunk(2) + |> Enum.map(&List.to_tuple/1) + |> Enum.into(%{}) + |> List.wrap + end +end diff --git a/lib/boltex/pack_stream/encoder.ex b/lib/boltex/pack_stream/encoder.ex new file mode 100644 index 0000000..bd7dc6e --- /dev/null +++ b/lib/boltex/pack_stream/encoder.ex @@ -0,0 +1,110 @@ +defprotocol Boltex.PackStream.Encoder do + @doc "Encodes an item to its binary PackStream Representation" + def encode(entitiy) +end + +defimpl Boltex.PackStream.Encoder, for: Atom do + def encode(nil), do: << 0xC0 >> + def encode(true), do: << 0xC3 >> + def encode(false), do: << 0xC2 >> + def encode(other) when is_atom(other) do + other + |> Atom.to_string + |> Boltex.PackStream.Encoder.encode + end +end + +defimpl Boltex.PackStream.Encoder, for: Integer do + @int8 -127..-17 + @int16_low -32_768..-129 + @int16_high 128..32_767 + @int32_low -2_147_483_648..-32_769 + @int32_high 32_768..2_147_483_647 + @int64_low -9_223_372_036_854_775_808..-2_147_483_649 + @int64_high 2_147_483_648..9_223_372_036_854_775_807 + + def encode(integer) when integer in -16..127 do + <> + end + def encode(integer) when integer in @int8 do + << 0xC8, integer >> + end + def encode(integer) when integer in @int16_low or integer in @int16_high do + << 0xC9, integer >> + end + def encode(integer) when integer in @int32_low or integer in @int32_high do + << 0xCA, integer >> + end + def encode(integer) when integer in @int64_low or integer in @int64_high do + << 0xCB, integer >> + end +end + +defimpl Boltex.PackStream.Encoder, for: BitString do + def encode(string), do: do_encode(string, byte_size(string)) + + defp do_encode(string, size) when size <= 15 do + << 0x8 :: 4, size :: 4 >> <> string + end + defp do_encode(string, size) when size <= 255 do + << 0xD0, size :: 8 >> <> string + end + defp do_encode(string, size) when size <= 65_535 do + << 0xD1, size :: 16 >> <> string + end + defp do_encode(string, size) when size <= 4_294_967_295 do + << 0xD2, size :: 32 >> <> string + end +end + +defimpl Boltex.PackStream.Encoder, for: List do + def encode(list) do + binary = Enum.map_join list, &Boltex.PackStream.Encoder.encode/1 + + do_encode binary, byte_size(binary) + end + + defp do_encode(binary, list_size) when list_size <= 15 do + << 0x9 :: 4, list_size :: 4 >> <> binary + end + defp do_encode(binary, list_size) when list_size <= 255 do + << 0xD4, list_size :: 8 >> <> binary + end + defp do_encode(binary, list_size) when list_size <= 65_535 do + << 0xD5, list_size :: 16 >> <> binary + end + defp do_encode(binary, list_size) when list_size <= 4_294_967_295 do + << 0xD6, list_size :: 32 >> <> binary + end + defp do_encode(binary, _size) do + << 0xD7 >> <> binary <> <<0xDF>> + end +end + +defimpl Boltex.PackStream.Encoder, for: Map do + def encode(map) do + do_encode map, map_size(map) + end + + defp do_encode(map, size) when size <= 15 do + << 0xA :: 4, size :: 4 >> <> encode_kv(map) + end + defp do_encode(map, size) when size <= 255 do + << 0xD8, size :: 8 >> <> encode_kv(map) + end + defp do_encode(map, size) when size <= 65_535 do + << 0xD9, size :: 16 >> <> encode_kv(map) + end + defp do_encode(map, size) when size <= 4_294_967_295 do + << 0xDA, size :: 32 >> <> encode_kv(map) + end + + defp encode_kv(map) do + Boltex.Utils.reduce_to_binary map, &do_reduce_kv/1 + end + + defp do_reduce_kv({key, value}) do + Boltex.PackStream.Encoder.encode(key) <> + Boltex.PackStream.Encoder.encode(value) + end +end diff --git a/lib/boltex/utils.ex b/lib/boltex/utils.ex new file mode 100644 index 0000000..333217e --- /dev/null +++ b/lib/boltex/utils.ex @@ -0,0 +1,14 @@ +defmodule Boltex.Utils do + def reduce_to_binary(enumerable, transform) do + Enum.reduce enumerable, <<>>, fn(data, acc) -> acc <> transform.(data) end + end + + def hex_encode(bytes) do + for << i <- bytes >>, do: Integer.to_string(i, 16) + end + + def hex_decode(hex_list) do + for(hex <- hex_list, do: Integer.parse(hex, 16) |> elem(0)) + |> reduce_to_binary(&<<&1>>) + end +end -- cgit v1.2.1